Enable lazy quotes for CSV files

fieldtypes.go

@@ -41,8 +41,10 @@ func (ft FieldType) ParseValue(reader Reader, s string) (any, error) {
 		return s, nil
 	case Integer:
 		return strconv.ParseInt(s, 10, 64)
-	case Float:
+	case Float, Money:
-		return strconv.ParseFloat(strings.ReplaceAll(s, ",", "."), 64)
+		s = strings.ReplaceAll(s, ",", ".")
+		s = strings.ReplaceAll(s, " ", "")
+		return strconv.ParseFloat(s, 64)
 	case Date:
 		if i, ok := reader.(CustomDateParser); ok {
 			t, err := i.ParseDate(s)
@@ -77,7 +79,7 @@ func (ft FieldType) SqlFieldType() string {
 	case Float:
 		return "float"
 	case Money:
-		panic("do not implemented - see https://github.com/denisenkom/go-mssqldb/issues/460") // TODO: https://github.com/denisenkom/go-mssqldb/issues/460
+		return "numeric(15, 2)" // TODO: https://github.com/denisenkom/go-mssqldb/issues/460
 	case Date:
 		return "date"
 	case Timestamp:

options.go

@@ -60,3 +60,14 @@ type Options struct {
 	// Column names list
 	columnNames []string
 }
+
+func (o *Options) fieldCount() int {
+	fCount := 0
+	for i := range o.fieldsTypes {
+		if o.fieldsTypes[i] != ' ' {
+			fCount++
+		}
+	}
+
+	return fCount
+}

readercsv.go

@@ -35,6 +35,7 @@ func newCsvReader(r io.Reader, options *Options) (*CsvReader, error) {
 	re := csv.NewReader(bufReader)
 	re.Comma = options.comma
 	re.FieldsPerRecord = len(options.fieldsTypes)
+	re.LazyQuotes = true
 
 	csvReader := &CsvReader{
 		reader:  re,

readercsv_test.go

@@ -17,7 +17,7 @@ func TestCsvReaderBasic(t *testing.T) {
 		encoding:        "win1251",
 		comma:           rune(";"[0]),
 		skipRows:        3,
-		fieldsTypes:     "s     ttffsssss",
+		fieldsTypes:     "s     ttmmsssss",
 		dateFormat:      "02.01.2006",
 		timestampFormat: "02.01.2006 15:04:05",
 		timezone:        time.Local}
@@ -34,7 +34,41 @@ func TestCsvReaderBasic(t *testing.T) {
 	t2 := time.Date(2023, 03, 20, 0, 0, 0, 0, time.Local)
 	assert.Equal(t, []any{"307814009186", t1, t2, 499.00, 488.52, "522598******7141", "REZE64", "Покупка", "35068281112", "307817403283"}, row)
 
-	row, err = csvReader.GetRow(false)
+	_, err = csvReader.GetRow(false)
+	assert.Equal(t, err, io.EOF)
+
+	err = csvReader.Close()
+	assert.NoError(t, err)
+}
+
+func TestCsvReaderBasicQuotes(t *testing.T) {
+	f, err := os.Open("testdata/csv/9729337841_20032023_084313668.csv")
+	assert.NoError(t, err)
+
+	options := &Options{
+		encoding:        "utf8",
+		comma:           rune(";"[0]),
+		fieldsTypes:     "s             ",
+		dateFormat:      "02.01.2006",
+		timestampFormat: "02.01.2006 15:04:05",
+		timezone:        time.Local}
+
+	csvReader, err := NewCsvReader(f, options)
+	assert.NoError(t, err)
+
+	assert.Equal(t, []string{"Территориальный банк"}, csvReader.GetHeader())
+
+	row, err := csvReader.GetRow(false)
+	assert.NoError(t, err)
+
+	assert.Equal(t, []any{`ПАО "Сбербанк"`}, row)
+
+	row, err = csvReader.GetRow(false)
+	assert.NoError(t, err)
+
+	assert.Equal(t, []any{`ПАО "Сбербанк"`}, row)
+
+	_, err = csvReader.GetRow(false)
 	assert.Equal(t, err, io.EOF)
 
 	err = csvReader.Close()

readerdbf_test.go

@@ -14,7 +14,7 @@ func TestDbfReaderBasic(t *testing.T) {
 	assert.NoError(t, err)
 
 	options := &Options{
-		fieldsTypes: "sssssstdffsss",
+		fieldsTypes: "sssssstdmmsss",
 		timezone:    time.Local,
 		encoding:    "cp866"}
 
@@ -30,7 +30,7 @@ func TestDbfReaderBasic(t *testing.T) {
 	t2 := time.Date(2023, 02, 21, 0, 0, 0, 0, time.Local)
 	assert.Equal(t, []any{"719089383780", "44", "8644", "570000009312", "STOLOVAYA TSPP", "844417", t1, t2, 1757.08, 1713.15, "536829XXXXXX9388", "UM1TS8", "D"}, row)
 
-	row, err = dbfReader.GetRow(false)
+	_, err = dbfReader.GetRow(false)
 	assert.Equal(t, err, io.EOF)
 
 	err = dbfReader.Close()

readerxlsx.go

@@ -109,6 +109,8 @@ func (r *XlsxReader) GetRow(asStrings bool) ([]any, error) {
 		args = append(args, parsedValue)
 	}
 
+	args = append(args, make([]any, r.options.fieldCount()-len(args))...)
+
 	return args, nil
 }
 

readerxlsx_test.go

@@ -29,6 +29,10 @@ func TestXlsxReaderBasic(t *testing.T) {
 	assert.Equal(t, []any{"7710146208", nil, nil, "99386901", "99386901", "304722813269", "TSENTRALNYY TELEGRAF", "780000334079", "780000334079", "10432641", t1, t2, 50.00, 0.80, 49.20, "553691******1214", "026094", "D", "MC OTHER"}, row)
 
 	row, err = xlsxReader.GetRow(false)
+	assert.NoError(t, err)
+	assert.Len(t, row, 19)
+
+	_, err = xlsxReader.GetRow(false)
 	assert.Equal(t, io.EOF, err)
 
 	err = xlsxReader.Close()

testdata/csv/9729337841_20032023_084313668.csv

@@ -0,0 +1,3 @@
+Территориальный банк;ГОСБ;Номер мерчанта;Наименование ТСТ;Номер терминала;Дата операции;Дата ПП;Сумма операции;Сумма расчета;Номер карты;Код авторизации;Тип операции;Доп. информация_1;Доп. информация_2
+"ПАО ""Сбербанк""";Киевское ОСБ;781000815902;WINK;28403560;19.03.2023 17:49:35;20.03.2023 00:00:00;499,00;488,52;522598******7141;REZE64;Покупка;35068281112;307817403283
+ПАО "Сбербанк";Киевское ОСБ;781000815902;WINK;28403560;19.03.2023 17:49:35;20.03.2023 00:00:00;499,00;488,52;522598******7141;REZE64;Покупка;35068281112;307817403283