Compare commits

...

9 Commits

9 changed files with 203 additions and 109 deletions

View File

@ -5,23 +5,30 @@ Bulk CSV files uploader into Microsoft SQL Server.
## Usage ## Usage
``` ```
Usage: NAME:
csv2db [OPTIONS] csv2db.exe - bulk CSV files uploader into Microsoft SQL Server
Application Options: USAGE:
/filepath: CSV file path csv2db.exe [global options] [arguments...]
/server: server address (default: 127.0.0.1)
/database: database name VERSION:
/table: table name in schema.name format 0.1.1
/fields: field types in [sifdt ] format
/comma:[,|;|t] CSV file comma character (default: ,) GLOBAL OPTIONS:
/create create table --comma value CSV file comma character (use 't' for tabs) (default: ",")
/overwrite overwrite existing table --create create table (default: false)
/encoding:[utf8|win1251] CSV file charset (default: utf8) --database value database name
/skiprows: number of rows to skip --dateformat value date format (Go style) (default: "02.01.2006")
/dateformat: date format (Go style) (default: 02.01.2006) --encoding value CSV file charset ("utf8", "win1251") (default: "utf8")
/timestampformat: timestamp format (Go style) (default: 02.01.2006 15:04:05) --fields value list of field types in [sifdt ]+ format
/unknowncolumnnames insert to table with unknown column names --filepath value CSV file path
--overwrite overwrite existing table (default: false)
--server value database server address (default: "127.0.0.1")
--skiprows value number of rows to skip before read CSV file header (default: 0)
--table value table name in schema.name format
--timestampformat value timestamp format (Go style) (default: "02.01.2006 15:04:05")
--unknowncolumnnames insert to table with unknown column names (default: false)
--version, -v print the version (default: false)
``` ```
## Build ## Build

5
consts.go Normal file
View File

@ -0,0 +1,5 @@
package main
const (
VERSION = "0.1.1"
)

8
csv.go
View File

@ -3,14 +3,16 @@ package main
import ( import (
"fmt" "fmt"
"os" "os"
"github.com/urfave/cli/v2"
) )
func processCsvFile(filePath string) error { func processCsvFile(c *cli.Context, filePath string) error {
f, err := os.Open(opts.FilePath) f, err := os.Open(c.String("filepath"))
if err != nil { if err != nil {
return fmt.Errorf("open file: %v", err) return fmt.Errorf("open file: %v", err)
} }
defer f.Close() defer f.Close()
return processReader(f) return processReader(c, f)
} }

12
go.mod
View File

@ -3,15 +3,17 @@ module github.com/nxshock/csv2db
go 1.19 go 1.19
require ( require (
github.com/denisenkom/go-mssqldb v0.12.2 github.com/denisenkom/go-mssqldb v0.12.3
golang.org/x/text v0.3.7
github.com/dimchansky/utfbom v1.1.1 github.com/dimchansky/utfbom v1.1.1
github.com/urfave/cli/v2 v2.24.3
golang.org/x/text v0.7.0
) )
require ( require (
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/jessevdk/go-flags v1.5.0 github.com/russross/blackfriday/v2 v2.1.0 // indirect
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 // indirect golang.org/x/crypto v0.6.0 // indirect
) )

28
go.sum
View File

@ -1,10 +1,12 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw=
github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
@ -13,31 +15,35 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0=
github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 h1:ohgcoMbSofXygzo6AD2I1kz3BFmW1QArPYTtwEM3UXc= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

170
main.go
View File

@ -4,6 +4,7 @@ import (
"bufio" "bufio"
"database/sql" "database/sql"
"encoding/csv" "encoding/csv"
"errors"
"fmt" "fmt"
"io" "io"
"log" "log"
@ -11,26 +12,100 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
_ "github.com/denisenkom/go-mssqldb"
mssql "github.com/denisenkom/go-mssqldb" mssql "github.com/denisenkom/go-mssqldb"
"github.com/jessevdk/go-flags" "github.com/urfave/cli/v2"
) )
var db *sql.DB var db *sql.DB
var opts struct {
FilePath string `long:"filepath" description:"CSV file path" required:"true"` var app = &cli.App{
ServerAddress string `long:"server" description:"server address" default:"127.0.0.1"` Usage: "bulk CSV files uploader into Microsoft SQL Server",
DatabaseName string `long:"database" description:"database name" required:"true"` HideHelp: true,
TableName string `long:"table" description:"table name in schema.name format" required:"true"` Version: VERSION,
FieldTypes string `long:"fields" description:"field types in [sifdt ] format" required:"true"` Flags: []cli.Flag{
Comma string `long:"comma" description:"CSV file comma character" choice:"," choice:";" choice:"t" default:","` &cli.StringFlag{
CreateTable bool `long:"create" description:"create table"` Name: "filepath",
OverwriteTable bool `long:"overwrite" description:"overwrite existing table"` Usage: "CSV file path",
Encoding string `long:"encoding" description:"CSV file charset" choice:"utf8" choice:"win1251" default:"utf8"` Required: true,
SkipRows int `long:"skiprows" description:"number of rows to skip"` TakesFile: true},
DateFormat string `long:"dateformat" description:"date format (Go style)" default:"02.01.2006"` &cli.StringFlag{
TimestampFormat string `long:"timestampformat" description:"timestamp format (Go style)" default:"02.01.2006 15:04:05"` Name: "server",
UnknownColumnNames bool `long:"unknowncolumnnames" description:"insert to table with unknown column names"` Usage: "database server address",
Value: "127.0.0.1"},
&cli.StringFlag{
Name: "database",
Usage: "database name",
Required: true},
&cli.StringFlag{
Name: "table",
Usage: "table name in schema.name format",
Required: true},
&cli.StringFlag{
Name: "fields",
Usage: "list of field types in [sifdt ]+ format",
Required: true},
&cli.StringFlag{
Name: "comma",
Usage: `CSV file comma character (use 't' for tabs)`,
Value: ","},
&cli.BoolFlag{
Name: "create",
Usage: "create table"},
&cli.BoolFlag{
Name: "overwrite",
Usage: "overwrite existing table"},
&cli.StringFlag{
Name: "encoding",
Usage: `CSV file charset ("utf8", "win1251")`,
Value: "utf8"},
&cli.IntFlag{
Name: "skiprows",
Usage: "number of rows to skip before read CSV file header"},
&cli.StringFlag{
Name: "dateformat",
Usage: "date format (Go style)",
Value: "02.01.2006"},
&cli.StringFlag{
Name: "timestampformat",
Usage: "timestamp format (Go style)",
Value: "02.01.2006 15:04:05"},
&cli.BoolFlag{
Name: "unknowncolumnnames",
Usage: "insert to table with unknown column names",
},
&cli.BoolFlag{
Name: "withoutheader",
Usage: "file does not contains a header"},
&cli.BoolFlag{
Name: "silent",
Usage: "silens progress output"},
},
Action: func(c *cli.Context) error {
if len(strings.Split(c.String("table"), ".")) != 2 {
return errors.New("table name must be in schema.name format")
}
var err error
db, err = sql.Open("sqlserver", fmt.Sprintf("sqlserver://%s?database=%s", c.String("server"), c.String("database")))
if err != nil {
return fmt.Errorf("open database: %v", err)
}
defer db.Close()
filePath := c.String("filepath")
switch strings.ToLower(filepath.Ext(filePath)) {
case ".zip":
err = processZipFile(c, filePath)
case ".csv":
err = processCsvFile(c, filePath)
}
if err != nil {
return err
}
return nil
},
} }
func init() { func init() {
@ -38,43 +113,28 @@ func init() {
} }
func main() { func main() {
_, err := flags.Parse(&opts) err := app.Run(os.Args)
if err != nil {
os.Exit(1)
}
db, err = sql.Open("sqlserver", fmt.Sprintf("sqlserver://%s?database=%s", opts.ServerAddress, opts.DatabaseName))
if err != nil {
log.Fatalln(fmt.Errorf("open database: %v", err))
}
defer db.Close()
switch strings.ToLower(filepath.Ext(opts.FilePath)) {
case ".zip":
err = processZipFile(opts.FilePath)
case ".csv":
err = processCsvFile(opts.FilePath)
}
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
} }
func processReader(r io.Reader) error { func processReader(c *cli.Context, r io.Reader) error {
var encoding Encoding var encoding Encoding
err := encoding.UnmarshalText([]byte(opts.Encoding))
err := encoding.UnmarshalText([]byte(c.String("encoding")))
if err != nil { if err != nil {
return fmt.Errorf("get decoder: %v", opts.Encoding) return fmt.Errorf("get decoder: %v", c.String("encoding"))
} }
decoder, err := encoding.Translate(r) decoder, err := encoding.Translate(r)
if err != nil { if err != nil {
return fmt.Errorf("enable decoder: %v", opts.Encoding) return fmt.Errorf("enable decoder: %v", c.String("encoding"))
} }
bufReader := bufio.NewReaderSize(decoder, 4*1024*1024) bufReader := bufio.NewReaderSize(decoder, 4*1024*1024)
for i := 0; i < opts.SkipRows; i++ { for i := 0; i < c.Int("skiprows"); i++ {
_, _, err := bufReader.ReadLine() _, _, err := bufReader.ReadLine()
if err != nil { if err != nil {
return fmt.Errorf("skip rows: %v", err) return fmt.Errorf("skip rows: %v", err)
@ -82,23 +142,26 @@ func processReader(r io.Reader) error {
} }
reader := csv.NewReader(bufReader) reader := csv.NewReader(bufReader)
reader.LazyQuotes = true
reader.TrimLeadingSpace = false reader.TrimLeadingSpace = false
reader.FieldsPerRecord = len(opts.FieldTypes) reader.FieldsPerRecord = len(c.String("fields"))
if []rune(opts.Comma)[0] == 't' { if runes := []rune(c.String("comma")); len(runes) > 0 && runes[0] == 't' {
reader.Comma = '\t' reader.Comma = '\t'
} else { } else {
reader.Comma = []rune(opts.Comma)[0] reader.Comma = []rune(c.String("comma"))[0]
} }
header, err := reader.Read() var header []string
if !c.Bool("withoutheader") {
header, err = reader.Read()
if err != nil { if err != nil {
return fmt.Errorf("read header: %v", err) return fmt.Errorf("read header: %v", err)
} }
headerList := `"` headerList := `"`
for i, v := range header { for i, v := range header {
if opts.FieldTypes[i] == ' ' { if c.String("fields")[i] == ' ' {
continue continue
} }
@ -110,11 +173,12 @@ func processReader(r io.Reader) error {
headerList += `"` headerList += `"`
} }
} }
}
var neededHeader []string var neededHeader []string
if opts.UnknownColumnNames { if c.Bool("unknowncolumnnames") {
sql := fmt.Sprintf("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA + '.' + TABLE_NAME = '%s' ORDER BY ORDINAL_POSITION", opts.TableName) sql := fmt.Sprintf("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA + '.' + TABLE_NAME = '%s' ORDER BY ORDINAL_POSITION", c.String("table"))
rows, err := db.Query(sql) rows, err := db.Query(sql)
if err != nil { if err != nil {
return fmt.Errorf("get column names from database: %v", err) return fmt.Errorf("get column names from database: %v", err)
@ -134,7 +198,7 @@ func processReader(r io.Reader) error {
} }
} else { } else {
for i, v := range header { for i, v := range header {
if opts.FieldTypes[i] == ' ' { if c.String("fields")[i] == ' ' {
continue continue
} }
@ -151,15 +215,15 @@ func processReader(r io.Reader) error {
return fmt.Errorf("start transaction: %v", err) return fmt.Errorf("start transaction: %v", err)
} }
if opts.CreateTable { if c.Bool("create") {
err = createTable(tx, header, opts.FieldTypes) err = createTable(tx, c.String("table"), neededHeader, c.String("fields"), c.Bool("overwrite"))
if err != nil { if err != nil {
_ = tx.Rollback() _ = tx.Rollback()
return fmt.Errorf("create table: %v", err) return fmt.Errorf("create table: %v", err)
} }
} }
sql := mssql.CopyIn(opts.TableName, mssql.BulkOptions{Tablock: true}, neededHeader...) sql := mssql.CopyIn(c.String("table"), mssql.BulkOptions{Tablock: true}, neededHeader...)
stmt, err := tx.Prepare(sql) stmt, err := tx.Prepare(sql)
if err != nil { if err != nil {
@ -170,8 +234,10 @@ func processReader(r io.Reader) error {
n := 0 n := 0
for { for {
if n%100000 == 0 { if n%100000 == 0 {
if !c.Bool("silent") {
fmt.Fprintf(os.Stderr, "Processed %d records...\r", n) fmt.Fprintf(os.Stderr, "Processed %d records...\r", n)
} }
}
record, err := reader.Read() record, err := reader.Read()
if err == io.EOF { if err == io.EOF {
@ -185,7 +251,7 @@ func processReader(r io.Reader) error {
for i, v := range record { for i, v := range record {
var fieldType FieldType var fieldType FieldType
err = fieldType.UnmarshalText([]byte{opts.FieldTypes[i]}) err = fieldType.UnmarshalText([]byte{c.String("fields")[i]})
if err != nil { if err != nil {
return fmt.Errorf("get record type: %v", err) return fmt.Errorf("get record type: %v", err)
} }
@ -193,7 +259,7 @@ func processReader(r io.Reader) error {
continue continue
} }
parsedValue, err := fieldType.ParseValue(v) parsedValue, err := fieldType.ParseValue(c, v)
if err != nil { if err != nil {
return fmt.Errorf("parse value: %v", err) return fmt.Errorf("parse value: %v", err)
} }
@ -219,7 +285,9 @@ func processReader(r io.Reader) error {
_ = tx.Rollback() _ = tx.Rollback()
return fmt.Errorf("calc rows affected: %v", err) return fmt.Errorf("calc rows affected: %v", err)
} }
if !c.Bool("silent") {
fmt.Fprintf(os.Stderr, "Processed %d records. \n", rowsAffected) fmt.Fprintf(os.Stderr, "Processed %d records. \n", rowsAffected)
}
err = stmt.Close() err = stmt.Close()
if err != nil { if err != nil {

8
sql.go
View File

@ -5,15 +5,15 @@ import (
"fmt" "fmt"
) )
func createTable(tx *sql.Tx, header []string, fieldTypes string) error { func createTable(tx *sql.Tx, tableName string, header []string, fieldTypes string, overWrite bool) error {
if opts.OverwriteTable { if overWrite {
_, err := tx.Exec(fmt.Sprintf("IF object_id('%s', 'U') IS NOT NULL DROP TABLE %s", opts.TableName, opts.TableName)) _, err := tx.Exec(fmt.Sprintf("IF object_id('%s', 'U') IS NOT NULL DROP TABLE %s", tableName, tableName))
if err != nil { if err != nil {
return fmt.Errorf("drop table: %v", err) return fmt.Errorf("drop table: %v", err)
} }
} }
sql := fmt.Sprintf("CREATE TABLE %s (", opts.TableName) sql := fmt.Sprintf("CREATE TABLE %s (", tableName)
for i, v := range header { for i, v := range header {
var fieldType FieldType var fieldType FieldType

View File

@ -5,6 +5,8 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/urfave/cli/v2"
) )
type FieldType int type FieldType int
@ -19,7 +21,7 @@ const (
Timestamp Timestamp
) )
func (ft FieldType) ParseValue(s string) (any, error) { func (ft FieldType) ParseValue(c *cli.Context, s string) (any, error) {
s = strings.TrimSpace(s) s = strings.TrimSpace(s)
if s == "" { if s == "" {
@ -34,9 +36,9 @@ func (ft FieldType) ParseValue(s string) (any, error) {
case Float: case Float:
return strconv.ParseFloat(strings.ReplaceAll(s, ",", "."), 64) return strconv.ParseFloat(strings.ReplaceAll(s, ",", "."), 64)
case Date: case Date:
return time.Parse(opts.DateFormat, s) return time.Parse(c.String("dateformat"), s)
case Timestamp: case Timestamp:
return time.Parse(opts.TimestampFormat, s) return time.Parse(c.String("timestampformat"), s)
} }
return nil, fmt.Errorf("unknown type id = %d", ft) return nil, fmt.Errorf("unknown type id = %d", ft)

6
zip.go
View File

@ -3,9 +3,11 @@ package main
import ( import (
"archive/zip" "archive/zip"
"fmt" "fmt"
"github.com/urfave/cli/v2"
) )
func processZipFile(filePath string) error { func processZipFile(c *cli.Context, filePath string) error {
r, err := zip.OpenReader(filePath) r, err := zip.OpenReader(filePath)
if err != nil { if err != nil {
return fmt.Errorf("open ZIP file: %v", err) return fmt.Errorf("open ZIP file: %v", err)
@ -21,5 +23,5 @@ func processZipFile(filePath string) error {
} }
defer zipFileReader.Close() defer zipFileReader.Close()
return processReader(zipFileReader) return processReader(c, zipFileReader)
} }