From fddb1895be7c36661de2e81a3f7f7550f715075f Mon Sep 17 00:00:00 2001 From: nxshock Date: Sun, 9 Oct 2022 17:23:59 +0500 Subject: [PATCH] Switch to urfave/cli app structure --- consts.go | 5 ++ csv.go | 8 +-- go.mod | 10 ++-- go.sum | 17 ++++--- main.go | 145 +++++++++++++++++++++++++++++++++++------------------- sql.go | 8 +-- types.go | 8 +-- zip.go | 6 ++- 8 files changed, 134 insertions(+), 73 deletions(-) create mode 100644 consts.go diff --git a/consts.go b/consts.go new file mode 100644 index 0000000..5fbf589 --- /dev/null +++ b/consts.go @@ -0,0 +1,5 @@ +package main + +const ( + VERSION = "0.1.0" +) diff --git a/csv.go b/csv.go index 1222316..097407c 100644 --- a/csv.go +++ b/csv.go @@ -3,14 +3,16 @@ package main import ( "fmt" "os" + + "github.com/urfave/cli/v2" ) -func processCsvFile(filePath string) error { - f, err := os.Open(opts.FilePath) +func processCsvFile(c *cli.Context, filePath string) error { + f, err := os.Open(c.String("filepath")) if err != nil { return fmt.Errorf("open file: %v", err) } defer f.Close() - return processReader(f) + return processReader(c, f) } diff --git a/go.mod b/go.mod index a227e1d..ef03796 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,16 @@ go 1.19 require ( github.com/denisenkom/go-mssqldb v0.12.2 - golang.org/x/text v0.3.7 github.com/dimchansky/utfbom v1.1.1 + github.com/urfave/cli/v2 v2.17.1 + golang.org/x/text v0.3.7 ) require ( + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/jessevdk/go-flags v1.5.0 - golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect - golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b // indirect ) diff --git a/go.sum b/go.sum index 90a6cdb..be072b5 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw= @@ -13,26 +15,27 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli/v2 v2.17.1 h1:UzjDEw2dJQUE3iRaiNQ1VrVFbyAtKGH3VdkMoHA58V0= +github.com/urfave/cli/v2 v2.17.1/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0= +golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 h1:ohgcoMbSofXygzo6AD2I1kz3BFmW1QArPYTtwEM3UXc= -golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/main.go b/main.go index 1cd7988..decfe9a 100644 --- a/main.go +++ b/main.go @@ -11,26 +11,87 @@ import ( "path/filepath" "strings" - _ "github.com/denisenkom/go-mssqldb" mssql "github.com/denisenkom/go-mssqldb" - "github.com/jessevdk/go-flags" + "github.com/urfave/cli/v2" ) var db *sql.DB -var opts struct { - FilePath string `long:"filepath" description:"CSV file path" required:"true"` - ServerAddress string `long:"server" description:"server address" default:"127.0.0.1"` - DatabaseName string `long:"database" description:"database name" required:"true"` - TableName string `long:"table" description:"table name in schema.name format" required:"true"` - FieldTypes string `long:"fields" description:"field types in [sifdt ] format" required:"true"` - Comma string `long:"comma" description:"CSV file comma character" choice:"," choice:";" choice:"t" default:","` - CreateTable bool `long:"create" description:"create table"` - OverwriteTable bool `long:"overwrite" description:"overwrite existing table"` - Encoding string `long:"encoding" description:"CSV file charset" choice:"utf8" choice:"win1251" default:"utf8"` - SkipRows int `long:"skiprows" description:"number of rows to skip"` - DateFormat string `long:"dateformat" description:"date format (Go style)" default:"02.01.2006"` - TimestampFormat string `long:"timestampformat" description:"timestamp format (Go style)" default:"02.01.2006 15:04:05"` - UnknownColumnNames bool `long:"unknowncolumnnames" description:"insert to table with unknown column names"` + +var app = &cli.App{ + Usage: "bulk CSV files uploader into Microsoft SQL Server", + HideHelp: true, + Version: VERSION, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filepath", + Usage: "CSV file path", + Required: true, + TakesFile: true}, + &cli.StringFlag{ + Name: "server", + Usage: "database server address", + DefaultText: "127.0.0.1"}, + &cli.StringFlag{ + Name: "database", + Usage: "database name", + Required: true}, + &cli.StringFlag{ + Name: "table", + Usage: "table name in schema.name format", + Required: true}, + &cli.StringFlag{ + Name: "fields", + Usage: "list of field types in [sifdt ]+ format", + Required: true}, + &cli.StringFlag{ + Name: "comma", + Usage: `CSV file comma character (use 't' for tabs)`, + DefaultText: ","}, + &cli.BoolFlag{ + Name: "create", + Usage: "create table"}, + &cli.BoolFlag{ + Name: "overwrite", + Usage: "overwrite existing table"}, + &cli.StringFlag{ + Name: "encoding", + Usage: `CSV file charset ("utf8", "win1251")`}, + &cli.IntFlag{ + Name: "skiprows", + Usage: "number of rows to skip before read CSV file header"}, + &cli.StringFlag{ + Name: "dateformat", + Usage: "date format (Go style)", + DefaultText: "02.01.2006"}, + &cli.StringFlag{ + Name: "timestampformat", + Usage: "timestamp format (Go style)", + DefaultText: "02.01.2006 15:04:05"}, + &cli.BoolFlag{ + Name: "unknowncolumnnames", + Usage: "insert to table with unknown column names", + }, + }, + Action: func(c *cli.Context) error { + db, err := sql.Open("sqlserver", fmt.Sprintf("sqlserver://%s?database=%s", c.String("server"), c.String("database"))) + if err != nil { + return fmt.Errorf("open database: %v", err) + } + defer db.Close() + + filePath := c.String("filepath") + switch strings.ToLower(filepath.Ext(filePath)) { + case ".zip": + err = processZipFile(c, filePath) + case ".csv": + err = processCsvFile(c, filePath) + } + if err != nil { + return err + } + + return nil + }, } func init() { @@ -38,43 +99,27 @@ func init() { } func main() { - _, err := flags.Parse(&opts) - if err != nil { - os.Exit(1) - } - - db, err = sql.Open("sqlserver", fmt.Sprintf("sqlserver://%s?database=%s", opts.ServerAddress, opts.DatabaseName)) - if err != nil { - log.Fatalln(fmt.Errorf("open database: %v", err)) - } - defer db.Close() - - switch strings.ToLower(filepath.Ext(opts.FilePath)) { - case ".zip": - err = processZipFile(opts.FilePath) - case ".csv": - err = processCsvFile(opts.FilePath) - } + err := app.Run(os.Args) if err != nil { log.Fatalln(err) } } -func processReader(r io.Reader) error { +func processReader(c *cli.Context, r io.Reader) error { var encoding Encoding - err := encoding.UnmarshalText([]byte(opts.Encoding)) + err := encoding.UnmarshalText([]byte(c.String("encoding"))) if err != nil { - return fmt.Errorf("get decoder: %v", opts.Encoding) + return fmt.Errorf("get decoder: %v", c.String("encoding")) } decoder, err := encoding.Translate(r) if err != nil { - return fmt.Errorf("enable decoder: %v", opts.Encoding) + return fmt.Errorf("enable decoder: %v", c.String("encoding")) } bufReader := bufio.NewReaderSize(decoder, 4*1024*1024) - for i := 0; i < opts.SkipRows; i++ { + for i := 0; i < c.Int("skiprows"); i++ { _, _, err := bufReader.ReadLine() if err != nil { return fmt.Errorf("skip rows: %v", err) @@ -84,12 +129,12 @@ func processReader(r io.Reader) error { reader := csv.NewReader(bufReader) reader.LazyQuotes = true reader.TrimLeadingSpace = false - reader.FieldsPerRecord = len(opts.FieldTypes) + reader.FieldsPerRecord = len(c.String("fields")) - if []rune(opts.Comma)[0] == 't' { + if []rune(c.String("comma"))[0] == 't' { reader.Comma = '\t' } else { - reader.Comma = []rune(opts.Comma)[0] + reader.Comma = []rune(c.String("comma"))[0] } header, err := reader.Read() @@ -99,7 +144,7 @@ func processReader(r io.Reader) error { headerList := `"` for i, v := range header { - if opts.FieldTypes[i] == ' ' { + if c.String("fields")[i] == ' ' { continue } @@ -114,8 +159,8 @@ func processReader(r io.Reader) error { var neededHeader []string - if opts.UnknownColumnNames { - sql := fmt.Sprintf("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA + '.' + TABLE_NAME = '%s' ORDER BY ORDINAL_POSITION", opts.TableName) + if c.Bool("unknowncolumnnames") { + sql := fmt.Sprintf("SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA + '.' + TABLE_NAME = '%s' ORDER BY ORDINAL_POSITION", c.String("table")) rows, err := db.Query(sql) if err != nil { return fmt.Errorf("get column names from database: %v", err) @@ -135,7 +180,7 @@ func processReader(r io.Reader) error { } } else { for i, v := range header { - if opts.FieldTypes[i] == ' ' { + if c.String("fields")[i] == ' ' { continue } @@ -152,15 +197,15 @@ func processReader(r io.Reader) error { return fmt.Errorf("start transaction: %v", err) } - if opts.CreateTable { - err = createTable(tx, header, opts.FieldTypes) + if c.Bool("create") { + err = createTable(tx, c.String("table"), header, c.String("fields"), c.Bool("overwrite")) if err != nil { _ = tx.Rollback() return fmt.Errorf("create table: %v", err) } } - sql := mssql.CopyIn(opts.TableName, mssql.BulkOptions{Tablock: true}, neededHeader...) + sql := mssql.CopyIn(c.String("table"), mssql.BulkOptions{Tablock: true}, neededHeader...) stmt, err := tx.Prepare(sql) if err != nil { @@ -186,7 +231,7 @@ func processReader(r io.Reader) error { for i, v := range record { var fieldType FieldType - err = fieldType.UnmarshalText([]byte{opts.FieldTypes[i]}) + err = fieldType.UnmarshalText([]byte{c.String("fields")[i]}) if err != nil { return fmt.Errorf("get record type: %v", err) } @@ -194,7 +239,7 @@ func processReader(r io.Reader) error { continue } - parsedValue, err := fieldType.ParseValue(v) + parsedValue, err := fieldType.ParseValue(c, v) if err != nil { return fmt.Errorf("parse value: %v", err) } diff --git a/sql.go b/sql.go index f04afd9..1f7e8ce 100644 --- a/sql.go +++ b/sql.go @@ -5,15 +5,15 @@ import ( "fmt" ) -func createTable(tx *sql.Tx, header []string, fieldTypes string) error { - if opts.OverwriteTable { - _, err := tx.Exec(fmt.Sprintf("IF object_id('%s', 'U') IS NOT NULL DROP TABLE %s", opts.TableName, opts.TableName)) +func createTable(tx *sql.Tx, tableName string, header []string, fieldTypes string, overWrite bool) error { + if overWrite { + _, err := tx.Exec(fmt.Sprintf("IF object_id('%s', 'U') IS NOT NULL DROP TABLE %s", tableName, tableName)) if err != nil { return fmt.Errorf("drop table: %v", err) } } - sql := fmt.Sprintf("CREATE TABLE %s (", opts.TableName) + sql := fmt.Sprintf("CREATE TABLE %s (", tableName) for i, v := range header { var fieldType FieldType diff --git a/types.go b/types.go index 9a261d1..1f1bd32 100644 --- a/types.go +++ b/types.go @@ -5,6 +5,8 @@ import ( "strconv" "strings" "time" + + "github.com/urfave/cli/v2" ) type FieldType int @@ -19,7 +21,7 @@ const ( Timestamp ) -func (ft FieldType) ParseValue(s string) (any, error) { +func (ft FieldType) ParseValue(c *cli.Context, s string) (any, error) { s = strings.TrimSpace(s) if s == "" { @@ -34,9 +36,9 @@ func (ft FieldType) ParseValue(s string) (any, error) { case Float: return strconv.ParseFloat(strings.ReplaceAll(s, ",", "."), 64) case Date: - return time.Parse(opts.DateFormat, s) + return time.Parse(c.String("dateformat"), s) case Timestamp: - return time.Parse(opts.TimestampFormat, s) + return time.Parse(c.String("timestampformat"), s) } return nil, fmt.Errorf("unknown type id = %d", ft) diff --git a/zip.go b/zip.go index c190793..893d768 100644 --- a/zip.go +++ b/zip.go @@ -3,9 +3,11 @@ package main import ( "archive/zip" "fmt" + + "github.com/urfave/cli/v2" ) -func processZipFile(filePath string) error { +func processZipFile(c *cli.Context, filePath string) error { r, err := zip.OpenReader(filePath) if err != nil { return fmt.Errorf("open ZIP file: %v", err) @@ -21,5 +23,5 @@ func processZipFile(filePath string) error { } defer zipFileReader.Close() - return processReader(zipFileReader) + return processReader(c, zipFileReader) }