Compare commits

..

No commits in common. "master" and "v0.1.0" have entirely different histories.

5 changed files with 61 additions and 93 deletions

View File

@ -5,30 +5,23 @@ Bulk CSV files uploader into Microsoft SQL Server.
## Usage ## Usage
``` ```
NAME: Usage:
csv2db.exe - bulk CSV files uploader into Microsoft SQL Server csv2db [OPTIONS]
USAGE: Application Options:
csv2db.exe [global options] [arguments...] /filepath: CSV file path
/server: server address (default: 127.0.0.1)
VERSION: /database: database name
0.1.1 /table: table name in schema.name format
/fields: field types in [sifdt ] format
GLOBAL OPTIONS: /comma:[,|;|t] CSV file comma character (default: ,)
--comma value CSV file comma character (use 't' for tabs) (default: ",") /create create table
--create create table (default: false) /overwrite overwrite existing table
--database value database name /encoding:[utf8|win1251] CSV file charset (default: utf8)
--dateformat value date format (Go style) (default: "02.01.2006") /skiprows: number of rows to skip
--encoding value CSV file charset ("utf8", "win1251") (default: "utf8") /dateformat: date format (Go style) (default: 02.01.2006)
--fields value list of field types in [sifdt ]+ format /timestampformat: timestamp format (Go style) (default: 02.01.2006 15:04:05)
--filepath value CSV file path /unknowncolumnnames insert to table with unknown column names
--overwrite overwrite existing table (default: false)
--server value database server address (default: "127.0.0.1")
--skiprows value number of rows to skip before read CSV file header (default: 0)
--table value table name in schema.name format
--timestampformat value timestamp format (Go style) (default: "02.01.2006 15:04:05")
--unknowncolumnnames insert to table with unknown column names (default: false)
--version, -v print the version (default: false)
``` ```
## Build ## Build

View File

@ -1,5 +1,5 @@
package main package main
const ( const (
VERSION = "0.1.1" VERSION = "0.1.0"
) )

8
go.mod
View File

@ -3,10 +3,10 @@ module github.com/nxshock/csv2db
go 1.19 go 1.19
require ( require (
github.com/denisenkom/go-mssqldb v0.12.3 github.com/denisenkom/go-mssqldb v0.12.2
github.com/dimchansky/utfbom v1.1.1 github.com/dimchansky/utfbom v1.1.1
github.com/urfave/cli/v2 v2.24.3 github.com/urfave/cli/v2 v2.17.1
golang.org/x/text v0.7.0 golang.org/x/text v0.3.7
) )
require ( require (
@ -15,5 +15,5 @@ require (
github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
golang.org/x/crypto v0.6.0 // indirect golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b // indirect
) )

19
go.sum
View File

@ -5,8 +5,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw=
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
@ -22,28 +22,25 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0= github.com/urfave/cli/v2 v2.17.1 h1:UzjDEw2dJQUE3iRaiNQ1VrVFbyAtKGH3VdkMoHA58V0=
github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/cli/v2 v2.17.1/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0=
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

86
main.go
View File

@ -4,7 +4,6 @@ import (
"bufio" "bufio"
"database/sql" "database/sql"
"encoding/csv" "encoding/csv"
"errors"
"fmt" "fmt"
"io" "io"
"log" "log"
@ -29,9 +28,9 @@ var app = &cli.App{
Required: true, Required: true,
TakesFile: true}, TakesFile: true},
&cli.StringFlag{ &cli.StringFlag{
Name: "server", Name: "server",
Usage: "database server address", Usage: "database server address",
Value: "127.0.0.1"}, DefaultText: "127.0.0.1"},
&cli.StringFlag{ &cli.StringFlag{
Name: "database", Name: "database",
Usage: "database name", Usage: "database name",
@ -45,9 +44,9 @@ var app = &cli.App{
Usage: "list of field types in [sifdt ]+ format", Usage: "list of field types in [sifdt ]+ format",
Required: true}, Required: true},
&cli.StringFlag{ &cli.StringFlag{
Name: "comma", Name: "comma",
Usage: `CSV file comma character (use 't' for tabs)`, Usage: `CSV file comma character (use 't' for tabs)`,
Value: ","}, DefaultText: ","},
&cli.BoolFlag{ &cli.BoolFlag{
Name: "create", Name: "create",
Usage: "create table"}, Usage: "create table"},
@ -56,38 +55,25 @@ var app = &cli.App{
Usage: "overwrite existing table"}, Usage: "overwrite existing table"},
&cli.StringFlag{ &cli.StringFlag{
Name: "encoding", Name: "encoding",
Usage: `CSV file charset ("utf8", "win1251")`, Usage: `CSV file charset ("utf8", "win1251")`},
Value: "utf8"},
&cli.IntFlag{ &cli.IntFlag{
Name: "skiprows", Name: "skiprows",
Usage: "number of rows to skip before read CSV file header"}, Usage: "number of rows to skip before read CSV file header"},
&cli.StringFlag{ &cli.StringFlag{
Name: "dateformat", Name: "dateformat",
Usage: "date format (Go style)", Usage: "date format (Go style)",
Value: "02.01.2006"}, DefaultText: "02.01.2006"},
&cli.StringFlag{ &cli.StringFlag{
Name: "timestampformat", Name: "timestampformat",
Usage: "timestamp format (Go style)", Usage: "timestamp format (Go style)",
Value: "02.01.2006 15:04:05"}, DefaultText: "02.01.2006 15:04:05"},
&cli.BoolFlag{ &cli.BoolFlag{
Name: "unknowncolumnnames", Name: "unknowncolumnnames",
Usage: "insert to table with unknown column names", Usage: "insert to table with unknown column names",
}, },
&cli.BoolFlag{
Name: "withoutheader",
Usage: "file does not contains a header"},
&cli.BoolFlag{
Name: "silent",
Usage: "silens progress output"},
}, },
Action: func(c *cli.Context) error { Action: func(c *cli.Context) error {
if len(strings.Split(c.String("table"), ".")) != 2 { db, err := sql.Open("sqlserver", fmt.Sprintf("sqlserver://%s?database=%s", c.String("server"), c.String("database")))
return errors.New("table name must be in schema.name format")
}
var err error
db, err = sql.Open("sqlserver", fmt.Sprintf("sqlserver://%s?database=%s", c.String("server"), c.String("database")))
if err != nil { if err != nil {
return fmt.Errorf("open database: %v", err) return fmt.Errorf("open database: %v", err)
} }
@ -121,7 +107,6 @@ func main() {
func processReader(c *cli.Context, r io.Reader) error { func processReader(c *cli.Context, r io.Reader) error {
var encoding Encoding var encoding Encoding
err := encoding.UnmarshalText([]byte(c.String("encoding"))) err := encoding.UnmarshalText([]byte(c.String("encoding")))
if err != nil { if err != nil {
return fmt.Errorf("get decoder: %v", c.String("encoding")) return fmt.Errorf("get decoder: %v", c.String("encoding"))
@ -146,32 +131,29 @@ func processReader(c *cli.Context, r io.Reader) error {
reader.TrimLeadingSpace = false reader.TrimLeadingSpace = false
reader.FieldsPerRecord = len(c.String("fields")) reader.FieldsPerRecord = len(c.String("fields"))
if runes := []rune(c.String("comma")); len(runes) > 0 && runes[0] == 't' { if []rune(c.String("comma"))[0] == 't' {
reader.Comma = '\t' reader.Comma = '\t'
} else { } else {
reader.Comma = []rune(c.String("comma"))[0] reader.Comma = []rune(c.String("comma"))[0]
} }
var header []string header, err := reader.Read()
if !c.Bool("withoutheader") { if err != nil {
header, err = reader.Read() return fmt.Errorf("read header: %v", err)
if err != nil { }
return fmt.Errorf("read header: %v", err)
headerList := `"`
for i, v := range header {
if c.String("fields")[i] == ' ' {
continue
} }
headerList := `"` headerList += v
for i, v := range header {
if c.String("fields")[i] == ' ' {
continue
}
headerList += v if i+1 < len(header) {
headerList += `", "`
if i+1 < len(header) { } else {
headerList += `", "` headerList += `"`
} else {
headerList += `"`
}
} }
} }
@ -216,7 +198,7 @@ func processReader(c *cli.Context, r io.Reader) error {
} }
if c.Bool("create") { if c.Bool("create") {
err = createTable(tx, c.String("table"), neededHeader, c.String("fields"), c.Bool("overwrite")) err = createTable(tx, c.String("table"), header, c.String("fields"), c.Bool("overwrite"))
if err != nil { if err != nil {
_ = tx.Rollback() _ = tx.Rollback()
return fmt.Errorf("create table: %v", err) return fmt.Errorf("create table: %v", err)
@ -234,9 +216,7 @@ func processReader(c *cli.Context, r io.Reader) error {
n := 0 n := 0
for { for {
if n%100000 == 0 { if n%100000 == 0 {
if !c.Bool("silent") { fmt.Fprintf(os.Stderr, "Processed %d records...\r", n)
fmt.Fprintf(os.Stderr, "Processed %d records...\r", n)
}
} }
record, err := reader.Read() record, err := reader.Read()
@ -285,9 +265,7 @@ func processReader(c *cli.Context, r io.Reader) error {
_ = tx.Rollback() _ = tx.Rollback()
return fmt.Errorf("calc rows affected: %v", err) return fmt.Errorf("calc rows affected: %v", err)
} }
if !c.Bool("silent") { fmt.Fprintf(os.Stderr, "Processed %d records. \n", rowsAffected)
fmt.Fprintf(os.Stderr, "Processed %d records. \n", rowsAffected)
}
err = stmt.Close() err = stmt.Close()
if err != nil { if err != nil {