nxshock/omq
1
0
Fork 0
Code Issues Pull Requests Projects Releases 4 Wiki Activity

Add buffered disk writes

Browse Source
This commit is contained in:
nxshock 2024-03-05 21:41:27 +05:00
parent 9b9eadfe92
commit 7c6ca4c81b
2 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
export_csv.go
View File

@ -1,6 +1,7 @@
package main package main
import ( import (
"bufio"
"encoding/csv" "encoding/csv"
"fmt" "fmt"
"os" "os"
@ -22,12 +23,14 @@ func (c *CsvExporter) Convert(filePath string, rows chan []any) error {
return err return err
} }
buf := bufio.NewWriterSize(f, 4*1024*1024)
enc, err := c.Encoding.Encoder() enc, err := c.Encoding.Encoder()
if err != nil { if err != nil {
return err return err
} }
w := csv.NewWriter(enc.Writer(f)) w := csv.NewWriter(enc.Writer(buf))
w.Comma = ';' w.Comma = ';'
rowNum := 0 rowNum := 0
@ -51,6 +54,12 @@ func (c *CsvExporter) Convert(filePath string, rows chan []any) error {
return err return err
} }
err = buf.Flush()
if err != nil {
f.Close()
return err
}
err = f.Close() err = f.Close()
if err != nil { if err != nil {
return err return err

11
export_csvzip.go
View File

@ -2,6 +2,7 @@ package main
import ( import (
"archive/zip" "archive/zip"
"bufio"
"encoding/csv" "encoding/csv"
"os" "os"
"path/filepath" "path/filepath"
@ -22,7 +23,9 @@ func (c *CsvZipExporter) Convert(filePath string, rows chan []any) error {
return err return err
} }
z := zip.NewWriter(f) buf := bufio.NewWriterSize(f, 4*1024*1024)
z := zip.NewWriter(buf)
zw, err := z.Create(filepath.Base(filePath) + ".csv") zw, err := z.Create(filepath.Base(filePath) + ".csv")
if err != nil { if err != nil {
@ -65,6 +68,12 @@ func (c *CsvZipExporter) Convert(filePath string, rows chan []any) error {
return err return err
} }
err = buf.Flush()
if err != nil {
f.Close()
return err
}
err = f.Close() err = f.Close()
if err != nil { if err != nil {
return err return err