diff --git a/README.md b/README.md index 4db5c20..0622264 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Simple key-value store for single-user applications. ## Cons -* Index stored in memory (`map[key hash (28 bytes)]file offset (int64)`) - average 200-250 Mb of RAM per 1M keys +* Index stored in memory (`map[key hash (28 bytes)]file offset (int64)`) * Need to read the whole file on store open to create file index * No way to recover disk space from deleted records * Write/Delete operations block Read and each other operations @@ -20,7 +20,7 @@ Simple key-value store for single-user applications. Create or open existing file: ```go -db, err := zkv.Open("path to file") +db, err := Open("path to file") ``` Data operations: @@ -66,5 +66,6 @@ File is log stuctured list of commands: ## TODO +- [ ] Test [seekable zstd streams](https://github.com/SaveTheRbtz/zstd-seekable-format-go) - [ ] Implement optional separate index file to speedup store initialization - [ ] Add recovery previous state of store file on write error diff --git a/defaults.go b/defaults.go index f22add0..ad476f8 100644 --- a/defaults.go +++ b/defaults.go @@ -9,6 +9,5 @@ import ( var defaultOptions = Options{ MaxParallelReads: runtime.NumCPU(), CompressionLevel: zstd.SpeedDefault, - MemoryBufferSize: 4 * 1024 * 1024, - DiskBufferSize: 1 * 1024 * 1024, + BufferSize: 4 * 1024 * 1024, } diff --git a/options.go b/options.go index 1471014..69b3c28 100644 --- a/options.go +++ b/options.go @@ -9,11 +9,8 @@ type Options struct { // Compression level CompressionLevel zstd.EncoderLevel - // Memory write buffer size in bytes - MemoryBufferSize int - - // Diwk write buffer size in bytes - DiskBufferSize int + // Write buffer size in bytes + BufferSize int } func (o *Options) setDefaults() { diff --git a/zkv.go b/zkv.go index 11d2cee..1bbd4b4 100644 --- a/zkv.go +++ b/zkv.go @@ -1,7 +1,6 @@ package zkv import ( - "bufio" "bytes" "crypto/sha256" "encoding/base64" @@ -125,7 +124,7 @@ func (s *Store) Delete(key interface{}) error { return err } - if s.buffer.Len() > s.options.MemoryBufferSize { + if s.buffer.Len() > s.options.BufferSize { err = s.flush() if err != nil { @@ -210,7 +209,7 @@ func (s *Store) setBytes(keyHash [sha256.Size224]byte, valueBytes []byte) error return err } - if s.buffer.Len() > s.options.MemoryBufferSize { + if s.buffer.Len() > s.options.BufferSize { err = s.flush() if err != nil { @@ -239,7 +238,7 @@ func (s *Store) set(key, value interface{}) error { return err } - if s.buffer.Len() > s.options.MemoryBufferSize { + if s.buffer.Len() > s.options.BufferSize { err = s.flush() if err != nil { @@ -378,9 +377,7 @@ func (s *Store) flush() error { return fmt.Errorf("open store file: %v", err) } - diskWriteBuffer := bufio.NewWriterSize(f, s.options.DiskBufferSize) - - encoder, err := zstd.NewWriter(diskWriteBuffer, zstd.WithEncoderLevel(s.options.CompressionLevel)) + encoder, err := zstd.NewWriter(f, zstd.WithEncoderLevel(s.options.CompressionLevel)) if err != nil { f.Close() return fmt.Errorf("open store file: %v", err) @@ -405,12 +402,6 @@ func (s *Store) flush() error { return err } - err = diskWriteBuffer.Flush() - if err != nil { - // TODO: truncate file to previous state - return err - } - err = f.Close() if err != nil { return err diff --git a/zkv_test.go b/zkv_test.go index 2b8005f..a252f9d 100644 --- a/zkv_test.go +++ b/zkv_test.go @@ -165,7 +165,7 @@ func TestBufferBasic(t *testing.T) { const filePath = "TestBuffer.zkv" defer os.Remove(filePath) - db, err := OpenWithOptions(filePath, Options{MemoryBufferSize: 100}) + db, err := OpenWithOptions(filePath, Options{BufferSize: 100}) assert.NoError(t, err) err = db.Set(1, make([]byte, 100)) @@ -190,7 +190,7 @@ func TestBufferRead(t *testing.T) { const recordCount = 100 defer os.Remove(filePath) - db, err := OpenWithOptions(filePath, Options{MemoryBufferSize: 100}) + db, err := OpenWithOptions(filePath, Options{BufferSize: 100}) assert.NoError(t, err) for i := 1; i <= recordCount; i++ {