diff --git a/README.md b/README.md index 5ed2b58..4db5c20 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,10 @@ Simple key-value store for single-user applications. ## Cons -* Index stored in memory (`map[key hash (28 bytes)]file offset (int64)`) -* Need to read the whole file on store open to create file index (you can use index file options to avoid this) +* Index stored in memory (`map[key hash (28 bytes)]file offset (int64)`) - average 200-250 Mb of RAM per 1M keys +* Need to read the whole file on store open to create file index * No way to recover disk space from deleted records * Write/Delete operations block Read and each other operations -* Need to decode whole file until stored value ## Usage @@ -48,28 +47,6 @@ err = db.Flush() err = db.Backup("new/file/path") ``` -## Store options - -```go -type Options struct { - // Maximum number of concurrent reads - MaxParallelReads int - - // Compression level - CompressionLevel zstd.EncoderLevel - - // Memory write buffer size in bytes - MemoryBufferSize int - - // Disk write buffer size in bytes - DiskBufferSize int - - // Use index file - UseIndexFile bool -} - -``` - ## File structure Record is `encoding/gob` structure: @@ -87,14 +64,7 @@ File is log stuctured list of commands: | Length | Record body bytes length | int64 | | Body | Gob-encoded record | variable | -## Resource consumption - -Store requirements: - -* around 300 Mb of RAM per 1 million of keys -* around 34 Mb of disk space for index file per 1 million of keys - ## TODO +- [ ] Implement optional separate index file to speedup store initialization - [ ] Add recovery previous state of store file on write error -- [ ] Add fast file seek to value (add compressed block start position) diff --git a/defaults.go b/defaults.go index d263685..f22add0 100644 --- a/defaults.go +++ b/defaults.go @@ -11,7 +11,4 @@ var defaultOptions = Options{ CompressionLevel: zstd.SpeedDefault, MemoryBufferSize: 4 * 1024 * 1024, DiskBufferSize: 1 * 1024 * 1024, - UseIndexFile: false, } - -const indexFileExt = ".idx" diff --git a/options.go b/options.go index f42f14f..1471014 100644 --- a/options.go +++ b/options.go @@ -12,11 +12,8 @@ type Options struct { // Memory write buffer size in bytes MemoryBufferSize int - // Disk write buffer size in bytes + // Diwk write buffer size in bytes DiskBufferSize int - - // Use index file - UseIndexFile bool } func (o *Options) setDefaults() { @@ -27,12 +24,4 @@ func (o *Options) setDefaults() { if o.CompressionLevel == 0 { o.CompressionLevel = defaultOptions.CompressionLevel } - - if o.MemoryBufferSize == 0 { - o.MemoryBufferSize = defaultOptions.MemoryBufferSize - } - - if o.DiskBufferSize == 0 { - o.DiskBufferSize = defaultOptions.DiskBufferSize - } } diff --git a/zkv.go b/zkv.go index 4706c28..11d2cee 100644 --- a/zkv.go +++ b/zkv.go @@ -5,7 +5,6 @@ import ( "bytes" "crypto/sha256" "encoding/base64" - "encoding/gob" "fmt" "io" "os" @@ -42,16 +41,6 @@ func OpenWithOptions(filePath string, options Options) (*Store, error) { options: options, readOrderChan: make(chan struct{}, int(options.MaxParallelReads))} - if options.UseIndexFile { - idxFile, err := os.Open(filePath + indexFileExt) - if err == nil { - err = gob.NewDecoder(idxFile).Decode(&database.dataOffset) - if err == nil { - return database, nil - } - } - } - // restore file data readF, err := os.Open(filePath) if os.IsNotExist(err) { @@ -92,8 +81,7 @@ func OpenWithOptions(filePath string, options Options) (*Store, error) { } func Open(filePath string) (*Store, error) { - options := defaultOptions - return OpenWithOptions(filePath, options) + return OpenWithOptions(filePath, defaultOptions) } func (s *Store) Set(key, value interface{}) error { @@ -428,20 +416,5 @@ func (s *Store) flush() error { return err } - // Update index file only on data update - if s.options.UseIndexFile && l > 0 { - idxBuf := new(bytes.Buffer) - - err = gob.NewEncoder(idxBuf).Encode(s.dataOffset) - if err != nil { - return err - } - - err = os.WriteFile(s.filePath+indexFileExt, idxBuf.Bytes(), 0644) - if err != nil { - return err - } - } - return nil } diff --git a/zkv_test.go b/zkv_test.go index b10a2a2..2b8005f 100644 --- a/zkv_test.go +++ b/zkv_test.go @@ -327,50 +327,5 @@ func TestBackupWithDeletedRecords(t *testing.T) { err = db.Close() assert.NoError(t, err) -} - -func TestIndexFileBasic(t *testing.T) { - const filePath = "TestReadWriteBasic.zkv" - const recordCount = 100 - defer os.Remove(filePath) - defer os.Remove(filePath + indexFileExt) - - db, err := OpenWithOptions(filePath, Options{UseIndexFile: true}) - assert.NoError(t, err) - - for i := 1; i <= recordCount; i++ { - err = db.Set(i, i) - assert.NoError(t, err) - } - - assert.Len(t, db.dataOffset, 0) - assert.Len(t, db.bufferDataOffset, recordCount) - - for i := 1; i <= recordCount; i++ { - var gotValue int - - err = db.Get(i, &gotValue) - assert.NoError(t, err) - assert.Equal(t, i, gotValue) - } - - err = db.Close() - assert.NoError(t, err) - - // try to read - db, err = OpenWithOptions(filePath, Options{UseIndexFile: true}) - assert.NoError(t, err) - - assert.Len(t, db.dataOffset, recordCount) - - for i := 1; i <= recordCount; i++ { - var gotValue int - - err = db.Get(i, &gotValue) - assert.NoError(t, err) - assert.Equal(t, i, gotValue) - } - - err = db.Close() - assert.NoError(t, err) + }