README.md

@@ -10,11 +10,10 @@ Simple key-value store for single-user applications.
 
 ## Cons
 
-* Index stored in memory (`map[key hash (28 bytes)]file offset (int64)`)
+* Index stored in memory (`map[key hash (28 bytes)]file offset (int64)`) - average 200-250 Mb of RAM per 1M keys
-* Need to read the whole file on store open to create file index (you can use index file options to avoid this)
+* Need to read the whole file on store open to create file index
 * No way to recover disk space from deleted records
 * Write/Delete operations block Read and each other operations
-* Need to decode whole file until stored value
 
 ## Usage
 
@@ -48,28 +47,6 @@ err = db.Flush()
 err = db.Backup("new/file/path")
 ```
 
-## Store options
-
-```go
-type Options struct {
-	// Maximum number of concurrent reads
-	MaxParallelReads int
-
-	// Compression level
-	CompressionLevel zstd.EncoderLevel
-
-	// Memory write buffer size in bytes
-	MemoryBufferSize int
-
-	// Disk write buffer size in bytes
-	DiskBufferSize int
-
-	// Use index file
-	UseIndexFile bool
-}
-
-```
-
 ## File structure
 
 Record is `encoding/gob` structure:
@@ -87,14 +64,7 @@ File is log stuctured list of commands:
 | Length | Record body bytes length | int64    |
 | Body   | Gob-encoded record       | variable |
 
-## Resource consumption
-
-Store requirements:
-
-* around 300 Mb of RAM per 1 million of keys
-* around 34 Mb of disk space for index file per 1 million of keys
-
 ## TODO
 
+- [ ] Implement optional separate index file to speedup store initialization
 - [ ] Add recovery previous state of store file on write error
-- [ ] Add fast file seek to value (add compressed block start position)

defaults.go

@@ -11,7 +11,4 @@ var defaultOptions = Options{
 	CompressionLevel: zstd.SpeedDefault,
 	MemoryBufferSize: 4 * 1024 * 1024,
 	DiskBufferSize:   1 * 1024 * 1024,
-	UseIndexFile:     false,
 }
-
-const indexFileExt = ".idx"

options.go

@@ -12,11 +12,8 @@ type Options struct {
 	// Memory write buffer size in bytes
 	MemoryBufferSize int
 
-	// Disk write buffer size in bytes
+	// Diwk write buffer size in bytes
 	DiskBufferSize int
-
-	// Use index file
-	UseIndexFile bool
 }
 
 func (o *Options) setDefaults() {
@@ -27,12 +24,4 @@ func (o *Options) setDefaults() {
 	if o.CompressionLevel == 0 {
 		o.CompressionLevel = defaultOptions.CompressionLevel
 	}
-
-	if o.MemoryBufferSize == 0 {
-		o.MemoryBufferSize = defaultOptions.MemoryBufferSize
-	}
-
-	if o.DiskBufferSize == 0 {
-		o.DiskBufferSize = defaultOptions.DiskBufferSize
-	}
 }

zkv.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
-	"encoding/gob"
 	"fmt"
 	"io"
 	"os"
@@ -42,16 +41,6 @@ func OpenWithOptions(filePath string, options Options) (*Store, error) {
 		options:          options,
 		readOrderChan:    make(chan struct{}, int(options.MaxParallelReads))}
 
-	if options.UseIndexFile {
-		idxFile, err := os.Open(filePath + indexFileExt)
-		if err == nil {
-			err = gob.NewDecoder(idxFile).Decode(&database.dataOffset)
-			if err == nil {
-				return database, nil
-			}
-		}
-	}
-
 	// restore file data
 	readF, err := os.Open(filePath)
 	if os.IsNotExist(err) {
@@ -92,8 +81,7 @@ func OpenWithOptions(filePath string, options Options) (*Store, error) {
 }
 
 func Open(filePath string) (*Store, error) {
-	options := defaultOptions
+	return OpenWithOptions(filePath, defaultOptions)
-	return OpenWithOptions(filePath, options)
 }
 
 func (s *Store) Set(key, value interface{}) error {
@@ -428,20 +416,5 @@ func (s *Store) flush() error {
 		return err
 	}
 
-	// Update index file only on data update
-	if s.options.UseIndexFile && l > 0 {
-		idxBuf := new(bytes.Buffer)
-
-		err = gob.NewEncoder(idxBuf).Encode(s.dataOffset)
-		if err != nil {
-			return err
-		}
-
-		err = os.WriteFile(s.filePath+indexFileExt, idxBuf.Bytes(), 0644)
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }

zkv_test.go

@@ -327,50 +327,5 @@ func TestBackupWithDeletedRecords(t *testing.T) {
 
 	err = db.Close()
 	assert.NoError(t, err)
-}
+
-
-func TestIndexFileBasic(t *testing.T) {
-	const filePath = "TestReadWriteBasic.zkv"
-	const recordCount = 100
-	defer os.Remove(filePath)
-	defer os.Remove(filePath + indexFileExt)
-
-	db, err := OpenWithOptions(filePath, Options{UseIndexFile: true})
-	assert.NoError(t, err)
-
-	for i := 1; i <= recordCount; i++ {
-		err = db.Set(i, i)
-		assert.NoError(t, err)
-	}
-
-	assert.Len(t, db.dataOffset, 0)
-	assert.Len(t, db.bufferDataOffset, recordCount)
-
-	for i := 1; i <= recordCount; i++ {
-		var gotValue int
-
-		err = db.Get(i, &gotValue)
-		assert.NoError(t, err)
-		assert.Equal(t, i, gotValue)
-	}
-
-	err = db.Close()
-	assert.NoError(t, err)
-
-	// try to read
-	db, err = OpenWithOptions(filePath, Options{UseIndexFile: true})
-	assert.NoError(t, err)
-
-	assert.Len(t, db.dataOffset, recordCount)
-
-	for i := 1; i <= recordCount; i++ {
-		var gotValue int
-
-		err = db.Get(i, &gotValue)
-		assert.NoError(t, err)
-		assert.Equal(t, i, gotValue)
-	}
-
-	err = db.Close()
-	assert.NoError(t, err)
 }