zkv/zkv_test.go

package zkv

import (
	"bufio"
	"io"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestReadWriteBasic(t *testing.T) {
	const filePath = "TestReadWriteBasic.zkv"
	const recordCount = 100
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	db, err := Open(filePath)
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		err = db.Set(i, i)
		assert.NoError(t, err)
	}

	assert.Len(t, db.dataOffset, 0)
	assert.Len(t, db.bufferDataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)

	// try to read
	db, err = Open(filePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)
}

func TestSmallWrites(t *testing.T) {
	const filePath = "TestSmallWrites.zkv"
	const recordCount = 100
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	for i := 1; i <= recordCount; i++ {
		db, err := Open(filePath)
		assert.NoError(t, err)

		err = db.Set(i, i)
		assert.NoError(t, err)

		err = db.Close()
		assert.NoError(t, err)
	}

	// try to read

	db, err := Open(filePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)
}

func TestDeleteBasic(t *testing.T) {
	const filePath = "TestDeleteBasic.zkv"
	const recordCount = 100
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	db, err := Open(filePath)
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		err = db.Set(i, i)
		assert.NoError(t, err)
	}

	assert.Len(t, db.dataOffset, 0)
	assert.Len(t, db.bufferDataOffset, recordCount)

	err = db.Delete(50)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, 0)
	assert.Len(t, db.bufferDataOffset, recordCount-1)

	var value int
	err = db.Get(50, &value)
	assert.Equal(t, 0, value)
	assert.ErrorIs(t, err, ErrNotExists)

	err = db.Close()
	assert.NoError(t, err)

	// try to read
	db, err = Open(filePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount-1)
	assert.Len(t, db.bufferDataOffset, 0)

	value = 0
	err = db.Get(50, &value)
	assert.Equal(t, 0, value)
	assert.ErrorIs(t, err, ErrNotExists)

	err = db.Close()
	assert.NoError(t, err)
}

func TestBufferBasic(t *testing.T) {
	const filePath = "TestBuffer.zkv"
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	db, err := OpenWithOptions(filePath, Options{MemoryBufferSize: 100})
	assert.NoError(t, err)

	err = db.Set(1, make([]byte, 100))
	assert.NoError(t, err)

	assert.NotEqual(t, 0, db.dataOffset)
	assert.Len(t, db.bufferDataOffset, 0)
	assert.Equal(t, 0, db.buffer.Len())

	var gotValue []byte
	err = db.Get(1, &gotValue)
	assert.NoError(t, err)

	assert.Equal(t, make([]byte, 100), gotValue)

	err = db.Close()
	assert.NoError(t, err)
}

func TestBufferRead(t *testing.T) {
	const filePath = "TestBufferRead.zkv"
	const recordCount = 2
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	db, err := OpenWithOptions(filePath, Options{MemoryBufferSize: 100})
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		err = db.Set(i, i)
		assert.NoError(t, err)
	}

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)

	// try to read
	db, err = Open(filePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)

}

func TestBackupBasic(t *testing.T) {
	const filePath = "TestBackupBasic.zkv"
	const newFilePath = "TestBackupBasic2.zkv"
	const recordCount = 100
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)
	defer os.Remove(newFilePath)
	defer os.Remove(newFilePath + indexFileExt)

	db, err := Open(filePath)
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		err = db.Set(i, i)
		assert.NoError(t, err)
	}

	err = db.Backup(newFilePath)
	assert.NoError(t, err)

	err = db.Close()
	assert.NoError(t, err)

	db, err = Open(newFilePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)

}

func TestBackupWithDeletedRecords(t *testing.T) {
	const filePath = "TestBackupWithDeletedRecords.zkv"
	const newFilePath = "TestBackupWithDeletedRecords2.zkv"
	const recordCount = 100
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)
	defer os.Remove(newFilePath)
	defer os.Remove(newFilePath + indexFileExt)

	db, err := Open(filePath)
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		err = db.Set(i, i)
		assert.NoError(t, err)
	}

	err = db.Flush()
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		if i%2 == 1 {
			continue
		}

		err = db.Delete(i)
		assert.NoError(t, err)
	}

	err = db.Backup(newFilePath)
	assert.NoError(t, err)

	err = db.Close()
	assert.NoError(t, err)

	db, err = Open(newFilePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount/2)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		if i%2 == 0 {
			assert.ErrorIs(t, err, ErrNotExists)
		} else {
			assert.NoError(t, err)
			assert.Equal(t, i, gotValue)
		}
	}

	err = db.Close()
	assert.NoError(t, err)
}

func TestIndexFileBasic(t *testing.T) {
	const filePath = "TestReadWriteBasic.zkv"
	const recordCount = 100
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	db, err := Open(filePath)
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		err = db.Set(i, i)
		assert.NoError(t, err)
	}

	assert.Len(t, db.dataOffset, 0)
	assert.Len(t, db.bufferDataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)

	// try to read
	db, err = Open(filePath)
	assert.NoError(t, err)

	assert.Len(t, db.dataOffset, recordCount)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}

	err = db.Close()
	assert.NoError(t, err)
}

func TestReadBlock(t *testing.T) {
	file, err := os.Open("testdata/TestReadBlock.zkv")
	assert.NoError(t, err)
	defer file.Close()

	r := bufio.NewReader(file)

	line, _, err := readBlock(r)
	assert.Equal(t, []byte{0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x00, 0x99, 0x04, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0xff, 0x81, 0x03, 0x01, 0x01, 0x06, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x01, 0xff, 0x82, 0x00, 0x01, 0x03, 0x01, 0x04, 0x54, 0x79, 0x70, 0x65, 0x01, 0x06, 0x00, 0x01, 0x07, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68, 0x01, 0xff, 0x84, 0x00, 0x01, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x01, 0x0a, 0x00, 0x00, 0x00, 0x19, 0xff, 0x83, 0x01, 0x01, 0x01, 0x09, 0x5b, 0x32, 0x38, 0x5d, 0x75, 0x69, 0x6e, 0x74, 0x38, 0x01, 0xff, 0x84, 0x00, 0x01, 0x06, 0x01, 0x38, 0x00, 0x00, 0x36, 0xff, 0x82, 0x01, 0x01, 0x01, 0x1c, 0xff, 0x90, 0xff, 0xf4, 0x25, 0x15, 0x70, 0x75, 0x5c, 0xff, 0xf4, 0xff, 0xbc, 0xff, 0xf9, 0xff, 0xde, 0xff, 0x93, 0xff, 0xf8, 0x0d, 0x0e, 0x78, 0x5b, 0xff, 0x81, 0xff, 0x95, 0x6e, 0xff, 0xab, 0x4b, 0xff, 0xe8, 0x37, 0xff, 0x97, 0x68, 0x41, 0x3d, 0x01, 0x04, 0x03, 0x04, 0x00, 0x02, 0x00, 0x25, 0xd5, 0x63, 0x21}, line)
	line, _, err = readBlock(r)
	assert.Equal(t, []byte{0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x00, 0x89, 0x04, 0x00, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0xff, 0x81, 0x03, 0x01, 0x01, 0x06, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x01, 0xff, 0x82, 0x00, 0x01, 0x03, 0x01, 0x04, 0x54, 0x79, 0x70, 0x65, 0x01, 0x06, 0x00, 0x01, 0x07, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68, 0x01, 0xff, 0x84, 0x00, 0x01, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x01, 0x0a, 0x00, 0x00, 0x00, 0x19, 0xff, 0x83, 0x01, 0x01, 0x01, 0x09, 0x5b, 0x32, 0x38, 0x5d, 0x75, 0x69, 0x6e, 0x74, 0x38, 0x01, 0xff, 0x84, 0x00, 0x01, 0x06, 0x01, 0x38, 0x00, 0x00, 0x34, 0xff, 0x82, 0x01, 0x01, 0x01, 0x1c, 0xff, 0x84, 0xff, 0x84, 0xff, 0xc1, 0x21, 0x02, 0xff, 0x8b, 0xff, 0xd7, 0x6d, 0xff, 0xd0, 0xff, 0xad, 0x1a, 0x55, 0x14, 0x5c, 0xff, 0xb1, 0x04, 0x37, 0x29, 0x2f, 0x78, 0x18, 0xff, 0xb5, 0xff, 0xe4, 0x56, 0x4e, 0xff, 0x8d, 0x19, 0x46, 0x01, 0x04, 0x03, 0x04, 0x00, 0x04, 0x00, 0x0c, 0x3b, 0xbf, 0x39}, line)
	line, _, err = readBlock(r)
	assert.Equal(t, []byte{0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x00, 0x99, 0x04, 0x00, 0x8b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0xff, 0x81, 0x03, 0x01, 0x01, 0x06, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x01, 0xff, 0x82, 0x00, 0x01, 0x03, 0x01, 0x04, 0x54, 0x79, 0x70, 0x65, 0x01, 0x06, 0x00, 0x01, 0x07, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68, 0x01, 0xff, 0x84, 0x00, 0x01, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x01, 0x0a, 0x00, 0x00, 0x00, 0x19, 0xff, 0x83, 0x01, 0x01, 0x01, 0x09, 0x5b, 0x32, 0x38, 0x5d, 0x75, 0x69, 0x6e, 0x74, 0x38, 0x01, 0xff, 0x84, 0x00, 0x01, 0x06, 0x01, 0x38, 0x00, 0x00, 0x36, 0xff, 0x82, 0x01, 0x01, 0x01, 0x1c, 0x25, 0x79, 0x3e, 0x46, 0x4e, 0xff, 0xac, 0x06, 0x27, 0xff, 0xb1, 0xff, 0xa3, 0xff, 0xaa, 0xff, 0xe3, 0xff, 0xde, 0x37, 0x71, 0x63, 0x72, 0xff, 0x89, 0x0d, 0xff, 0x85, 0x39, 0xff, 0xb5, 0xff, 0xb9, 0xff, 0x8a, 0xff, 0x9e, 0x60, 0xff, 0xad, 0x17, 0x01, 0x04, 0x03, 0x04, 0x00, 0x06, 0x00, 0x52, 0x08, 0x3e, 0x26}, line)
	line, _, err = readBlock(r)
	assert.Equal(t, []byte{0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x00, 0xc9, 0x04, 0x00, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0xff, 0x81, 0x03, 0x01, 0x01, 0x06, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x01, 0xff, 0x82, 0x00, 0x01, 0x03, 0x01, 0x04, 0x54, 0x79, 0x70, 0x65, 0x01, 0x06, 0x00, 0x01, 0x07, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68, 0x01, 0xff, 0x84, 0x00, 0x01, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x01, 0x0a, 0x00, 0x00, 0x00, 0x19, 0xff, 0x83, 0x01, 0x01, 0x01, 0x09, 0x5b, 0x32, 0x38, 0x5d, 0x75, 0x69, 0x6e, 0x74, 0x38, 0x01, 0xff, 0x84, 0x00, 0x01, 0x06, 0x01, 0x38, 0x00, 0x00, 0x3c, 0xff, 0x82, 0x01, 0x01, 0x01, 0x1c, 0xff, 0xbf, 0x25, 0xff, 0xef, 0xff, 0xc8, 0xff, 0x85, 0x2c, 0xff, 0xbf, 0xff, 0xb5, 0xff, 0xad, 0xff, 0xfa, 0xff, 0xaf, 0x1c, 0xff, 0xe7, 0x71, 0xff, 0xfa, 0x36, 0xff, 0x95, 0x1b, 0xff, 0x91, 0xff, 0xab, 0x36, 0xff, 0xcd, 0x7a, 0x33, 0xff, 0xf7, 0xff, 0xec, 0xff, 0xee, 0xff, 0xc1, 0x01, 0x04, 0x03, 0x04, 0x00, 0x08, 0x00, 0xa5, 0x0e, 0x62, 0x53}, line)

	line, _, err = readBlock(r)
	assert.Equal(t, line, []byte{})
	assert.Equal(t, io.EOF, err)
}

func TestRebuildIndex(t *testing.T) {
	const filePath = "TestRebuiltIndex.zkv"
	const recordCount = 4
	defer os.Remove(filePath)
	defer os.Remove(filePath + indexFileExt)

	for i := 1; i <= recordCount; i++ {
		db, err := Open(filePath)
		assert.NoError(t, err)

		err = db.Set(i, i)
		assert.NoError(t, err)

		err = db.Close()
		assert.NoError(t, err)
	}

	db, err := Open(filePath)
	assert.NoError(t, err)

	err = db.RebuildIndex()
	assert.NoError(t, err)

	for i := 1; i <= recordCount; i++ {
		var gotValue int

		err = db.Get(i, &gotValue)
		assert.NoError(t, err)
		assert.Equal(t, i, gotValue)
	}
}