Refactor storage code and update data structure methods and fix sync …

…issue

README.md

@@ -1,4 +1,7 @@
 # Nodis
+
+English | [简体中文](https://github.com/diiyw/nodis/blob/main/README_zh-cn.md)
+
 A Golang implemented Redis data structure. 
 It is a simple and easy to embed in your application.
 

README_zh-cn.md

@@ -0,0 +1,46 @@
+# Nodis
+
+[English](https://github.com/diiyw/nodis/blob/main/README.md) | 简体中文
+
+Nodis 是一个简单可嵌入到应用中内存数据库，实现Redis的数据结构。
+
+## 支持的类型
+
+- String
+- List
+- Hash
+- Set
+- Sorted Set
+
+## 特点
+
+- 快速可嵌入的
+- 低内存使用，只有热数据才在内存中
+- 快照和WAL存储的支持
+
+## Get Started
+
+```bash
+ go get github.com/diiyw/nodis 
+```
+
+```go
+package main
+
+import "github.com/diiyw/nodis"
+
+func main() {
+	// Create a new Nodis instance
+	opt := nodis.DefaultOptions
+	n := nodis.Open(opt)
+
+	// Set a key-value pair
+	n.Set("key", []byte("value"), 0)
+	n.LPush("list", []byte("value1"))
+}
+
+```
+
+## Note
+
+Nodis 实现了Redis的数据结构. 但是并不是完整的Redis Server服务，它只是可以方便的切入到各自的应用使用

ds/ds.go

@@ -1,8 +1,8 @@
 package ds
 
 type DataStruct interface {
-	Marshal() ([]byte, error)
-	Unmarshal([]byte) error
+	MarshalBinary() ([]byte, error)
+	UnmarshalBinary([]byte) error
 	Lock()
 	Unlock()
 	RLock()

ds/hash/hash.go

@@ -161,13 +161,13 @@ func (s *HashMap) HScan(cursor int, match string, count int) (int, map[string][]
 }
 
 // Marshal the set to bytes
-func (s *HashMap) Marshal() ([]byte, error) {
+func (s *HashMap) MarshalBinary() ([]byte, error) {
 	var data = s.HGetAll()
 	return binary.Marshal(data)
 }
 
 // Unmarshal the set from bytes
-func (s *HashMap) Unmarshal(data []byte) error {
+func (s *HashMap) UnmarshalBinary(data []byte) error {
 	var values map[string][]byte
 	if err := binary.Unmarshal(data, &values); err != nil {
 		return err

ds/list/list.go

@@ -304,13 +304,13 @@ func (l *DoublyLinkedList) LTrim(start, end int) {
 	}
 }
 
-// Marshal returns the byte slice of the list
-func (l *DoublyLinkedList) Marshal() ([]byte, error) {
+// MarshalBinary returns the byte slice of the list
+func (l *DoublyLinkedList) MarshalBinary() ([]byte, error) {
 	return binary.Marshal(l.LRange(0, -1))
 }
 
-// Unmarshal restores the list from the byte slice
-func (l *DoublyLinkedList) Unmarshal(data []byte) error {
+// UnmarshalBinary restores the list from the byte slice
+func (l *DoublyLinkedList) UnmarshalBinary(data []byte) error {
 	var list [][]byte
 	if err := binary.Unmarshal(data, &list); err != nil {
 		return err

ds/set/set.go

@@ -161,13 +161,13 @@ func (s *Set) GetType() ds.DataType {
 	return ds.Set
 }
 
-// Marshal the string to bytes
-func (s *Set) Marshal() ([]byte, error) {
+// MarshalBinary the string to bytes
+func (s *Set) MarshalBinary() ([]byte, error) {
 	return binary.Marshal(s.members)
 }
 
-// Unmarshal the bytes to string
-func (s *Set) Unmarshal(data []byte) error {
+// UnmarshalBinary the bytes to string
+func (s *Set) UnmarshalBinary(data []byte) error {
 	err := binary.Unmarshal(data, &s.members)
 	if err != nil {
 		return err

ds/str/str.go

@@ -35,12 +35,12 @@ func (s *String) Get() []byte {
 	return s.V
 }
 
-// Marshal the string to bytes
-func (s *String) Marshal() ([]byte, error) {
+// MarshalBinary the string to bytes
+func (s *String) MarshalBinary() ([]byte, error) {
 	return binary.Marshal(s.V)
 }
 
-// Unmarshal the bytes to string
-func (s *String) Unmarshal(data []byte) error {
+// UnmarshalBinary the bytes to string
+func (s *String) UnmarshalBinary(data []byte) error {
 	return binary.Unmarshal(data, &s.V)
 }

ds/zset/sorted_set.go

@@ -318,7 +318,7 @@ func (sortedSet *SortedSet) ZIncrBy(member string, score float64) float64 {
 	return element.Score + score
 }
 
-func (sortedSet *SortedSet) Marshal() ([]byte, error) {
+func (sortedSet *SortedSet) MarshalBinary() ([]byte, error) {
 	var m = make(map[string]*Item)
 	sortedSet.dict.Iter(func(key string, value *Item) bool {
 		m[key] = value
@@ -327,7 +327,7 @@ func (sortedSet *SortedSet) Marshal() ([]byte, error) {
 	return binary.Marshal(m)
 }
 
-func (sortedSet *SortedSet) Unmarshal(data []byte) error {
+func (sortedSet *SortedSet) UnmarshalBinary(data []byte) error {
 	var m = make(map[string]*Item)
 	err := binary.Unmarshal(data, &m)
 	if err != nil {

entry.go

@@ -0,0 +1,84 @@
+package nodis
+
+import (
+	"errors"
+	"hash/crc32"
+
+	"github.com/diiyw/nodis/ds"
+	"github.com/diiyw/nodis/ds/hash"
+	"github.com/diiyw/nodis/ds/list"
+	"github.com/diiyw/nodis/ds/set"
+	"github.com/diiyw/nodis/ds/str"
+	"github.com/diiyw/nodis/ds/zset"
+	"github.com/kelindar/binary"
+)
+
+var (
+	ErrCorruptedData = errors.New("corrupted data")
+)
+
+type Entry struct {
+	Key       string
+	Value     ds.DataStruct
+	ExpiredAt int64
+}
+
+type entryBlock struct {
+	Crc32 uint32
+	Type  ds.DataType
+	Body  []byte
+}
+
+// newEntry creates a new entry
+func newEntry(key string, value ds.DataStruct, expiredAt int64) *Entry {
+	return &Entry{
+		Key:       key,
+		Value:     value,
+		ExpiredAt: expiredAt,
+	}
+}
+
+// Marshal marshals the entry
+func (e *Entry) Marshal() ([]byte, error) {
+	var err error
+	data, err := binary.Marshal(e)
+	if err != nil {
+		return nil, err
+	}
+	var buf = make([]byte, len(data)+1, len(data)+1)
+	buf[0] = byte(e.Value.GetType())
+	copy(buf[1:], data)
+	var block = entryBlock{
+		Crc32: crc32.ChecksumIEEE(buf),
+		Type:  e.Value.GetType(),
+		Body:  data,
+	}
+	return binary.Marshal(block)
+}
+
+// Unmarshal unmarshals the entry
+func (e *Entry) Unmarshal(data []byte) error {
+	var block entryBlock
+	if err := binary.Unmarshal(data, &block); err != nil {
+		return err
+	}
+	var buf = make([]byte, len(block.Body)+1, len(block.Body)+1)
+	buf[0] = byte(block.Type)
+	copy(buf[1:], block.Body)
+	if block.Crc32 != crc32.ChecksumIEEE(buf) {
+		return ErrCorruptedData
+	}
+	switch block.Type {
+	case ds.String:
+		e.Value = str.NewString()
+	case ds.ZSet:
+		e.Value = zset.NewSortedSet()
+	case ds.List:
+		e.Value = list.NewDoublyLinkedList()
+	case ds.Hash:
+		e.Value = hash.NewHashMap()
+	case ds.Set:
+		e.Value = set.NewSet()
+	}
+	return binary.Unmarshal(block.Body, e)
+}

hash.go

@@ -105,7 +105,7 @@ func (n *Nodis) HMGet(key string, fields ...string) [][]byte {
 }
 
 func (n *Nodis) HClear(key string) {
-	n.Clear(key)
+	n.Del(key)
 }
 
 func (n *Nodis) HScan(key string, cursor int, match string, count int) (int, map[string][]byte) {

key.go

@@ -2,6 +2,7 @@ package nodis
 
 import (
 	"errors"
+	"log"
 	"path/filepath"
 	"time"
 
@@ -71,10 +72,14 @@ func (n *Nodis) exists(key string) (k *Key, ok bool) {
 		// try get from store
 		v, err := n.store.get(key)
 		if err == nil && len(v) > 0 {
-			d := parseDs(v)
-			if d != nil {
-				n.dataStructs.Put(key, d)
-				k = newKey(d.GetType(), 0)
+			e, err := parseDs(v)
+			if err != nil {
+				log.Println("Parse Datastruct:", err)
+				return
+			}
+			if e != nil {
+				n.dataStructs.Put(key, e.Value)
+				k = newKey(e.Value.GetType(), 0)
 				k.changed = false
 				ok = true
 				n.keys.Put(key, k)