From e7909783b94cd6c998f1ac8dc4fb48890dd37aa5 Mon Sep 17 00:00:00 2001 From: thxCode Date: Fri, 12 Jul 2024 13:05:41 +0800 Subject: [PATCH] feat: support cache Signed-off-by: thxCode --- cache.go | 105 +++++++++++++++++++++++++++++++++++ cmd/gguf-parser/README.md | 2 + cmd/gguf-parser/main.go | 8 +++ file.go | 113 ++++++++++++++++++++++---------------- file_from_distro.go | 28 +++++++++- file_from_remote.go | 26 ++++++++- file_option.go | 63 ++++++++++++++++++++- util/osx/file.go | 19 +++++++ util/stringx/bytes.go | 4 +- util/stringx/sum.go | 85 ++++++++++++++++++++++++++++ 10 files changed, 400 insertions(+), 53 deletions(-) create mode 100644 cache.go create mode 100644 util/stringx/sum.go diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..e14c564 --- /dev/null +++ b/cache.go @@ -0,0 +1,105 @@ +package gguf_parser + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/thxcode/gguf-parser-go/util/json" + "github.com/thxcode/gguf-parser-go/util/osx" + "github.com/thxcode/gguf-parser-go/util/stringx" +) + +var ( + ErrGGUFFileCacheDisabled = errors.New("GGUF file cache disabled") + ErrGGUFFileCacheMissed = errors.New("GGUF file cache missed") + ErrGGUFFileCacheCorrupted = errors.New("GGUF file cache corrupted") +) + +type GGUFFileCache string + +func (c GGUFFileCache) getKeyPath(key string) string { + k := stringx.SumByFNV64a(key) + p := filepath.Join(string(c), k[:1], k) + return p +} + +func (c GGUFFileCache) Get(key string, exp time.Duration) (*GGUFFile, error) { + if c == "" { + return nil, ErrGGUFFileCacheDisabled + } + + if key == "" { + return nil, ErrGGUFFileCacheMissed + } + + p := c.getKeyPath(key) + if !osx.Exists(p, func(stat os.FileInfo) bool { + if !stat.Mode().IsRegular() { + return false + } + return time.Since(stat.ModTime()) < exp + }) { + return nil, ErrGGUFFileCacheMissed + } + + var gf GGUFFile + { + bs, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("GGUF file cache get: %w", err) + } + if err = json.Unmarshal(bs, &gf); err != nil { + return nil, fmt.Errorf("GGUF file cache get: %w", err) + } + } + if len(gf.Header.MetadataKV) == 0 || len(gf.TensorInfos) == 0 { + _ = os.Remove(p) + return nil, ErrGGUFFileCacheCorrupted + } + + return &gf, nil +} + +func (c GGUFFileCache) Put(key string, gf *GGUFFile) error { + if c == "" { + return ErrGGUFFileCacheDisabled + } + + if key == "" || gf == nil { + return nil + } + + bs, err := json.Marshal(gf) + if err != nil { + return fmt.Errorf("GGUF file cache put: %w", err) + } + + p := c.getKeyPath(key) + if err = osx.WriteFile(p, bs, 0o600); err != nil { + return fmt.Errorf("GGUF file cache put: %w", err) + } + return nil +} + +func (c GGUFFileCache) Delete(key string) error { + if c == "" { + return ErrGGUFFileCacheDisabled + } + + if key == "" { + return ErrGGUFFileCacheMissed + } + + p := c.getKeyPath(key) + if !osx.ExistsFile(p) { + return ErrGGUFFileCacheMissed + } + + if err := os.Remove(p); err != nil { + return fmt.Errorf("GGUF file cache delete: %w", err) + } + return nil +} diff --git a/cmd/gguf-parser/README.md b/cmd/gguf-parser/README.md index 9bf3b75..033687c 100644 --- a/cmd/gguf-parser/README.md +++ b/cmd/gguf-parser/README.md @@ -63,6 +63,8 @@ Usage of gguf-parser ...: [DEPRECATED, use --hf-repo instead] Repository of HuggingFace which the GGUF file store, e.g. NousResearch/Hermes-2-Theta-Llama-3-8B-GGUF, works with --file. -skip-architecture Skip to display architecture metadata. + -skip-cache + Skip cache, works with --url/--hf-*/--ms-*/--ol-*, default is caching the read result. -skip-dns-cache Skip DNS cache, works with --url/--hf-*/--ms-*/--ol-*, default is caching the DNS lookup result. -skip-estimate diff --git a/cmd/gguf-parser/main.go b/cmd/gguf-parser/main.go index d15d5c6..ac42734 100644 --- a/cmd/gguf-parser/main.go +++ b/cmd/gguf-parser/main.go @@ -42,6 +42,7 @@ func main() { skipTLSVerify bool skipDNSCache bool skipRangDownloadDetect bool + skipCache bool // estimate options ctxSize = -1 inMaxCtxSize bool @@ -112,6 +113,9 @@ func main() { fs.BoolVar(&skipRangDownloadDetect, "skip-rang-download-detect", skipRangDownloadDetect, "Skip range download detect, "+ "works with --url/--hf-*/--ms-*/--ol-*, "+ "default is detecting the range download support.") + fs.BoolVar(&skipCache, "skip-cache", skipCache, "Skip cache, "+ + "works with --url/--hf-*/--ms-*/--ol-*, "+ + "default is caching the read result.") fs.IntVar(&ctxSize, "ctx-size", ctxSize, "Specify the size of prompt context, "+ "which is used to estimate the usage, "+ "default is equal to the model's maximum context size.") @@ -178,6 +182,7 @@ func main() { ropts := []GGUFReadOption{ SkipLargeMetadata(), UseMMap(), + UseCache(), } if debug { ropts = append(ropts, UseDebug()) @@ -194,6 +199,9 @@ func main() { if skipRangDownloadDetect { ropts = append(ropts, SkipRangeDownloadDetection()) } + if skipCache { + ropts = append(ropts, SkipCache()) + } eopts := []LLaMACppUsageEstimateOption{ WithCacheValueType(GGMLTypeF16), diff --git a/file.go b/file.go index 6f8841d..94b0600 100644 --- a/file.go +++ b/file.go @@ -12,6 +12,7 @@ import ( "golang.org/x/exp/constraints" + "github.com/thxcode/gguf-parser-go/util/anyx" "github.com/thxcode/gguf-parser-go/util/bytex" "github.com/thxcode/gguf-parser-go/util/funcx" "github.com/thxcode/gguf-parser-go/util/osx" @@ -557,91 +558,109 @@ func (kv GGUFMetadataKV) ValueUint8() uint8 { if kv.ValueType != GGUFMetadataValueTypeUint8 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(uint8) + return anyx.Number[uint8](kv.Value) } func (kv GGUFMetadataKV) ValueInt8() int8 { if kv.ValueType != GGUFMetadataValueTypeInt8 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(int8) + return anyx.Number[int8](kv.Value) } func (kv GGUFMetadataKV) ValueUint16() uint16 { if kv.ValueType != GGUFMetadataValueTypeUint16 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(uint16) + return anyx.Number[uint16](kv.Value) } func (kv GGUFMetadataKV) ValueInt16() int16 { if kv.ValueType != GGUFMetadataValueTypeInt16 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(int16) + return anyx.Number[int16](kv.Value) } func (kv GGUFMetadataKV) ValueUint32() uint32 { if kv.ValueType != GGUFMetadataValueTypeUint32 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(uint32) + return anyx.Number[uint32](kv.Value) } func (kv GGUFMetadataKV) ValueInt32() int32 { if kv.ValueType != GGUFMetadataValueTypeInt32 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(int32) + return anyx.Number[int32](kv.Value) } func (kv GGUFMetadataKV) ValueFloat32() float32 { if kv.ValueType != GGUFMetadataValueTypeFloat32 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(float32) + return anyx.Number[float32](kv.Value) } func (kv GGUFMetadataKV) ValueBool() bool { if kv.ValueType != GGUFMetadataValueTypeBool { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(bool) + return anyx.Bool(kv.Value) } func (kv GGUFMetadataKV) ValueString() string { if kv.ValueType != GGUFMetadataValueTypeString { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(string) + return anyx.String(kv.Value) } func (kv GGUFMetadataKV) ValueArray() GGUFMetadataKVArrayValue { if kv.ValueType != GGUFMetadataValueTypeArray { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(GGUFMetadataKVArrayValue) + switch t := kv.Value.(type) { + case GGUFMetadataKVArrayValue: + return t + case map[string]any: + return GGUFMetadataKVArrayValue{ + Type: anyx.Number[GGUFMetadataValueType](t["type"]), + Len: anyx.Number[uint64](t["len"]), + Array: func() []any { + if vv, ok := t["array"].([]any); ok { + return vv + } + return nil + }(), + StartOffset: anyx.Number[int64](t["startOffset"]), + Size: anyx.Number[int64](t["size"]), + } + default: + panic(fmt.Errorf("invalid type: %T", kv.Value)) + } } func (kv GGUFMetadataKV) ValueUint64() uint64 { if kv.ValueType != GGUFMetadataValueTypeUint64 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(uint64) + return anyx.Number[uint64](kv.Value) } func (kv GGUFMetadataKV) ValueInt64() int64 { if kv.ValueType != GGUFMetadataValueTypeInt64 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(int64) + return anyx.Number[int64](kv.Value) } func (kv GGUFMetadataKV) ValueFloat64() float64 { if kv.ValueType != GGUFMetadataValueTypeFloat64 { panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - return kv.Value.(float64) + return anyx.Number[float64](kv.Value) } // ValueNumeric returns the numeric values of the GGUFMetadataKV, @@ -654,28 +673,19 @@ func (kv GGUFMetadataKV) ValueFloat64() float64 { func ValueNumeric[T constraints.Integer | constraints.Float](kv GGUFMetadataKV) T { switch kv.ValueType { case GGUFMetadataValueTypeUint8: - return T(kv.Value.(uint8)) case GGUFMetadataValueTypeInt8: - return T(kv.Value.(int8)) case GGUFMetadataValueTypeUint16: - return T(kv.Value.(int16)) case GGUFMetadataValueTypeInt16: - return T(kv.Value.(int16)) case GGUFMetadataValueTypeUint32: - return T(kv.Value.(uint32)) case GGUFMetadataValueTypeInt32: - return T(kv.Value.(int32)) case GGUFMetadataValueTypeFloat32: - return T(kv.Value.(float32)) case GGUFMetadataValueTypeUint64: - return T(kv.Value.(uint64)) case GGUFMetadataValueTypeInt64: - return T(kv.Value.(int64)) case GGUFMetadataValueTypeFloat64: - return T(kv.Value.(float64)) default: + panic(fmt.Errorf("invalid type: %v", kv.ValueType)) } - panic(fmt.Errorf("invalid type: %v", kv.ValueType)) + return anyx.Number[T](kv.Value) } func (av GGUFMetadataKVArrayValue) ValuesUint8() []uint8 { @@ -684,7 +694,7 @@ func (av GGUFMetadataKVArrayValue) ValuesUint8() []uint8 { } v := make([]uint8, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(uint8) + v[i] = anyx.Number[uint8](av.Array[i]) } return v } @@ -695,7 +705,7 @@ func (av GGUFMetadataKVArrayValue) ValuesInt8() []int8 { } v := make([]int8, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(int8) + v[i] = anyx.Number[int8](av.Array[i]) } return v } @@ -706,7 +716,7 @@ func (av GGUFMetadataKVArrayValue) ValuesUint16() []uint16 { } v := make([]uint16, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(uint16) + v[i] = anyx.Number[uint16](av.Array[i]) } return v } @@ -717,7 +727,7 @@ func (av GGUFMetadataKVArrayValue) ValuesInt16() []int16 { } v := make([]int16, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(int16) + v[i] = anyx.Number[int16](av.Array[i]) } return v } @@ -728,7 +738,7 @@ func (av GGUFMetadataKVArrayValue) ValuesUint32() []uint32 { } v := make([]uint32, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(uint32) + v[i] = anyx.Number[uint32](av.Array[i]) } return v } @@ -739,7 +749,7 @@ func (av GGUFMetadataKVArrayValue) ValuesInt32() []int32 { } v := make([]int32, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(int32) + v[i] = anyx.Number[int32](av.Array[i]) } return v } @@ -750,7 +760,7 @@ func (av GGUFMetadataKVArrayValue) ValuesFloat32() []float32 { } v := make([]float32, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(float32) + v[i] = anyx.Number[float32](av.Array[i]) } return v } @@ -761,7 +771,7 @@ func (av GGUFMetadataKVArrayValue) ValuesBool() []bool { } v := make([]bool, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(bool) + v[i] = anyx.Bool(av.Array[i]) } return v } @@ -772,7 +782,7 @@ func (av GGUFMetadataKVArrayValue) ValuesString() []string { } v := make([]string, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(string) + v[i] = anyx.String(av.Array[i]) } return v } @@ -783,7 +793,25 @@ func (av GGUFMetadataKVArrayValue) ValuesArray() []GGUFMetadataKVArrayValue { } v := make([]GGUFMetadataKVArrayValue, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(GGUFMetadataKVArrayValue) + switch t := av.Array[i].(type) { + case GGUFMetadataKVArrayValue: + v[i] = t + case map[string]any: + v[i] = GGUFMetadataKVArrayValue{ + Type: anyx.Number[GGUFMetadataValueType](t["type"]), + Len: anyx.Number[uint64](t["len"]), + Array: func() []any { + if vv, ok := t["array"].([]any); ok { + return vv + } + return nil + }(), + StartOffset: anyx.Number[int64](t["startOffset"]), + Size: anyx.Number[int64](t["size"]), + } + default: + panic(fmt.Errorf("invalid type: %T", av.Array[i])) + } } return v } @@ -794,7 +822,7 @@ func (av GGUFMetadataKVArrayValue) ValuesUint64() []uint64 { } v := make([]uint64, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(uint64) + v[i] = anyx.Number[uint64](av.Array[i]) } return v } @@ -805,7 +833,7 @@ func (av GGUFMetadataKVArrayValue) ValuesInt64() []int64 { } v := make([]int64, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(int64) + v[i] = anyx.Number[int64](av.Array[i]) } return v } @@ -816,7 +844,7 @@ func (av GGUFMetadataKVArrayValue) ValuesFloat64() []float64 { } v := make([]float64, av.Len) for i := uint64(0); i < av.Len; i++ { - v[i] = av.Array[i].(float64) + v[i] = anyx.Number[float64](av.Array[i]) } return v } @@ -833,28 +861,19 @@ func ValuesNumeric[T constraints.Integer | constraints.Float](av GGUFMetadataKVA for i := uint64(0); i < av.Len; i++ { switch av.Type { case GGUFMetadataValueTypeUint8: - v[i] = T(av.Array[i].(uint8)) case GGUFMetadataValueTypeInt8: - v[i] = T(av.Array[i].(int8)) case GGUFMetadataValueTypeUint16: - v[i] = T(av.Array[i].(uint16)) case GGUFMetadataValueTypeInt16: - v[i] = T(av.Array[i].(int16)) case GGUFMetadataValueTypeUint32: - v[i] = T(av.Array[i].(uint32)) case GGUFMetadataValueTypeInt32: - v[i] = T(av.Array[i].(int32)) case GGUFMetadataValueTypeFloat32: - v[i] = T(av.Array[i].(float32)) case GGUFMetadataValueTypeUint64: - v[i] = T(av.Array[i].(uint64)) case GGUFMetadataValueTypeInt64: - v[i] = T(av.Array[i].(int64)) case GGUFMetadataValueTypeFloat64: - v[i] = T(av.Array[i].(float64)) default: panic(fmt.Errorf("invalid type: %v", av.Type)) } + v[i] = anyx.Number[T](av.Array[i]) } return v } diff --git a/file_from_distro.go b/file_from_distro.go index c15f907..be47b83 100644 --- a/file_from_distro.go +++ b/file_from_distro.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "regexp" "sort" "strconv" @@ -41,7 +42,7 @@ func ParseGGUFFileFromOllama(ctx context.Context, model string, crawl bool, opts // If the crawl is true, it will try to crawl the metadata from Ollama website instead of blobs fetching, // which will be more efficient and faster, but lossy. // If the crawling fails, it will fall back to the default behavior. -func ParseGGUFFileFromOllamaModel(ctx context.Context, model *OllamaModel, crawl bool, opts ...GGUFReadOption) (*GGUFFile, error) { +func ParseGGUFFileFromOllamaModel(ctx context.Context, model *OllamaModel, crawl bool, opts ...GGUFReadOption) (gf *GGUFFile, err error) { if model == nil { return nil, ErrOllamaInvalidModel } @@ -51,6 +52,29 @@ func ParseGGUFFileFromOllamaModel(ctx context.Context, model *OllamaModel, crawl opt(&o) } + // Cache. + { + if o.CachePath != "" { + o.CachePath = filepath.Join(o.CachePath, "distro", "ollama") + if crawl { + o.CachePath = filepath.Join(o.CachePath, "brief") + } + } + c := GGUFFileCache(o.CachePath) + + // Get from cache. + if gf, err = c.Get(model.String(), o.CacheExpiration); err == nil { + return gf, nil + } + + // Put to cache. + defer func() { + if err == nil { + _ = c.Put(model.String(), gf) + } + }() + } + cli := httpx.Client( httpx.ClientOptions(). WithUserAgent("gguf-parser-go"). @@ -94,7 +118,7 @@ func ParseGGUFFileFromOllamaModel(ctx context.Context, model *OllamaModel, crawl if crawl { r, err := ml.FetchWebPage(ctx, cli) if err == nil { - gf, err := parseGGUFFileFromDistroMetadata("ollama", r, ml.Size) + gf, err = parseGGUFFileFromDistroMetadata("ollama", r, ml.Size) if err == nil { return gf, nil } diff --git a/file_from_remote.go b/file_from_remote.go index 8cdea00..b83c8f7 100644 --- a/file_from_remote.go +++ b/file_from_remote.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "path/filepath" "time" "github.com/thxcode/gguf-parser-go/util/httpx" @@ -26,12 +27,35 @@ func ParseGGUFFileFromModelScope(ctx context.Context, repo, file string, opts .. // ParseGGUFFileRemote parses a GGUF file from a remote BlobURL, // and returns a GGUFFile, or an error if any. -func ParseGGUFFileRemote(ctx context.Context, url string, opts ...GGUFReadOption) (*GGUFFile, error) { +func ParseGGUFFileRemote(ctx context.Context, url string, opts ...GGUFReadOption) (gf *GGUFFile, err error) { var o _GGUFReadOptions for _, opt := range opts { opt(&o) } + // Cache. + { + if o.CachePath != "" { + o.CachePath = filepath.Join(o.CachePath, "remote") + if o.SkipLargeMetadata { + o.CachePath = filepath.Join(o.CachePath, "brief") + } + } + c := GGUFFileCache(o.CachePath) + + // Get from cache. + if gf, err = c.Get(url, o.CacheExpiration); err == nil { + return gf, nil + } + + // Put to cache. + defer func() { + if err == nil { + _ = c.Put(url, gf) + } + }() + } + cli := httpx.Client( httpx.ClientOptions(). WithUserAgent("gguf-parser-go"). diff --git a/file_option.go b/file_option.go index f201569..5a4d17c 100644 --- a/file_option.go +++ b/file_option.go @@ -1,6 +1,15 @@ package gguf_parser -import "net/url" +import ( + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/thxcode/gguf-parser-go/util/osx" +) type ( _GGUFReadOptions struct { @@ -17,6 +26,8 @@ type ( SkipDNSCache bool BufferSize int SkipRangeDownloadDetection bool + CachePath string + CacheExpiration time.Duration } GGUFReadOption func(o *_GGUFReadOptions) ) @@ -88,3 +99,53 @@ func SkipRangeDownloadDetection() GGUFReadOption { o.SkipRangeDownloadDetection = true } } + +// UseCache caches the remote reading result. +func UseCache() GGUFReadOption { + var cd string + { + hd, err := os.UserHomeDir() + if err != nil { + hd = filepath.Join(os.TempDir(), time.Now().Format(time.DateOnly)) + } + cd = filepath.Join(hd, ".cache") + if runtime.GOOS == "windows" { + cd = osx.Getenv("APPDATA", cd) + } + } + return func(o *_GGUFReadOptions) { + o.CachePath = filepath.Join(cd, "gguf-parser") + o.CacheExpiration = 24 * time.Hour + } +} + +// SkipCache skips the cache when reading from remote. +func SkipCache() GGUFReadOption { + return func(o *_GGUFReadOptions) { + o.CachePath = "" + o.CacheExpiration = 0 + } +} + +// UseCachePath uses the given path to cache the remote reading result. +func UseCachePath(path string) GGUFReadOption { + path = strings.TrimSpace(filepath.Clean(path)) + return func(o *_GGUFReadOptions) { + if path == "" { + return + } + o.CachePath = path + } +} + +// UseCacheExpiration uses the given expiration to cache the remote reading result. +// +// Disable cache expiration by setting it to 0. +func UseCacheExpiration(expiration time.Duration) GGUFReadOption { + if expiration < 0 { + expiration = 0 + } + return func(o *_GGUFReadOptions) { + o.CacheExpiration = expiration + } +} diff --git a/util/osx/file.go b/util/osx/file.go index e226076..a30add9 100644 --- a/util/osx/file.go +++ b/util/osx/file.go @@ -82,3 +82,22 @@ func Close(c io.Closer) { } _ = c.Close() } + +// WriteFile is similar to os.WriteFile but supports ~ as the home directory, +// and also supports the parent directory creation. +func WriteFile(name string, data []byte, perm os.FileMode) error { + p := filepath.Clean(name) + if strings.HasPrefix(p, "~"+string(filepath.Separator)) { + hd, err := os.UserHomeDir() + if err != nil { + return err + } + p = filepath.Join(hd, p[2:]) + } + + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + return err + } + + return os.WriteFile(p, data, perm) +} diff --git a/util/stringx/bytes.go b/util/stringx/bytes.go index 7433a2a..55f0477 100644 --- a/util/stringx/bytes.go +++ b/util/stringx/bytes.go @@ -9,6 +9,6 @@ func FromBytes(b *[]byte) string { // ToBytes converts a string to a byte slice, // which is impossible to modify the item of slice. -func ToBytes(s string) (bs []byte) { - return unsafe.Slice(unsafe.StringData(s), len(s)) +func ToBytes(s *string) (bs []byte) { + return unsafe.Slice(unsafe.StringData(*s), len(*s)) } diff --git a/util/stringx/sum.go b/util/stringx/sum.go new file mode 100644 index 0000000..8f60031 --- /dev/null +++ b/util/stringx/sum.go @@ -0,0 +1,85 @@ +package stringx + +import ( + "crypto/sha256" + "encoding/hex" + "hash/fnv" +) + +// SumByFNV64a sums up the string(s) by FNV-64a hash algorithm. +func SumByFNV64a(s string, ss ...string) string { + h := fnv.New64a() + + _, _ = h.Write(ToBytes(&s)) + for i := range ss { + _, _ = h.Write(ToBytes(&ss[i])) + } + + sum := h.Sum(nil) + return hex.EncodeToString(sum) +} + +// SumBytesByFNV64a sums up the byte slice(s) by FNV-64a hash algorithm. +func SumBytesByFNV64a(bs []byte, bss ...[]byte) string { + h := fnv.New64a() + + _, _ = h.Write(bs) + for i := range bss { + _, _ = h.Write(bss[i]) + } + + sum := h.Sum(nil) + return hex.EncodeToString(sum) +} + +// SumBySHA256 sums up the string(s) by SHA256 hash algorithm. +func SumBySHA256(s string, ss ...string) string { + h := sha256.New() + + _, _ = h.Write(ToBytes(&s)) + for i := range ss { + _, _ = h.Write(ToBytes(&ss[i])) + } + + sum := h.Sum(nil) + return hex.EncodeToString(sum) +} + +// SumBytesBySHA256 sums up the byte slice(s) by SHA256 hash algorithm. +func SumBytesBySHA256(bs []byte, bss ...[]byte) string { + h := sha256.New() + + _, _ = h.Write(bs) + for i := range bss { + _, _ = h.Write(bss[i]) + } + + sum := h.Sum(nil) + return hex.EncodeToString(sum) +} + +// SumBySHA224 sums up the string(s) by SHA224 hash algorithm. +func SumBySHA224(s string, ss ...string) string { + h := sha256.New224() + + _, _ = h.Write(ToBytes(&s)) + for i := range ss { + _, _ = h.Write(ToBytes(&ss[i])) + } + + sum := h.Sum(nil) + return hex.EncodeToString(sum) +} + +// SumBytesBySHA224 sums up the byte slice(s) by SHA224 hash algorithm. +func SumBytesBySHA224(bs []byte, bss ...[]byte) string { + h := sha256.New224() + + _, _ = h.Write(bs) + for i := range bss { + _, _ = h.Write(bss[i]) + } + + sum := h.Sum(nil) + return hex.EncodeToString(sum) +}