Merge pull request #550 from fairDataSociety/saveFeedsInLRU.1

cacheSize and cacheTTL
fairDataSociety · Oct 13, 2023 · ff3a4ec · ff3a4ec
2 parents 06a68d4 + fb198ed
commit ff3a4ec
Show file tree

Hide file tree

Showing 62 changed files with 236 additions and 209 deletions.
diff --git a/cmd/dfs/cmd/server_test.go b/cmd/dfs/cmd/server_test.go
@@ -60,7 +60,7 @@ func TestApis(t *testing.T) {
 	mockClient := bee.NewBeeClient(beeUrl, mock.BatchOkStr, true, logger)
 	ens := mock2.NewMockNamespaceManager()
 
-	users := user.NewUsers(mockClient, ens, logger)
+	users := user.NewUsers(mockClient, ens, 500, 0, logger)
 	dfsApi := dfs.NewMockDfsAPI(mockClient, users, logger)
 	handler = api.NewMockHandler(dfsApi, logger, []string{"http://localhost:3000"})
 	defer handler.Close()
@@ -77,7 +77,7 @@ func TestApis(t *testing.T) {
 		}
 	}()
 
-	// wait 10 seconds for the server to start
+	// wait for the server to start
 	<-time.After(time.Second * 3)
 	t.Run("login-fail-test", func(t *testing.T) {
 		c := http.Client{Timeout: time.Duration(1) * time.Minute}

diff --git a/pkg/api/dir_ls.go b/pkg/api/dir_ls.go
@@ -20,13 +20,11 @@ import (
 	"net/http"
 
 	"github.com/fairdatasociety/fairOS-dfs/pkg/auth"
-
-	"resenje.org/jsonhttp"
-
 	"github.com/fairdatasociety/fairOS-dfs/pkg/dfs"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/dir"
 	"github.com/fairdatasociety/fairOS-dfs/pkg/file"
 	p "github.com/fairdatasociety/fairOS-dfs/pkg/pod"
+	"resenje.org/jsonhttp"
 )
 
 // ListFileResponse is used to list directories and files

diff --git a/pkg/api/handler.go b/pkg/api/handler.go
@@ -24,6 +24,11 @@ import (
 	"github.com/fairdatasociety/fairOS-dfs/pkg/logging"
 )
 
+const (
+	feedCacheSize = 100
+	feedCacheTTL  = 0
+)
+
 // Handler is the api handler
 type Handler struct {
 	ctx                context.Context
@@ -54,6 +59,8 @@ func New(ctx context.Context, opts *Options) (*Handler, error) {
 		SubscriptionConfig: opts.SubscriptionConfig,
 		Logger:             opts.Logger,
 		FeedTracker:        opts.FeedTracker,
+		FeedCacheSize:      feedCacheSize,
+		FeedCacheTTL:       feedCacheTTL,
 	}
 	api, err := dfs.NewDfsAPI(ctx, dfsOpts)
 	if err != nil {

diff --git a/pkg/collection/batch_test.go b/pkg/collection/batch_test.go
@@ -52,7 +52,7 @@ func TestBatchIndex(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger)
+	fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, 0, logger)
 	user := acc.GetAddress(account.UserAccountIndex)
 	podPassword, _ := utils.GetRandString(pod.PasswordLength)
 	t.Run("batch-add-docs", func(t *testing.T) {

diff --git a/pkg/collection/document_test.go b/pkg/collection/document_test.go
@@ -68,7 +68,7 @@ func TestDocumentStore(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger)
+	fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, 0, logger)
 	user := acc.GetAddress(account.UserAccountIndex)
 	tm := taskmanager.New(1, 10, time.Second*15, logger)
 	defer func() {
@@ -79,7 +79,7 @@ func TestDocumentStore(t *testing.T) {
 	docStore := collection.NewDocumentStore("pod1", fd, ai, user, file, tm, mockClient, logger)
 	podPassword, _ := utils.GetRandString(pod.PasswordLength)
 	t.Run("create_document_db_errors", func(t *testing.T) {
-		nilFd := feed.New(&account.Info{}, mockClient, logger)
+		nilFd := feed.New(&account.Info{}, mockClient, 500, 0, logger)
 		nilDocStore := collection.NewDocumentStore("pod1", nilFd, ai, user, file, tm, mockClient, logger)
 		err := nilDocStore.CreateDocumentDB("docdb_err", podPassword, nil, true)
 		if !errors.Is(err, collection.ErrReadOnlyIndex) {

diff --git a/pkg/collection/index_api_test.go b/pkg/collection/index_api_test.go
@@ -54,7 +54,7 @@ func TestIndexAPI(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger)
+	fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, 0, logger)
 	user := acc.GetAddress(account.UserAccountIndex)
 	podPassword, _ := utils.GetRandString(pod.PasswordLength)
 	t.Run("get-doc", func(t *testing.T) {

diff --git a/pkg/collection/index_test.go b/pkg/collection/index_test.go
@@ -56,7 +56,7 @@ func TestIndex(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger)
+	fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, 0, logger)
 	user := acc.GetAddress(account.UserAccountIndex)
 	podPassword, _ := utils.GetRandString(pod.PasswordLength)
 	t.Run("create_index", func(t *testing.T) {

diff --git a/pkg/collection/iterator_test.go b/pkg/collection/iterator_test.go
@@ -57,7 +57,7 @@ func TestIndexIterator(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fd := feed.New(acc.GetUserAccountInfo(), mockClient, logger)
+	fd := feed.New(acc.GetUserAccountInfo(), mockClient, 500, 0, logger)
 	user := acc.GetAddress(account.UserAccountIndex)
 	podPassword, _ := utils.GetRandString(pod.PasswordLength)
 	t.Run("iterate_all_string_keys", func(t *testing.T) {

diff --git a/pkg/collection/kv.go b/pkg/collection/kv.go
@@ -141,6 +141,10 @@ func (kv *KeyValue) DeleteKVTable(name, encryptionPassword string) error {
 	return kv.storeKVTables(kvtables, encryptionPassword)
 }
 
+func (kv *KeyValue) Commit() {
+	kv.fd.CommitFeeds()
+}
+
 // DeleteAllKVTables deletes all key value tables with all their index and data entries.
 func (kv *KeyValue) DeleteAllKVTables(encryptionPassword string) error {
 	if kv.fd.IsReadOnlyFeed() { // skipcq: TCV-001