diff --git a/src/cache/badger/badger.go b/src/cache/badger/badger.go index 95d0af60..a84081a1 100644 --- a/src/cache/badger/badger.go +++ b/src/cache/badger/badger.go @@ -67,7 +67,7 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { setTtl = ttl[0] } - key := fmt.Sprintf("%v%v", drv.keyPrefix, k) + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) if val, err := json.Marshal(v); err != nil { return fmt.Errorf("badger.Set(): error marshaling value: %w", err) } else if err := drv.client.Update(func(txn *badger.Txn) error { diff --git a/src/cache/badger/badger_test.go b/src/cache/badger/badger_test.go index 29a7da22..1536a7f9 100644 --- a/src/cache/badger/badger_test.go +++ b/src/cache/badger/badger_test.go @@ -10,7 +10,7 @@ import ( ) func TestNewInMemory(t *testing.T) { - _, err := badger.New("", "", config.Badger{Persist: false}) + _, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -18,14 +18,14 @@ func TestNewInMemory(t *testing.T) { func TestNewPersistence(t *testing.T) { path := "./testdump/new" - _, err := badger.New(path, "", config.Badger{Persist: true}) + _, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } } func TestCloseInMemory(t *testing.T) { - db, err := badger.New("", "", config.Badger{Persist: false}) + db, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -35,7 +35,7 @@ func TestCloseInMemory(t *testing.T) { func TestClosePersistence(t *testing.T) { path := "./testdump/close" - db, err := badger.New(path, "", config.Badger{Persist: true}) + db, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -44,7 +44,7 @@ func TestClosePersistence(t *testing.T) { } func TestSetInMemory(t *testing.T) { - db, err := badger.New("", "", config.Badger{Persist: false}) + db, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -59,7 +59,7 @@ func TestSetInMemory(t *testing.T) { func TestSetPersistence(t *testing.T) { path := "./testdump/set" - db, err := badger.New(path, "", config.Badger{Persist: true}) + db, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -73,7 +73,7 @@ func TestSetPersistence(t *testing.T) { } func TestSetTTLInMemory(t *testing.T) { - db, err := badger.New("", "", config.Badger{Persist: false}) + db, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -88,7 +88,7 @@ func TestSetTTLInMemory(t *testing.T) { func TestSetTTLPersistence(t *testing.T) { path := "./testdump/setttl" - db, err := badger.New(path, "", config.Badger{Persist: true}) + db, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -102,7 +102,7 @@ func TestSetTTLPersistence(t *testing.T) { } func TestGetInMemory(t *testing.T) { - db, err := badger.New("", "", config.Badger{Persist: false}) + db, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -127,7 +127,7 @@ func TestGetInMemory(t *testing.T) { func TestGetPersistence(t *testing.T) { path := "./testdump/get" - db, err := badger.New(path, "", config.Badger{Persist: true}) + db, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -151,7 +151,7 @@ func TestGetPersistence(t *testing.T) { } func TestGetTTLInMemory(t *testing.T) { - db, err := badger.New("", "", config.Badger{Persist: false}) + db, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -176,7 +176,7 @@ func TestGetTTLInMemory(t *testing.T) { func TestGetTTLPersistence(t *testing.T) { path := "./testdump/getttl" - db, err := badger.New(path, "", config.Badger{Persist: true}) + db, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -200,7 +200,7 @@ func TestGetTTLPersistence(t *testing.T) { } func TestGetInMemoryExpired(t *testing.T) { - db, err := badger.New("", "", config.Badger{Persist: false}) + db, err := badger.New("", "TEST_", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -223,7 +223,7 @@ func TestGetInMemoryExpired(t *testing.T) { func TestGetPersistenceExpired(t *testing.T) { path := "./testdump/getexpired" - db, err := badger.New(path, "", config.Badger{Persist: true}) + db, err := badger.New(path, "TEST_", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } diff --git a/src/cache/redis/redis.go b/src/cache/redis/redis.go index f462c961..4f84a119 100644 --- a/src/cache/redis/redis.go +++ b/src/cache/redis/redis.go @@ -59,9 +59,10 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { setTtl = ttl[0] } + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) if val, err := json.Marshal(v); err != nil { return fmt.Errorf("redis.Set(): error marshaling value: %w", err) - } else if err := drv.client.Set(drv.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil { + } else if err := drv.client.Set(drv.ctx, key, val, setTtl).Err(); err != nil { return fmt.Errorf("redis.Set(): error setting KV to redis: %w", err) } else { log.Trace(). @@ -73,7 +74,7 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { } func (drv DRV) Get(k string, o any) error { - key := anonymize.HashToSHA256B64(k) + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) val, err := drv.client.Get(drv.ctx, key).Result() if err == redis.Nil { @@ -91,7 +92,7 @@ func (drv DRV) Get(k string, o any) error { // returns time until the key expires, not the time it will be considered expired func (drv DRV) GetTTL(k string) (time.Duration, error) { - key := anonymize.HashToSHA256B64(k) + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) // returns time with time.Second precision expiresIn, err := drv.client.TTL(drv.ctx, key).Result() diff --git a/src/cache/redis/redis_test.go b/src/cache/redis/redis_test.go index 44046f47..8ff4962e 100644 --- a/src/cache/redis/redis_test.go +++ b/src/cache/redis/redis_test.go @@ -57,7 +57,7 @@ var redisConf = newRedisConf() func TestNew(t *testing.T) { ctx := context.Background() - _, err := redis.New(ctx, "", redisConf) + _, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -65,7 +65,7 @@ func TestNew(t *testing.T) { func TestClose(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, "", redisConf) + db, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -75,7 +75,7 @@ func TestClose(t *testing.T) { func TestSet(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, "", redisConf) + db, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -90,7 +90,7 @@ func TestSet(t *testing.T) { func TestSetTTL(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, "", redisConf) + db, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -105,7 +105,7 @@ func TestSetTTL(t *testing.T) { func TestGet(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, "", redisConf) + db, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -130,7 +130,7 @@ func TestGet(t *testing.T) { func TestGetTTL(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, "", redisConf) + db, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -155,7 +155,7 @@ func TestGetTTL(t *testing.T) { func TestGetExpired(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, "", redisConf) + db, err := redis.New(ctx, "TEST_", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go index cb6bb0d4..c5e67636 100644 --- a/src/search/engines/_sedefaults/init.go +++ b/src/search/engines/_sedefaults/init.go @@ -24,7 +24,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options colly.UserAgent(userAgent), colly.IgnoreRobotsTxt(), colly.Headers(map[string]string{ - "Sec-Ch-Ua": secChUa, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + // "Accept-Encoding": "gzip, deflate, br", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari + "Accept-Language": "en-US,en;q=0.9", + "Sec-Ch-Ua": secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24" + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": "\"Windows\"", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", }), ) pagesCol := colly.NewCollector( @@ -33,7 +41,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options colly.UserAgent(userAgent), colly.IgnoreRobotsTxt(), colly.Headers(map[string]string{ - "Sec-Ch-Ua": secChUa, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + // "Accept-Encoding": "gzip, deflate, br", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari + "Accept-Language": "en-US,en;q=0.9", + "Sec-Ch-Ua": secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24" + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": "\"Windows\"", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", }), )