Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: buggy key in cache drivers and colly headers #320

Merged
merged 1 commit into from
May 28, 2024
Merged

fix: buggy key in cache drivers and colly headers #320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cache/badger/badger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error {
setTtl = ttl[0]
}

key := fmt.Sprintf("%v%v", drv.keyPrefix, k)
key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))
if val, err := json.Marshal(v); err != nil {
return fmt.Errorf("badger.Set(): error marshaling value: %w", err)
} else if err := drv.client.Update(func(txn *badger.Txn) error {
Expand Down
28 changes: 14 additions & 14 deletions src/cache/badger/badger_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,22 @@ import (
)

func TestNewInMemory(t *testing.T) {
_, err := badger.New("", "", config.Badger{Persist: false})
_, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
}

func TestNewPersistence(t *testing.T) {
path := "./testdump/new"
_, err := badger.New(path, "", config.Badger{Persist: true})
_, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
}

func TestCloseInMemory(t *testing.T) {
db, err := badger.New("", "", config.Badger{Persist: false})
db, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
Expand All @@ -35,7 +35,7 @@ func TestCloseInMemory(t *testing.T) {

func TestClosePersistence(t *testing.T) {
path := "./testdump/close"
db, err := badger.New(path, "", config.Badger{Persist: true})
db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
Expand All @@ -44,7 +44,7 @@ func TestClosePersistence(t *testing.T) {
}

func TestSetInMemory(t *testing.T) {
db, err := badger.New("", "", config.Badger{Persist: false})
db, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
Expand All @@ -59,7 +59,7 @@ func TestSetInMemory(t *testing.T) {

func TestSetPersistence(t *testing.T) {
path := "./testdump/set"
db, err := badger.New(path, "", config.Badger{Persist: true})
db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
Expand All @@ -73,7 +73,7 @@ func TestSetPersistence(t *testing.T) {
}

func TestSetTTLInMemory(t *testing.T) {
db, err := badger.New("", "", config.Badger{Persist: false})
db, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
Expand All @@ -88,7 +88,7 @@ func TestSetTTLInMemory(t *testing.T) {

func TestSetTTLPersistence(t *testing.T) {
path := "./testdump/setttl"
db, err := badger.New(path, "", config.Badger{Persist: true})
db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
Expand All @@ -102,7 +102,7 @@ func TestSetTTLPersistence(t *testing.T) {
}

func TestGetInMemory(t *testing.T) {
db, err := badger.New("", "", config.Badger{Persist: false})
db, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
Expand All @@ -127,7 +127,7 @@ func TestGetInMemory(t *testing.T) {

func TestGetPersistence(t *testing.T) {
path := "./testdump/get"
db, err := badger.New(path, "", config.Badger{Persist: true})
db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
Expand All @@ -151,7 +151,7 @@ func TestGetPersistence(t *testing.T) {
}

func TestGetTTLInMemory(t *testing.T) {
db, err := badger.New("", "", config.Badger{Persist: false})
db, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
Expand All @@ -176,7 +176,7 @@ func TestGetTTLInMemory(t *testing.T) {

func TestGetTTLPersistence(t *testing.T) {
path := "./testdump/getttl"
db, err := badger.New(path, "", config.Badger{Persist: true})
db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
Expand All @@ -200,7 +200,7 @@ func TestGetTTLPersistence(t *testing.T) {
}

func TestGetInMemoryExpired(t *testing.T) {
db, err := badger.New("", "", config.Badger{Persist: false})
db, err := badger.New("", "TEST_", config.Badger{Persist: false})
if err != nil {
t.Errorf("error opening in-memory badger: %v", err)
}
Expand All @@ -223,7 +223,7 @@ func TestGetInMemoryExpired(t *testing.T) {

func TestGetPersistenceExpired(t *testing.T) {
path := "./testdump/getexpired"
db, err := badger.New(path, "", config.Badger{Persist: true})
db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
if err != nil {
t.Errorf("error opening badger at %v: %v", path, err)
}
Expand Down
7 changes: 4 additions & 3 deletions src/cache/redis/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,10 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error {
setTtl = ttl[0]
}

key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))
if val, err := json.Marshal(v); err != nil {
return fmt.Errorf("redis.Set(): error marshaling value: %w", err)
} else if err := drv.client.Set(drv.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil {
} else if err := drv.client.Set(drv.ctx, key, val, setTtl).Err(); err != nil {
return fmt.Errorf("redis.Set(): error setting KV to redis: %w", err)
} else {
log.Trace().
Expand All @@ -73,7 +74,7 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error {
}

func (drv DRV) Get(k string, o any) error {
key := anonymize.HashToSHA256B64(k)
key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))

val, err := drv.client.Get(drv.ctx, key).Result()
if err == redis.Nil {
Expand All @@ -91,7 +92,7 @@ func (drv DRV) Get(k string, o any) error {

// returns time until the key expires, not the time it will be considered expired
func (drv DRV) GetTTL(k string) (time.Duration, error) {
key := anonymize.HashToSHA256B64(k)
key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))

// returns time with time.Second precision
expiresIn, err := drv.client.TTL(drv.ctx, key).Result()
Expand Down
14 changes: 7 additions & 7 deletions src/cache/redis/redis_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,15 @@ var redisConf = newRedisConf()

func TestNew(t *testing.T) {
ctx := context.Background()
_, err := redis.New(ctx, "", redisConf)
_, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
}

func TestClose(t *testing.T) {
ctx := context.Background()
db, err := redis.New(ctx, "", redisConf)
db, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
Expand All @@ -75,7 +75,7 @@ func TestClose(t *testing.T) {

func TestSet(t *testing.T) {
ctx := context.Background()
db, err := redis.New(ctx, "", redisConf)
db, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
Expand All @@ -90,7 +90,7 @@ func TestSet(t *testing.T) {

func TestSetTTL(t *testing.T) {
ctx := context.Background()
db, err := redis.New(ctx, "", redisConf)
db, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
Expand All @@ -105,7 +105,7 @@ func TestSetTTL(t *testing.T) {

func TestGet(t *testing.T) {
ctx := context.Background()
db, err := redis.New(ctx, "", redisConf)
db, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
Expand All @@ -130,7 +130,7 @@ func TestGet(t *testing.T) {

func TestGetTTL(t *testing.T) {
ctx := context.Background()
db, err := redis.New(ctx, "", redisConf)
db, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
Expand All @@ -155,7 +155,7 @@ func TestGetTTL(t *testing.T) {

func TestGetExpired(t *testing.T) {
ctx := context.Background()
db, err := redis.New(ctx, "", redisConf)
db, err := redis.New(ctx, "TEST_", redisConf)
if err != nil {
t.Errorf("error creating redis: %v", err)
}
Expand Down
20 changes: 18 additions & 2 deletions src/search/engines/_sedefaults/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
colly.UserAgent(userAgent),
colly.IgnoreRobotsTxt(),
colly.Headers(map[string]string{
"Sec-Ch-Ua": secChUa,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
// "Accept-Encoding": "gzip, deflate, br", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
"Accept-Language": "en-US,en;q=0.9",
"Sec-Ch-Ua": secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": "\"Windows\"",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
}),
)
pagesCol := colly.NewCollector(
Expand All @@ -33,7 +41,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
colly.UserAgent(userAgent),
colly.IgnoreRobotsTxt(),
colly.Headers(map[string]string{
"Sec-Ch-Ua": secChUa,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
// "Accept-Encoding": "gzip, deflate, br", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
"Accept-Language": "en-US,en;q=0.9",
"Sec-Ch-Ua": secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": "\"Windows\"",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
}),
)

Expand Down
Loading