From 7a0c838d935cbacbcacfde1b427f4a96d61d44de Mon Sep 17 00:00:00 2001 From: jarrel Date: Fri, 19 Apr 2024 12:24:27 -0700 Subject: [PATCH] Delete indexer code (#1488) --- Makefile | 38 - README.md | 24 - cmd/contract_owners/main.go | 209 ---- cmd/contract_owners/orchestrator/main.go | 111 -- cmd/indexer/main.go | 13 - cmd/reprocess/main.go | 241 ---- cmd/reprocess/orchestrator/main.go | 111 -- cmd/token_normalization_migrate/main.go | 618 ---------- db/gen/indexerdb/db_gen.go | 32 - db/gen/indexerdb/models_gen.go | 76 -- db/gen/indexerdb/query.sql.go | 224 ---- .../000001_create_core_tables.down.sql | 2 - .../indexer/000001_create_core_tables.up.sql | 44 - .../indexer/000002_contract_owner.up.sql | 1 - db/migrations/indexer/000003_stats.up.sql | 19 - .../000004_backfill_contract_owner.up.sql | 7 - db/queries/indexer/query.sql | 39 - docker-compose.yml | 43 - docker/cloud_sql_proxy/docker-compose.yml | 2 - docker/docker.go | 22 - docker/indexer/Dockerfile | 27 - docker/indexer_api/Dockerfile | 27 - docker/postgres_indexer/01_init.sql | 5 - docker/postgres_indexer/02_init.sql | 52 - docker/postgres_indexer/DOCKERFILE | 14 - indexer/cmd/root.go | 106 -- indexer/contracts.go | 176 --- indexer/core.go | 202 ---- indexer/handler.go | 26 - indexer/hook.go | 90 -- indexer/indexer.go | 1061 ----------------- indexer/plugin.go | 257 ---- indexer/server.go | 34 - indexer/t__integration_test.go | 46 - indexer/t__utils_test.go | 279 ----- secrets/dev/app-dev-indexer.yaml | 40 - secrets/dev/backend-env.yaml | 5 +- secrets/dev/backend-sandbox-env.yaml | 5 +- secrets/dev/indexer-server-env.yaml | 30 - secrets/dev/local/app-dev-backend.yaml | 6 +- .../dev/local/app-dev-tokenprocessing.yaml | 5 +- secrets/dev/makevars.yaml | 8 +- secrets/dev/tokenprocessing-env.yaml | 5 +- secrets/local/local/app-local-backend.yaml | 6 +- .../local/local/app-local-indexer-server.yaml | 20 - secrets/local/local/app-local-indexer.yaml | 18 - .../local/app-local-tokenprocessing.yaml | 5 +- secrets/prod/backend-env.yaml | 5 +- secrets/prod/indexer-env.yaml | 38 - secrets/prod/indexer-server-env.yaml | 40 - .../prod/local/app-prod-indexer-server.yaml | 26 - secrets/prod/local/app-prod-indexer.yaml | 32 - .../prod/local/app-prod-tokenprocessing.yaml | 5 +- secrets/prod/makevars.yaml | 15 +- secrets/prod/tokenprocessing-env.yaml | 5 +- server/inject.go | 15 +- server/server.go | 2 - server/wire_gen.go | 18 +- service/auth/auth.go | 3 - service/eth/eth.go | 110 ++ service/multichain/indexer/indexer.go | 237 ---- service/multichain/simplehash/simplehash.go | 22 - service/rpc/rpc.go | 239 +--- sqlc.yaml | 32 - tokenprocessing/process.go | 4 - 65 files changed, 158 insertions(+), 5121 deletions(-) delete mode 100644 cmd/contract_owners/main.go delete mode 100644 cmd/contract_owners/orchestrator/main.go delete mode 100644 cmd/indexer/main.go delete mode 100644 cmd/reprocess/main.go delete mode 100644 cmd/reprocess/orchestrator/main.go delete mode 100644 cmd/token_normalization_migrate/main.go delete mode 100644 db/gen/indexerdb/db_gen.go delete mode 100644 db/gen/indexerdb/models_gen.go delete mode 100644 db/gen/indexerdb/query.sql.go delete mode 100644 db/migrations/indexer/000001_create_core_tables.down.sql delete mode 100644 db/migrations/indexer/000001_create_core_tables.up.sql delete mode 100644 db/migrations/indexer/000002_contract_owner.up.sql delete mode 100644 db/migrations/indexer/000003_stats.up.sql delete mode 100644 db/migrations/indexer/000004_backfill_contract_owner.up.sql delete mode 100644 db/queries/indexer/query.sql delete mode 100644 docker/indexer/Dockerfile delete mode 100644 docker/indexer_api/Dockerfile delete mode 100644 docker/postgres_indexer/01_init.sql delete mode 100644 docker/postgres_indexer/02_init.sql delete mode 100644 docker/postgres_indexer/DOCKERFILE delete mode 100644 indexer/cmd/root.go delete mode 100644 indexer/contracts.go delete mode 100644 indexer/core.go delete mode 100644 indexer/handler.go delete mode 100644 indexer/hook.go delete mode 100644 indexer/indexer.go delete mode 100644 indexer/plugin.go delete mode 100644 indexer/server.go delete mode 100644 indexer/t__integration_test.go delete mode 100644 indexer/t__utils_test.go delete mode 100644 secrets/dev/app-dev-indexer.yaml delete mode 100644 secrets/dev/indexer-server-env.yaml delete mode 100644 secrets/local/local/app-local-indexer-server.yaml delete mode 100644 secrets/local/local/app-local-indexer.yaml delete mode 100644 secrets/prod/indexer-env.yaml delete mode 100644 secrets/prod/indexer-server-env.yaml delete mode 100644 secrets/prod/local/app-prod-indexer-server.yaml delete mode 100644 secrets/prod/local/app-prod-indexer.yaml delete mode 100644 service/multichain/indexer/indexer.go diff --git a/Makefile b/Makefile index 010118fac..68ae64dbe 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,6 @@ start-dev-sql-proxy : REQUIRED_SOPS_SECRETS := $(SOPS_DEV_SECRETS) start-prod-sql-proxy : REQUIRED_SOPS_SECRETS := $(SOPS_PROD_SECRETS) migrate-dev-coredb : REQUIRED_SOPS_SECRETS := $(SOPS_DEV_SECRETS) migrate-prod-coredb : REQUIRED_SOPS_SECRETS := $(SOPS_PROD_SECRETS) -migrate-prod-indexerdb : REQUIRED_SOPS_SECRETS := $(SOPS_PROD_SECRETS) # Environment-specific settings $(DEPLOY)-$(DEV)-% : ENV := $(DEV) @@ -71,7 +70,6 @@ $(PROMOTE)-$(PROD)-% : REQUIRED_SOPS_SECRETS := $(SOPS_PROD_SECRET # Service files, add a line for each service and environment you want to deploy. $(DEPLOY)-$(DEV)-backend : SERVICE_FILE := backend-env.yaml -$(DEPLOY)-$(DEV)-indexer-server : SERVICE_FILE := indexer-server-env.yaml $(DEPLOY)-$(DEV)-admin : SERVICE_FILE := app-dev-admin.yaml $(DEPLOY)-$(DEV)-feed : SERVICE_FILE := feed-env.yaml $(DEPLOY)-$(DEV)-tokenprocessing : SERVICE_FILE := tokenprocessing-env.yaml @@ -87,8 +85,6 @@ $(DEPLOY)-$(DEV)-graphql-gateway : SERVICE_FILE := graphql-gateway.yml $(DEPLOY)-$(DEV)-userpref-upload : SERVICE_FILE := userpref-upload.yaml $(DEPLOY)-$(SANDBOX)-backend : SERVICE_FILE := backend-sandbox-env.yaml $(DEPLOY)-$(PROD)-backend : SERVICE_FILE := backend-env.yaml -$(DEPLOY)-$(PROD)-indexer : SERVICE_FILE := indexer-env.yaml -$(DEPLOY)-$(PROD)-indexer-server : SERVICE_FILE := indexer-server-env.yaml $(DEPLOY)-$(PROD)-admin : SERVICE_FILE := app-prod-admin.yaml $(DEPLOY)-$(PROD)-feed : SERVICE_FILE := feed-env.yaml $(DEPLOY)-$(PROD)-feedbot : SERVICE_FILE := feedbot-env.yaml @@ -107,8 +103,6 @@ $(DEPLOY)-$(PROD)-rasterizer : SERVICE_FILE := rasterizer.yaml # Service to Sentry project mapping $(DEPLOY)-%-backend : SENTRY_PROJECT := gallery-backend -$(DEPLOY)-%-indexer : SENTRY_PROJECT := indexer -$(DEPLOY)-%-indexer-server : SENTRY_PROJECT := indexer-api $(DEPLOY)-%-tokenprocessing : SENTRY_PROJECT := tokenprocessing $(DEPLOY)-%-pushnotifications : SENTRY_PROJECT := pushnotifications $(DEPLOY)-%-dummymetadata : SENTRY_PROJECT := dummymetadata @@ -176,24 +170,6 @@ $(DEPLOY)-%-dummymetadata : CPU := $(DUMMYMETADATA_CPU) $(DEPLOY)-%-dummymetadata : MEMORY := $(DUMMYMETADATA_MEMORY) $(DEPLOY)-%-dummymetadata : CONCURRENCY := $(DUMMYMETADATA_CONCURRENCY) $(DEPLOY)-%-dummymetadata : SERVICE := dummymetadata -$(DEPLOY)-%-indexer-server : REPO := indexer-api -$(DEPLOY)-%-indexer-server : DOCKER_FILE := $(DOCKER_DIR)/indexer_api/Dockerfile -$(DEPLOY)-%-indexer-server : PORT := 6000 -$(DEPLOY)-%-indexer-server : TIMEOUT := $(INDEXER_SERVER_TIMEOUT) -$(DEPLOY)-%-indexer-server : CPU := $(INDEXER_SERVER_CPU) -$(DEPLOY)-%-indexer-server : MEMORY := $(INDEXER_SERVER_MEMORY) -$(DEPLOY)-%-indexer-server : CONCURRENCY := $(INDEXER_SERVER_CONCURRENCY) -$(DEPLOY)-$(DEV)-indexer-server : SERVICE := indexer-api-dev -$(DEPLOY)-$(PROD)-indexer-server : SERVICE := indexer-api -$(DEPLOY)-%-indexer : REPO := indexer -$(DEPLOY)-%-indexer : DOCKER_FILE := $(DOCKER_DIR)/indexer/Dockerfile -$(DEPLOY)-%-indexer : PORT := 4000 -$(DEPLOY)-%-indexer : TIMEOUT := $(INDEXER_TIMEOUT) -$(DEPLOY)-%-indexer : CPU := $(INDEXER_CPU) -$(DEPLOY)-%-indexer : MEMORY := $(INDEXER_MEMORY) -$(DEPLOY)-%-indexer : CONCURRENCY := $(INDEXER_CONCURRENCY) -$(DEPLOY)-%-indexer : DEPLOY_FLAGS = $(BASE_DEPLOY_FLAGS) --no-cpu-throttling -$(DEPLOY)-$(PROD)-indexer : SERVICE := indexer $(DEPLOY)-%-emails : REPO := emails $(DEPLOY)-%-emails : DOCKER_FILE := $(DOCKER_DIR)/emails/Dockerfile $(DEPLOY)-%-emails : PORT := 5500 @@ -328,8 +304,6 @@ $(DEPLOY)-%-userpref-upload : DOCKER_FILE := $(DOCKER_DIR)/userpre # Service name mappings $(PROMOTE)-%-backend : SERVICE := default -$(PROMOTE)-%-indexer : SERVICE := indexer -$(PROMOTE)-%-indexer-server : SERVICE := indexer-api $(PROMOTE)-%-emails : SERVICE := emails-v2 $(PROMOTE)-%-tokenprocessing : SERVICE := tokenprocessing $(PROMOTE)-%-tokenprocessing : SERVICE := tokenprocessing-v3 @@ -459,7 +433,6 @@ _$(RELEASE)-%: # DEV deployments $(DEPLOY)-$(DEV)-backend : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-backend _$(RELEASE)-backend -$(DEPLOY)-$(DEV)-indexer-server : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-indexer-server _$(RELEASE)-indexer-server $(DEPLOY)-$(DEV)-tokenprocessing : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-tokenprocessing _$(RELEASE)-tokenprocessing $(DEPLOY)-$(DEV)-autosocial : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-autosocial _$(RELEASE)-autosocial $(DEPLOY)-$(DEV)-autosocial-orch : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-autosocial-orch _$(RELEASE)-autosocial-orc @@ -485,8 +458,6 @@ $(DEPLOY)-$(SANDBOX)-backend : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-bac # PROD deployments $(DEPLOY)-$(PROD)-backend : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-backend _$(RELEASE)-backend -$(DEPLOY)-$(PROD)-indexer : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-indexer _$(RELEASE)-indexer -$(DEPLOY)-$(PROD)-indexer-server : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-indexer-server _$(RELEASE)-indexer-server $(DEPLOY)-$(PROD)-tokenprocessing : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-tokenprocessing _$(RELEASE)-tokenprocessing $(DEPLOY)-$(PROD)-autosocial : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-autosocial _$(RELEASE)-autosocial $(DEPLOY)-$(PROD)-autosocial-orch : _set-project-$(ENV) _$(DOCKER)-$(DEPLOY)-autosocial-orch _$(RELEASE)-autosocial-orch @@ -516,8 +487,6 @@ $(DEPLOY)-$(PROD)-rasterizer : _set-project-$(ENV) _$(DOCKER)-$(DE # $ make promote-prod-backend version=myVersion # $(PROMOTE)-$(PROD)-backend : _set-project-$(ENV) _$(DOCKER)-$(PROMOTE)-backend -$(PROMOTE)-$(PROD)-indexer : _set-project-$(ENV) _$(DOCKER)-$(PROMOTE)-indexer -$(PROMOTE)-$(PROD)-indexer-server : _set-project-$(ENV) _$(DOCKER)-$(PROMOTE)-indexer-server $(PROMOTE)-$(PROD)-tokenprocessing : _set-project-$(ENV) _$(DOCKER)-$(PROMOTE)-tokenprocessing $(PROMOTE)-$(PROD)-autosocial : _set-project-$(ENV) _$(DOCKER)-$(PROMOTE)-autosocial $(PROMOTE)-$(PROD)-autosocial-orch : _set-project-$(ENV) _$(DOCKER)-$(PROMOTE)-autosocial-orch @@ -618,10 +587,6 @@ start-prod-sql-proxy: stop-sql-proxy: docker-compose -f docker/cloud_sql_proxy/docker-compose.yml down -# Migrations -migrate-local-indexerdb: - migrate -path ./db/migrations/indexer -database "postgresql://postgres@localhost:5433/postgres?sslmode=disable" up - migrate-local-coredb: go run cmd/migrate/main.go @@ -645,9 +610,6 @@ migrate-prod-coredb: start-prod-sql-proxy confirm-prod-migrate POSTGRES_PORT=6543 \ go run cmd/migrate/main.go -migrate-prod-indexerdb: start-prod-sql-proxy confirm-prod-migrate - migrate -path ./db/migrations/indexer -database "postgresql://postgres:$(POSTGRES_INDEXER_PASSWORD)@localhost:6545/postgres?sslmode=disable" up - fix-sops-macs: @cd secrets; ../scripts/fix-sops-macs.sh diff --git a/README.md b/README.md index 8e3acaf55..516215318 100644 --- a/README.md +++ b/README.md @@ -76,9 +76,6 @@ Create a new migration: ```bash # New migration for the backend db migrate create -ext sql -dir db/migrations/core -seq - -# New migration for the indexer db -migrate create -ext sql -dir db/migrations/indexer -seq ``` Run a migration locally: @@ -86,9 +83,6 @@ Run a migration locally: ```bash # Run all migrations for the local backend db make migrate-coredb - -# Run all migrations for the local indexer db -make migrate-indexerdb ``` Run a migration on dev backend db: @@ -101,16 +95,6 @@ migrate -path db/migrations/core -database "postgresql://postgres:@34.102.59.201:5432/postgres" down 1 ``` -Run a migration on the indexer db: - -```bash -# Apply an up migration to the indexer db -migrate -path db/migrations/indexer -database "postgresql://postgres:@:5432/postgres" up - -# Undo the last migration to the indexer db -migrate -path db/migrations/indexer -database "postgresql://postgres:@:5432/postgres" down 1 -``` - ### Healthcheck Verify that the server is running: @@ -165,14 +149,6 @@ For example, to run the server locally with live data from the `dev` environment go run cmd/server/main.go dev ``` -To run the indexer server connected to production, run the following command: - -```bash -go run cmd/indexer/server/main.go prod -``` - -When testing the indexer locally, you may want to sync log data from the prod to dev bucket. You can do this by running the sync command below. This command can take a few minutes depending on when the buckets were last synced. - ```bash # Do not switch the order of the buckets! Doing so may overwrite prod data. gsutil -m rsync -r gs://prod-eth-token-logs gs://dev-eth-token-logs diff --git a/cmd/contract_owners/main.go b/cmd/contract_owners/main.go deleted file mode 100644 index 62f4aaf53..000000000 --- a/cmd/contract_owners/main.go +++ /dev/null @@ -1,209 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "os" - "time" - - "net/http" - _ "net/http/pprof" - - "github.com/mikeydub/go-gallery/db/gen/indexerdb" - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/indexer" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/persist/postgres" - "github.com/mikeydub/go-gallery/service/rpc" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" - "github.com/mikeydub/go-gallery/service/tracing" - "github.com/mikeydub/go-gallery/util" - "github.com/sirupsen/logrus" - "github.com/sourcegraph/conc/pool" - "github.com/spf13/viper" -) - -func main() { - - setDefaults() - - start := time.Now() - defer func() { - elapsed := time.Since(start) - fmt.Printf("Took %s", elapsed) - }() - ctx := context.Background() - - pgx := postgres.NewPgxClient() - - queries := indexerdb.New(pgx) - - var c int - pgx.QueryRow(ctx, `select count(*) from contracts;`).Scan(&c) - logrus.Infof("total contracts: %d", c) - - ethClient := rpc.NewEthSocketClient() - httpClient := &http.Client{Timeout: 10 * time.Minute, Transport: tracing.NewTracingTransport(http.DefaultTransport, false)} - - var rows []indexerdb.Contract - var err error - - if env.GetString("CLOUD_RUN_JOB") != "" { - logrus.Infof("running as cloud run job") - - indexer.InitSentry() - logger.InitWithGCPDefaults() - - jobIndex := env.GetInt("CLOUD_RUN_TASK_INDEX") - jobCount := env.GetInt("CLOUD_RUN_TASK_COUNT") - - // given the totalTokenCount, and the jobCount, we can calculate the offset and limit for this job - // we want to evenly distribute the work across the jobs - // so we can calculate the limit by dividing the totalTokenCount by the jobCount - // and the offset by multiplying the jobIndex by the limit - - logrus.Infof("jobIndex: %d, jobCount: %d", jobIndex, jobCount) - - r, err := queries.GetReprocessJobRangeByID(ctx, jobIndex) - if err != nil { - logrus.Errorf("error getting job range: %v", err) - panic(err) - } - - rows, err = queries.GetContractsByIDRange(ctx, indexerdb.GetContractsByIDRangeParams{ - StartID: r.StartID, - EndID: r.EndID, - }) - if err != nil { - panic(err) - } - } else { - - logrus.Infof("running as local job") - logger.SetLoggerOptions(func(logger *logrus.Logger) { - fi, err := os.Create(fmt.Sprintf("reprocess-%s.log", time.Now().Format("2006-01-02T15-04-05"))) - if err != nil { - panic(err) - } - logger.SetOutput(io.MultiWriter(fi, os.Stdout)) - }) - - r, err := queries.GetReprocessJobRangeByID(ctx, 0) - if err != nil { - logrus.Errorf("error getting job range: %v", err) - panic(err) - } - rows, err = queries.GetContractsByIDRange(ctx, indexerdb.GetContractsByIDRangeParams{ - StartID: r.StartID, - EndID: r.EndID, - }) - if err != nil { - panic(err) - } - } - - if err != nil { - logrus.Errorf("error getting contracts: %v", err) - panic(err) - } - - wp := pool.New().WithMaxGoroutines(50).WithContext(ctx) - - logrus.Infof("processing (%d) contracts...", len(rows)) - - asContracts, _ := util.Map(rows, func(r indexerdb.Contract) (persist.Contract, error) { - return persist.Contract{ - ID: r.ID, - Address: r.Address, - OwnerMethod: r.OwnerMethod, - CreationTime: r.CreatedAt, - Deleted: persist.NullBool(r.Deleted), - LastUpdated: r.LastUpdated, - Chain: r.Chain, - Symbol: persist.NullString(r.Symbol.String), - Name: persist.NullString(r.Name.String), - OwnerAddress: r.OwnerAddress, - CreatorAddress: r.CreatorAddress, - LatestBlock: persist.BlockNumber(r.LatestBlock.Int64), - }, nil - }) - - // group contracts into groups of 100 - - groups := util.Chunk(asContracts, 100) - - for i, group := range groups { - i := i - - wp.Go(func(ctx context.Context) error { - ctx = sentryutil.NewSentryHubContext(ctx) - - defer func() { - logger.For(ctx).Infof("finished processing %d", i) - }() - results, err := indexer.GetContractMetadatas(ctx, group, httpClient, ethClient) - if err != nil { - return err - } - - errs := []error{} - for _, result := range results { - logger.For(ctx).Infof("updating contract %s (%s, %s, %d)", result.Contract.Address, result.Contract.OwnerAddress, result.Contract.CreatorAddress, result.OwnerMethod) - err = queries.UpdateContractOwnerByID(ctx, indexerdb.UpdateContractOwnerByIDParams{ - OwnerAddress: result.Contract.OwnerAddress, - CreatorAddress: result.Contract.CreatorAddress, - OwnerMethod: result.OwnerMethod, - ID: result.Contract.ID, - }) - if err != nil { - errs = append(errs, err) - } - } - if len(errs) > 0 { - return util.MultiErr(errs) - } - return nil - }) - } - - go func() { - http.ListenAndServe(":6060", nil) - }() - err = wp.Wait() - logrus.Infof("finished processes with err: %s", err) - -} - -func setDefaults() { - viper.SetDefault("ENV", "local") - viper.SetDefault("POSTGRES_HOST", "0.0.0.0") - viper.SetDefault("POSTGRES_PORT", 5433) - viper.SetDefault("POSTGRES_USER", "postgres") - viper.SetDefault("POSTGRES_PASSWORD", "") - viper.SetDefault("POSTGRES_DB", "postgres") - viper.SetDefault("INDEXER_HOST", "http://localhost:6000") - viper.SetDefault("ALCHEMY_API_URL", "") - viper.SetDefault("RPC_URL", "https://eth-goerli.g.alchemy.com/v2/_2u--i79yarLYdOT4Bgydqa0dBceVRLD") - viper.SetDefault("REDIS_URL", "localhost:6379") - viper.SetDefault("CLOUD_RUN_JOB", "") - viper.SetDefault("CLOUD_RUN_TASK_INDEX", 0) - viper.SetDefault("CLOUD_RUN_TASK_COUNT", 1) - viper.SetDefault("SENTRY_DSN", "") - viper.SetDefault("SENTRY_TRACES_SAMPLE_RATE", 0.2) - viper.SetDefault("VERSION", "0") - - viper.AutomaticEnv() - - if env.GetString("ENV") != "local" { - logrus.Info("running in non-local environment, skipping environment configuration") - } else { - fi := "local" - if len(os.Args) > 1 { - fi = os.Args[1] - } - envFile := util.ResolveEnvFile("indexer", fi) - util.LoadEncryptedEnvFile(envFile) - } -} diff --git a/cmd/contract_owners/orchestrator/main.go b/cmd/contract_owners/orchestrator/main.go deleted file mode 100644 index 462d4c8ab..000000000 --- a/cmd/contract_owners/orchestrator/main.go +++ /dev/null @@ -1,111 +0,0 @@ -package main - -import ( - "context" - "fmt" - "math" - "os" - "time" - - _ "net/http/pprof" - - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/persist/postgres" - "github.com/mikeydub/go-gallery/util" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" -) - -const ( - totalJobs = 1000 -) - -type jobRange struct { - id int - start persist.DBID - end persist.DBID -} - -func main() { - - setDefaults() - - start := time.Now() - defer func() { - elapsed := time.Since(start) - fmt.Printf("Took %s", elapsed) - }() - ctx := context.Background() - pg := postgres.NewPgxClient() - - var totalContractCount int - - err := pg.QueryRow(ctx, `select count(*) from contracts where contracts.deleted = false and (contracts.owner_address is null or contracts.owner_address = '' OR contracts.creator_address IS NULL OR contracts.creator_address = '');`).Scan(&totalContractCount) - if err != nil { - logrus.Errorf("error getting total contract count: %v", err) - panic(err) - } - - limit := int(math.Ceil(float64(totalContractCount) / float64(totalJobs))) - - rows, err := pg.Query(ctx, `select contracts.id from contracts where contracts.deleted = false and (contracts.owner_address is null or contracts.owner_address = '' OR contracts.creator_address IS NULL OR contracts.creator_address = '') order by contracts.id;`) - if err != nil { - logrus.Errorf("error getting contract ids: %v", err) - panic(err) - } - - var curRange jobRange - - for i := 0; rows.Next(); i++ { - var id persist.DBID - err := rows.Scan(&id) - if err != nil { - logrus.Errorf("error scanning contract id: %v", err) - panic(err) - } - - if i%limit == 0 { - fmt.Printf("starting job range %d (start: %s)\n", i/limit, id) - curRange = jobRange{ - id: i / limit, - start: id, - } - } - - if i%limit == limit-1 { - fmt.Printf("ending job range %d (end: %s)\n", curRange.id, id) - curRange.end = id - // repurpose the reprocess table - _, err = pg.Exec(ctx, `INSERT INTO reprocess_jobs (id, start_id, end_id) VALUES ($1, $2, $3);`, curRange.id, curRange.start, curRange.end) - if err != nil { - logrus.Errorf("error inserting job: %v", err) - panic(err) - } - } - } - - fmt.Printf("Inserted %d jobs with a limit of %d and %d total contracts\n", curRange.id, limit, totalContractCount) -} - -func setDefaults() { - viper.SetDefault("ENV", "local") - viper.SetDefault("POSTGRES_HOST", "0.0.0.0") - viper.SetDefault("POSTGRES_PORT", 5433) - viper.SetDefault("POSTGRES_USER", "postgres") - viper.SetDefault("POSTGRES_PASSWORD", "") - viper.SetDefault("POSTGRES_DB", "postgres") - - viper.AutomaticEnv() - - if env.GetString("ENV") != "local" { - logrus.Info("running in non-local environment, skipping environment configuration") - } else { - fi := "local" - if len(os.Args) > 1 { - fi = os.Args[1] - } - envFile := util.ResolveEnvFile("indexer", fi) - util.LoadEncryptedEnvFile(envFile) - } -} diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go deleted file mode 100644 index 0c2c4c706..000000000 --- a/cmd/indexer/main.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - _ "net/http/pprof" - - "github.com/mikeydub/go-gallery/indexer/cmd" -) - -func main() { - - cmd.Execute() - -} diff --git a/cmd/reprocess/main.go b/cmd/reprocess/main.go deleted file mode 100644 index 58547443c..000000000 --- a/cmd/reprocess/main.go +++ /dev/null @@ -1,241 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "os" - "time" - - "net/http" - _ "net/http/pprof" - - "github.com/mikeydub/go-gallery/db/gen/coredb" - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/server" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/metric" - "github.com/mikeydub/go-gallery/service/persist" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" - "github.com/mikeydub/go-gallery/tokenprocessing" - "github.com/mikeydub/go-gallery/util" - "github.com/sirupsen/logrus" - "github.com/sourcegraph/conc/pool" - "github.com/spf13/viper" -) - -func main() { - - setDefaults() - - start := time.Now() - defer func() { - elapsed := time.Since(start) - fmt.Printf("Took %s", elapsed) - }() - ctx := context.Background() - clients := server.ClientInit(ctx) - defer clients.Close() - - provider, cleanup := server.NewMultichainProvider(ctx, server.SetDefaults) - defer cleanup() - tp := tokenprocessing.NewTokenProcessor(clients.Queries, clients.EthClient, clients.HTTPClient, provider, clients.IPFSClient, clients.ArweaveClient, clients.StorageClient, env.GetString("GCLOUD_TOKEN_CONTENT_BUCKET"), clients.Repos.TokenRepository, metric.NewLogMetricReporter()) - - var rows []coredb.GetSVGTokensWithContractsByIDsRow - var err error - - if env.GetString("CLOUD_RUN_JOB") != "" { - logrus.Infof("running as cloud run job") - - tokenprocessing.InitSentry() - logger.InitWithGCPDefaults() - - jobIndex := env.GetInt("CLOUD_RUN_TASK_INDEX") - jobCount := env.GetInt("CLOUD_RUN_TASK_COUNT") - - // given the totalTokenCount, and the jobCount, we can calculate the offset and limit for this job - // we want to evenly distribute the work across the jobs - // so we can calculate the limit by dividing the totalTokenCount by the jobCount - // and the offset by multiplying the jobIndex by the limit - - logrus.Infof("jobIndex: %d, jobCount: %d", jobIndex, jobCount) - - r, err := clients.Queries.GetReprocessJobRangeByID(ctx, jobIndex) - if err != nil { - logrus.Errorf("error getting job range: %v", err) - panic(err) - } - - rows, err = clients.Queries.GetSVGTokensWithContractsByIDs(ctx, coredb.GetSVGTokensWithContractsByIDsParams{ - StartID: r.TokenStartID, - EndID: r.TokenEndID, - }) - if err != nil { - panic(err) - } - } else { - - logrus.Infof("running as local job") - logger.SetLoggerOptions(func(logger *logrus.Logger) { - fi, err := os.Create(fmt.Sprintf("reprocess-%s.log", time.Now().Format("2006-01-02T15-04-05"))) - if err != nil { - panic(err) - } - logger.SetOutput(io.MultiWriter(fi, os.Stdout)) - }) - - r, err := clients.Queries.GetReprocessJobRangeByID(ctx, 0) - if err != nil { - logrus.Errorf("error getting job range: %v", err) - panic(err) - } - rows, err = clients.Queries.GetSVGTokensWithContractsByIDs(ctx, coredb.GetSVGTokensWithContractsByIDsParams{ - StartID: r.TokenStartID, - EndID: r.TokenEndID, - }) - if err != nil { - panic(err) - } - } - - if err != nil { - logrus.Errorf("error getting tokens: %v", err) - panic(err) - } - - wp := pool.New().WithMaxGoroutines(25).WithContext(ctx) - - logrus.Infof("processing (%d) tokens...", len(rows)) - - totalTokens := 0 - - for _, row := range rows { - - if row.IsProviderMarkedSpam.Bool || row.IsUserMarkedSpam.Bool || row.IsProviderMarkedSpam_2 { - logrus.Infof("skipping token %s because it is marked as spam", row.ID) - continue - } - - totalTokens++ - - wallets := []persist.Wallet{} - for _, w := range row.OwnedByWallets { - wallets = append(wallets, persist.Wallet{ID: w}) - } - - token := persist.TokenGallery{ - Version: persist.NullInt32(row.Version.Int32), - ID: row.ID, - CreationTime: row.CreatedAt, - Deleted: persist.NullBool(row.Deleted), - LastUpdated: row.LastUpdated, - LastSynced: row.LastSynced, - CollectorsNote: persist.NullString(row.CollectorsNote.String), - // Media: row.Media, don't include this row because we want all media to be replaced - FallbackMedia: row.FallbackMedia, - TokenType: persist.TokenType(row.TokenType.String), - Chain: row.Chain, - Name: persist.NullString(row.Name.String), - Description: persist.NullString(row.Description.String), - TokenURI: persist.TokenURI(row.TokenUri.String), - TokenID: row.TokenID, - Quantity: row.Quantity, - OwnerUserID: row.OwnerUserID, - OwnedByWallets: wallets, - OwnershipHistory: row.OwnershipHistory, - IsCreatorToken: row.IsCreatorToken, - Contract: persist.ContractGallery{ - ID: row.Contract, - }, - ExternalURL: persist.NullString(row.ExternalUrl.String), - BlockNumber: persist.BlockNumber(row.BlockNumber.Int64), - } - contract := persist.ContractGallery{ - Version: persist.NullInt32(row.Version_2.Int32), - ID: row.ID_2, - CreationTime: row.CreatedAt_2, - Deleted: persist.NullBool(row.Deleted_2), - LastUpdated: row.LastUpdated_2, - Chain: row.Chain_2, - Address: row.Address, - Symbol: persist.NullString(row.Symbol.String), - Name: persist.NullString(row.Name_2.String), - Description: persist.NullString(row.Description_2.String), - OwnerAddress: row.OwnerAddress, - CreatorAddress: row.CreatorAddress, - ProfileImageURL: persist.NullString(row.ProfileImageUrl.String), - ProfileBannerURL: persist.NullString(row.ProfileBannerUrl.String), - BadgeURL: persist.NullString(row.BadgeUrl.String), - OverrideCreatorUserID: row.OverrideCreatorUserID, - } - - wp.Go(func(ctx context.Context) error { - ctx = sentryutil.NewSentryHubContext(ctx) - logrus.Infof("processing %s", token.ID) - defer func() { - logger.For(ctx).Infof("finished processing %s", token.ID) - }() - t := persist.TokenIdentifiers{ - Chain: token.Chain, - TokenID: token.TokenID, - ContractAddress: contract.Address, - } - _, err = tp.ProcessTokenPipeline(ctx, t, contract, persist.ProcessingCauseRefresh) - return err - }) - } - go func() { - http.ListenAndServe(":6060", nil) - }() - err = wp.Wait() - logrus.Infof("finished processes for %d tokens with err: %s", totalTokens, err) - -} - -func setDefaults() { - viper.SetDefault("ENV", "local") - viper.SetDefault("POSTGRES_HOST", "0.0.0.0") - viper.SetDefault("POSTGRES_PORT", 5432) - viper.SetDefault("POSTGRES_USER", "gallery_backend") - viper.SetDefault("POSTGRES_PASSWORD", "") - viper.SetDefault("POSTGRES_DB", "postgres") - viper.SetDefault("INDEXER_HOST", "http://localhost:6000") - viper.SetDefault("ALCHEMY_API_URL", "") - viper.SetDefault("ALCHEMY_OPTIMISM_API_URL", "") - viper.SetDefault("ALCHEMY_POLYGON_API_URL", "") - viper.SetDefault("ALCHEMY_NFT_API_URL", "") - viper.SetDefault("INFURA_API_KEY", "") - viper.SetDefault("INFURA_API_SECRET", "") - viper.SetDefault("TEZOS_API_URL", "https://api.tzkt.io") - viper.SetDefault("POAP_API_KEY", "") - viper.SetDefault("POAP_AUTH_TOKEN", "") - viper.SetDefault("OPENSEA_API_KEY", "") - viper.SetDefault("RPC_URL", "https://eth-goerli.g.alchemy.com/v2/_2u--i79yarLYdOT4Bgydqa0dBceVRLD") - viper.SetDefault("REDIS_URL", "localhost:6379") - viper.SetDefault("IPFS_URL", "https://gallery.infura-ipfs.io") - viper.SetDefault("FALLBACK_IPFS_URL", "https://ipfs.io") - viper.SetDefault("IPFS_API_URL", "https://ipfs.infura.io:5001") - viper.SetDefault("IPFS_PROJECT_ID", "") - viper.SetDefault("IPFS_PROJECT_SECRET", "") - viper.SetDefault("IMGIX_API_KEY", "") - viper.SetDefault("GCLOUD_TOKEN_CONTENT_BUCKET", "dev-token-content") - viper.SetDefault("CLOUD_RUN_JOB", "") - viper.SetDefault("CLOUD_RUN_TASK_INDEX", 0) - viper.SetDefault("CLOUD_RUN_TASK_COUNT", 1) - viper.SetDefault("SENTRY_DSN", "") - viper.SetDefault("SENTRY_TRACES_SAMPLE_RATE", 0.2) - viper.SetDefault("VERSION", "0") - - viper.AutomaticEnv() - - if env.GetString("ENV") != "local" { - logrus.Info("running in non-local environment, skipping environment configuration") - } else { - fi := "local" - if len(os.Args) > 1 { - fi = os.Args[1] - } - envFile := util.ResolveEnvFile("tokenprocessing", fi) - util.LoadEncryptedEnvFile(envFile) - } -} diff --git a/cmd/reprocess/orchestrator/main.go b/cmd/reprocess/orchestrator/main.go deleted file mode 100644 index 565bfd974..000000000 --- a/cmd/reprocess/orchestrator/main.go +++ /dev/null @@ -1,111 +0,0 @@ -package main - -import ( - "context" - "fmt" - "math" - "os" - "time" - - _ "net/http/pprof" - - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/persist/postgres" - "github.com/mikeydub/go-gallery/tokenprocessing" - "github.com/mikeydub/go-gallery/util" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" -) - -const ( - totalJobs = 1000 -) - -type jobRange struct { - id int - start persist.DBID - end persist.DBID -} - -func main() { - - setDefaults() - - start := time.Now() - defer func() { - elapsed := time.Since(start) - fmt.Printf("Took %s", elapsed) - }() - ctx := context.Background() - pg := postgres.NewPgxClient() - - var totalTokenCount int - - err := pg.QueryRow(ctx, fmt.Sprintf(`select count(*) from tokens %s;`, tokenprocessing.SVGQuery)).Scan(&totalTokenCount) - if err != nil { - logrus.Errorf("error getting total token count: %v", err) - panic(err) - } - - limit := int(math.Ceil(float64(totalTokenCount) / float64(totalJobs))) - - rows, err := pg.Query(ctx, fmt.Sprintf(`select tokens.id from tokens %s order by tokens.id;`, tokenprocessing.SVGQuery)) - if err != nil { - logrus.Errorf("error getting token ids: %v", err) - panic(err) - } - - var curRange jobRange - - for i := 0; rows.Next(); i++ { - var id persist.DBID - err := rows.Scan(&id) - if err != nil { - logrus.Errorf("error scanning token id: %v", err) - panic(err) - } - - if i%limit == 0 { - fmt.Printf("starting job range %d (start: %s)\n", i/limit, id) - curRange = jobRange{ - id: i / limit, - start: id, - } - } - - if i%limit == limit-1 { - fmt.Printf("ending job range %d (end: %s)\n", curRange.id, id) - curRange.end = id - _, err = pg.Exec(ctx, `INSERT INTO reprocess_jobs (id, token_start_id, token_end_id) VALUES ($1, $2, $3);`, curRange.id, curRange.start, curRange.end) - if err != nil { - logrus.Errorf("error inserting job: %v", err) - panic(err) - } - } - } - - fmt.Printf("Inserted %d jobs with a limit of %d and %d total tokens\n", curRange.id, limit, totalTokenCount) -} - -func setDefaults() { - viper.SetDefault("ENV", "local") - viper.SetDefault("POSTGRES_HOST", "0.0.0.0") - viper.SetDefault("POSTGRES_PORT", 5432) - viper.SetDefault("POSTGRES_USER", "gallery_backend") - viper.SetDefault("POSTGRES_PASSWORD", "") - viper.SetDefault("POSTGRES_DB", "postgres") - - viper.AutomaticEnv() - - if env.GetString("ENV") != "local" { - logrus.Info("running in non-local environment, skipping environment configuration") - } else { - fi := "local" - if len(os.Args) > 1 { - fi = os.Args[1] - } - envFile := util.ResolveEnvFile("tokenprocessing", fi) - util.LoadEncryptedEnvFile(envFile) - } -} diff --git a/cmd/token_normalization_migrate/main.go b/cmd/token_normalization_migrate/main.go deleted file mode 100644 index a6e3d4365..000000000 --- a/cmd/token_normalization_migrate/main.go +++ /dev/null @@ -1,618 +0,0 @@ -// Script to generate the token_definitions table from tokens and token_medias tables -package main - -import ( - "context" - "database/sql" - "fmt" - "strings" - "time" - - "github.com/gammazero/workerpool" - "github.com/spf13/cobra" - - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/server" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/util" -) - -const ( - poolSize = 64 // concurrent workers to use - batchSize = 100 // number of tokens to process per worker -) - -var targetEnv string - -func init() { - rootCmd.AddCommand(saveCmd) - rootCmd.AddCommand(migrateCmd) - rootCmd.PersistentFlags().StringVarP(&targetEnv, "env", "e", "local", "env to run with") - server.SetDefaults() -} - -func setEnv() { - if targetEnv != "local" { - envFile := util.ResolveEnvFile("tokennorm", targetEnv) - util.LoadEncryptedEnvFile(envFile) - } -} - -func main() { - rootCmd.Execute() -} - -var rootCmd = &cobra.Command{ - Use: "tokenmigrate", - Short: "create token_definitions table", -} - -var saveCmd = &cobra.Command{ - Use: "stage", - Short: "create token_chunks table (for dev purposes)", - Run: func(cmd *cobra.Command, args []string) { - setEnv() - pq = createPostgresClient() - pq.SetMaxIdleConns(2 * poolSize) - defer pq.Close() - saveStagingTable() - }, -} - -var migrateCmd = &cobra.Command{ - Use: "migrate", - Short: "create token_definitions table", - Run: func(cmd *cobra.Command, args []string) { - setEnv() - pq = createPostgresClient() - pq.SetMaxIdleConns(2 * poolSize) - defer pq.Close() - createTokenDefinitions(context.Background(), pq) - }, -} - -var ( - pq *sql.DB - batchTokensStmt *sql.Stmt - batchMediasStmt *sql.Stmt - batchInsertStmt *sql.Stmt -) - -const createStagingTable = ` -drop table if exists token_chunks; - -create table token_chunks( - id serial primary key, - chain int not null, - contract_id varchar not null, - token_id varchar not null, - address varchar not null -); - -insert into token_chunks(chain, contract_id, token_id, address) ( -select - tokens.chain, - tokens.contract, - tokens.token_id, - contracts.address -from tokens -join contracts on tokens.contract = contracts.id -group by 1, 2, 3, 4 -order by 1, 2, 3, 4);` - -const batchTokensQuery = ` -with chunk as (select * from token_chunks where id >= $1 and id < $2) -select - c.id chunk_id, - c.chain chunk_chain, - c.contract_id chunk_contract_id, - c.token_id chunk_token_id, - t.token_type token_token_type, - t.external_url token_external_url, - t.fallback_media token_fallback_media, - t.deleted as token_deleted, - c.address contract_address -from tokens t -join chunk c on (t.chain, t.contract, t.token_id) = (c.chain, c.contract_id, c.token_id) -order by t.deleted desc, t.last_updated desc;` - -const batchMediasQuery = ` -with chunk as (select * from token_chunks where id >= $1 and id < $2) -select - c.id chunk_id, - c.chain chunk_chain, - c.contract_id chunk_contract_id, - c.token_id chunk_token_id, - tm.id token_media_id, - tm.name media_name, - tm.description media_description, - tm.metadata media_metadata, - tm.media media_media, - tm.active media_active -from token_medias tm -join chunk c on (tm.chain, tm.contract_id, tm.token_id) = (c.chain, c.contract_id, c.token_id) - and not tm.deleted -order by tm.active desc, tm.last_updated desc;` - -const dropConstraints = ` -alter table token_definitions set (autovacuum_enabled = false); -alter table token_definitions set unlogged; -alter table token_definitions drop constraint if exists token_definitions_pkey; -alter table token_definitions drop constraint if exists token_definitions_contract_id_fkey; -alter table token_definitions drop constraint if exists token_definitions_token_media_id_fkey; -alter table token_definitions drop constraint if exists token_definitions_contract_id_chain_contract_address_fkey; -drop index if exists token_definitions_chain_contract_id_token_idx; -drop index if exists token_definitions_chain_contract_address_token_idx; -drop index if exists token_definitions_contract_id_idx;` - -const addConstraints = ` -alter table token_definitions add primary key(id); -alter table token_definitions add constraint token_definitions_contract_id_fkey foreign key(contract_id) references contracts(id); -alter table token_definitions add constraint token_definitions_token_media_id_fkey foreign key(token_media_id) references token_medias(id); -alter table token_definitions add constraint token_definitions_contract_id_chain_contract_address_fkey foreign key(contract_id, chain, contract_address) references contracts(id, chain, address) on update cascade; -create unique index if not exists token_definitions_chain_contract_id_token_idx on token_definitions(chain, contract_id, token_id) where not deleted; -create unique index token_definitions_chain_contract_address_token_idx on token_definitions(chain, contract_address, token_id) where not deleted; -create index token_definitions_contract_id_idx on token_definitions(contract_id) where not deleted; -alter table token_definitions set (autovacuum_enabled = true); -alter table token_definitions set logged; -analyze token_definitions;` - -const insertBatch = ` -insert into token_definitions ( - id, - name, - description, - external_url, - metadata, - fallback_media, - contract_id, - token_media_id, - chain, - contract_address, - token_id, - token_type -) ( - select - id, - nullif(name, ''), - nullif(description, ''), - nullif(external_url, ''), - metadata, - fallback_media, - contract_id, - nullif(token_media_id, ''), - chain, - contract_address, - token_id, - token_type - from ( - select - unnest($1::varchar[]) id, - unnest($2::varchar[]) name, - unnest($3::varchar[]) description, - unnest($4::varchar[]) external_url, - unnest($5::jsonb[]) metadata, - unnest($6::jsonb[]) fallback_media, - unnest($7::varchar[]) contract_id, - unnest($8::varchar[]) token_media_id, - unnest($9::int[]) chain, - unnest($10::varchar[]) contract_address, - unnest($11::varchar[]) token_id, - unnest($12::varchar[]) token_type - ) vals -);` - -type instanceData struct { - ChunkID int - Chain persist.Chain - ContractID persist.NullString - TokenTokenID persist.NullString - TokenName persist.NullString - TokenDescription persist.NullString - TokenTokenType persist.TokenType - TokenExternalURL persist.NullString - TokenFallbackMedia persist.FallbackMedia - TokenMediaID persist.NullString - TokenDeleted bool - ContractAddress persist.Address - MediaName persist.NullString - MediaDescription persist.NullString - MediaMetadata persist.TokenMetadata - MediaMedia persist.Media - MediaActive persist.NullBool -} - -type mergedData struct { - ID []persist.DBID - Name []persist.NullString - Description []persist.NullString - ExternalURL []persist.NullString - Metadata []persist.TokenMetadata - Fallback []persist.FallbackMedia - ContractID []persist.NullString - MediaID []persist.NullString - Chain []persist.Chain - ContractAddress []persist.Address - TokenID []persist.NullString - TokenType []persist.TokenType - MediaActive []bool - MediaMedia []persist.Media -} - -func requireMustBeEmpty() { - var currentCount int - err := pq.QueryRow("select count(*) from token_definitions").Scan(¤tCount) - check(err) - if currentCount > 0 { - panic(fmt.Sprintf("token_definitions table is not empty, current count: %d", currentCount)) - } -} - -func analyzeTokens() { - fmt.Print("analyzing tokens table") - _, err := pq.Exec("analyze tokens;") - check(err) - fmt.Println("...done") -} - -func lockTables(tx *sql.Tx) { - _, err := tx.Exec("lock table tokens in access share mode;") - check(err) - _, err = tx.Exec("lock table token_medias in access share mode;") - check(err) -} - -func prepareStatements() { - fmt.Print("preparing statements") - var err error - batchTokensStmt, err = pq.Prepare(batchTokensQuery) - check(err) - batchMediasStmt, err = pq.Prepare(batchMediasQuery) - check(err) - batchInsertStmt, err = pq.Prepare(insertBatch) - check(err) - fmt.Println("...done") -} - -func dropTokenDefinitionConstraints() { - fmt.Print("dropping constraints") - _, err := pq.Exec(dropConstraints) - check(err) - fmt.Println("...done") -} - -func createTokenDefinitions(ctx context.Context, pq *sql.DB) { - globalStart := time.Now() - - defer func() { - pq.Exec("drop table if exists token_chunks") - }() - - tx, err := pq.BeginTx(ctx, nil) - check(err) - - prepareStatements() - requireMustBeEmpty() - analyzeTokens() - dropTokenDefinitionConstraints() - lockTables(tx) - totalTokens := saveStagingTable() - - wp := workerpool.New(poolSize) - - start := 0 - end := start + batchSize - totalBatches := totalTokens / batchSize - - for chunkID := 0; start < totalTokens; chunkID++ { - chunkID := chunkID - s := start - e := end - wp.Submit(func() { - batchStart := time.Now() - - tokenCh := make(chan bool) - mediaCh := make(chan bool) - - var tokenRows *sql.Rows - var tokenQueryStart time.Time - var tokenQueryEnd time.Time - var mediaRows *sql.Rows - var mediaQueryStart time.Time - var mediaQueryEnd time.Time - - go func() { - tokenQueryStart = time.Now() - tokenRows, err = batchTokensStmt.Query(s, e) - check(err) - tokenQueryEnd = time.Now() - tokenCh <- true - }() - - go func() { - mediaQueryStart = time.Now() - mediaRows, err = batchMediasStmt.Query(s, e) - check(err) - mediaQueryEnd = time.Now() - mediaCh <- true - }() - - <-tokenCh - <-mediaCh - - idToIdx := make(map[int]int) - - m := mergedData{ - ID: make([]persist.DBID, 0, batchSize), - Name: make([]persist.NullString, 0, batchSize), - Description: make([]persist.NullString, 0, batchSize), - ExternalURL: make([]persist.NullString, 0, batchSize), - Metadata: make([]persist.TokenMetadata, 0, batchSize), - Fallback: make([]persist.FallbackMedia, 0, batchSize), - ContractID: make([]persist.NullString, 0, batchSize), - MediaID: make([]persist.NullString, 0, batchSize), - Chain: make([]persist.Chain, 0, batchSize), - ContractAddress: make([]persist.Address, 0, batchSize), - TokenID: make([]persist.NullString, 0, batchSize), - TokenType: make([]persist.TokenType, 0, batchSize), - MediaActive: make([]bool, 0, batchSize), - MediaMedia: make([]persist.Media, 0, batchSize), - } - - instanceCount := 0 - - for tokenRows.Next() { - instanceCount += 1 - var r instanceData - err := tokenRows.Scan( - &r.ChunkID, - &r.Chain, - &r.ContractID, - &r.TokenTokenID, - &r.TokenTokenType, - &r.TokenExternalURL, - &r.TokenFallbackMedia, - &r.TokenDeleted, - &r.ContractAddress, - ) - check(err) - - if _, ok := idToIdx[r.ChunkID]; !ok { - idToIdx[r.ChunkID] = len(m.ID) - m.ID = append(m.ID, persist.GenerateID()) - m.Chain = append(m.Chain, r.Chain) - m.ContractID = append(m.ContractID, r.ContractID) - m.TokenID = append(m.TokenID, r.TokenTokenID) - m.TokenType = append(m.TokenType, r.TokenTokenType) - m.ExternalURL = append(m.ExternalURL, r.TokenExternalURL) - m.Fallback = append(m.Fallback, r.TokenFallbackMedia) - m.ContractAddress = append(m.ContractAddress, r.ContractAddress) - // Filled in with data from media query - m.Name = append(m.Name, "") - m.Description = append(m.Description, "") - m.Metadata = append(m.Metadata, persist.TokenMetadata{}) - m.MediaID = append(m.MediaID, "") - m.MediaActive = append(m.MediaActive, false) - m.MediaMedia = append(m.MediaMedia, persist.Media{}) - } else if !r.TokenDeleted { - idx := idToIdx[r.ChunkID] - if m.TokenType[idx] == "" { - m.TokenType[idx] = r.TokenTokenType - } - if m.ContractID[idx].String() == "" { - m.ContractID[idx] = r.ContractID - } - if m.Fallback[idx].ImageURL.String() == "" { - m.Metadata[idx] = r.MediaMetadata - } - if m.ExternalURL[idx].String() == "" { - m.ExternalURL[idx] = r.TokenExternalURL - } - } - } - tokenRows.Close() - - for mediaRows.Next() { - var r instanceData - err := mediaRows.Scan( - &r.ChunkID, - &r.Chain, - &r.ContractID, - &r.TokenTokenID, - &r.TokenMediaID, - &r.MediaName, - &r.MediaDescription, - &r.MediaMetadata, - &r.MediaMedia, - &r.MediaActive, - ) - check(err) - idx, ok := idToIdx[r.ChunkID] - if !ok { - panic(fmt.Sprintf("no token found for media with staging_id=%d", r.ChunkID)) - } - if m.Name[idx].String() == "" { - m.Name[idx] = r.MediaName - } - if m.Description[idx].String() == "" { - m.Description[idx] = r.MediaDescription - } - if len(m.Metadata[idx]) == 0 { - m.Metadata[idx] = r.MediaMetadata - } - if m.MediaID[idx].String() == "" { - m.MediaID[idx] = r.TokenMediaID - } else if !m.MediaActive[idx] && r.MediaActive.Bool() { - m.MediaID[idx] = r.TokenMediaID - } else if !m.MediaMedia[idx].IsServable() && r.MediaMedia.IsServable() { - m.MediaID[idx] = r.TokenMediaID - } else if r.MediaMedia.MediaType.IsMorePriorityThan(m.MediaMedia[idx].MediaType) { - m.MediaID[idx] = r.TokenMediaID - } - } - mediaRows.Close() - - insertStart := time.Now() - _, err := batchInsertStmt.Exec( - m.ID, - m.Name, - m.Description, - m.ExternalURL, - m.Metadata, - m.Fallback, - m.ContractID, - m.MediaID, - m.Chain, - m.ContractAddress, - m.TokenID, - m.TokenType, - ) - check(err) - insertEnd := time.Now() - - fmt.Printf("chunk(id=%d) [%d, %d) %d/%d; tokenQuery %s; mediaQuery %s; insert %s; total %s\n", chunkID, s, e, chunkID, totalBatches, tokenQueryEnd.Sub(tokenQueryStart), mediaQueryEnd.Sub(mediaQueryStart), insertEnd.Sub(insertStart), time.Since(batchStart)) - }) - - start = end - end = start + batchSize - } - - wp.StopWait() - - err = tx.Commit() - check(err) - - fmt.Println("adding back constraints") - now := time.Now() - _, err = pq.Exec(addConstraints) - check(err) - - fmt.Printf("took %s to migrate tokens; adding constaints=%s\n", time.Since(globalStart), time.Since(now)) -} - -func saveStagingTable() int { - fmt.Print("creating token_chunks table") - r, err := pq.Exec(createStagingTable) - check(err) - rows, err := r.RowsAffected() - check(err) - _, err = pq.Exec("analyze token_chunks") - check(err) - fmt.Printf("...created token_chunks table with %d rows\n", rows) - return int(rows) -} - -func check(err error) { - if err != nil { - panic(err) - } - -} - -func createPostgresClient() *sql.DB { - c, err := NewClient() - if err != nil { - panic(err) - } - return c -} - -// Start of copied code --------------------------------------- -// Copied from persist/postgres/postgres.go -// Need to create an SSL connection without the proxy because it -// doesn't support connection pooling - -type ErrRoleDoesNotExist struct { - role string -} - -func (e ErrRoleDoesNotExist) Error() string { - return fmt.Sprintf("role '%s' does not exist", e.role) -} - -type connectionParams struct { - user string - password string - dbname string - host string - port int -} - -func (c *connectionParams) toConnectionString() string { - port := c.port - if port == 0 { - port = 5432 - } - - connStr := fmt.Sprintf("user=%s dbname=%s host=%s port=%d", c.user, c.dbname, c.host, port) - - // Empty passwords should be omitted so they don't interfere with other parameters - // (e.g. "password= dbname=something" causes Postgres to ignore the dbname) - if c.password != "" { - connStr += fmt.Sprintf(" password=%s", c.password) - } - - countNonEmptyStrings := func(str ...string) int { - numNotEmpty := 0 - for _, s := range str { - if s != "" { - numNotEmpty++ - } - } - - return numNotEmpty - } - - dbServerCa := env.GetString("POSTGRES_SERVER_CA") - dbClientKey := env.GetString("POSTGRES_CLIENT_KEY") - dbClientCert := env.GetString("POSTGRES_CLIENT_CERT") - - numSSLParams := countNonEmptyStrings(dbServerCa, dbClientKey, dbClientCert) - if numSSLParams == 0 { - return connStr - } else if numSSLParams == 3 { - return connStr + fmt.Sprintf(" sslmode=verify-ca sslrootcert=%s sslcert=%s sslkey=%s", dbServerCa, dbClientCert, dbClientKey) - } - - panic(fmt.Errorf("POSTGRES_SERVER_CA, POSTGRES_CLIENT_KEY, and POSTGRES_CLIENT_CERT must be set together (all must have values or all must be empty)")) -} - -func newConnectionParamsFromEnv() connectionParams { - return connectionParams{ - user: env.GetString("POSTGRES_USER"), - password: env.GetString("POSTGRES_PASSWORD"), - dbname: env.GetString("POSTGRES_DB"), - host: env.GetString("POSTGRES_HOST"), - port: env.GetInt("POSTGRES_PORT"), - } -} - -type ConnectionOption func(params *connectionParams) - -func NewClient(opts ...ConnectionOption) (*sql.DB, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - params := newConnectionParamsFromEnv() - for _, opt := range opts { - opt(¶ms) - } - - db, err := sql.Open("pgx", params.toConnectionString()) - if err != nil { - return nil, err - } - - db.SetMaxOpenConns(50) - - err = db.PingContext(ctx) - if err != nil && strings.Contains(err.Error(), fmt.Sprintf("role \"%s\" does not exist", params.user)) { - return nil, ErrRoleDoesNotExist{params.user} - } - if err != nil { - return nil, err - } - return db, nil -} - -// End of copied code --------------------------------------- diff --git a/db/gen/indexerdb/db_gen.go b/db/gen/indexerdb/db_gen.go deleted file mode 100644 index e3c6fff83..000000000 --- a/db/gen/indexerdb/db_gen.go +++ /dev/null @@ -1,32 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.26.0 - -package indexerdb - -import ( - "context" - - "github.com/jackc/pgconn" - "github.com/jackc/pgx/v4" -) - -type DBTX interface { - Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) - Query(context.Context, string, ...interface{}) (pgx.Rows, error) - QueryRow(context.Context, string, ...interface{}) pgx.Row -} - -func New(db DBTX) *Queries { - return &Queries{db: db} -} - -type Queries struct { - db DBTX -} - -func (q *Queries) WithTx(tx pgx.Tx) *Queries { - return &Queries{ - db: tx, - } -} diff --git a/db/gen/indexerdb/models_gen.go b/db/gen/indexerdb/models_gen.go deleted file mode 100644 index 24d8be52c..000000000 --- a/db/gen/indexerdb/models_gen.go +++ /dev/null @@ -1,76 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.26.0 - -package indexerdb - -import ( - "database/sql" - "time" - - "github.com/jackc/pgtype" - "github.com/mikeydub/go-gallery/service/persist" -) - -type BlockchainStatistic struct { - ID persist.DBID - Deleted bool - Version int32 - CreatedAt time.Time - LastUpdated time.Time - BlockStart persist.BlockNumber - BlockEnd persist.BlockNumber - TotalLogs sql.NullInt64 - TotalTransfers sql.NullInt64 - TotalTokens sql.NullInt64 - TotalContracts sql.NullInt64 - Success bool - ContractStats pgtype.JSONB - TokenStats pgtype.JSONB - ProcessingTimeSeconds sql.NullInt64 -} - -type Contract struct { - ID persist.DBID - Deleted bool - Version sql.NullInt32 - CreatedAt time.Time - LastUpdated time.Time - Name sql.NullString - Symbol sql.NullString - Address persist.EthereumAddress - CreatorAddress persist.EthereumAddress - Chain persist.Chain - LatestBlock sql.NullInt64 - OwnerAddress persist.EthereumAddress - OwnerMethod persist.ContractOwnerMethod -} - -type ReprocessJob struct { - ID int - StartID string - EndID string -} - -type Token struct { - ID persist.DBID - Deleted bool - Version sql.NullInt32 - CreatedAt time.Time - LastUpdated time.Time - Name sql.NullString - Description sql.NullString - ContractAddress sql.NullString - Media pgtype.JSONB - OwnerAddress sql.NullString - TokenUri sql.NullString - TokenType sql.NullString - TokenID sql.NullString - Quantity sql.NullString - OwnershipHistory []pgtype.JSONB - TokenMetadata pgtype.JSONB - ExternalUrl sql.NullString - BlockNumber sql.NullInt64 - Chain persist.Chain - IsSpam sql.NullBool -} diff --git a/db/gen/indexerdb/query.sql.go b/db/gen/indexerdb/query.sql.go deleted file mode 100644 index 3711c4ea5..000000000 --- a/db/gen/indexerdb/query.sql.go +++ /dev/null @@ -1,224 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.26.0 -// source: query.sql - -package indexerdb - -import ( - "context" - "database/sql" - - "github.com/jackc/pgtype" - "github.com/mikeydub/go-gallery/service/persist" -) - -const firstContract = `-- name: FirstContract :one -SELECT id, deleted, version, created_at, last_updated, name, symbol, address, creator_address, chain, latest_block, owner_address, owner_method FROM contracts LIMIT 1 -` - -// sqlc needs at least one query in order to generate the models. -func (q *Queries) FirstContract(ctx context.Context) (Contract, error) { - row := q.db.QueryRow(ctx, firstContract) - var i Contract - err := row.Scan( - &i.ID, - &i.Deleted, - &i.Version, - &i.CreatedAt, - &i.LastUpdated, - &i.Name, - &i.Symbol, - &i.Address, - &i.CreatorAddress, - &i.Chain, - &i.LatestBlock, - &i.OwnerAddress, - &i.OwnerMethod, - ) - return i, err -} - -const getContractsByIDRange = `-- name: GetContractsByIDRange :many -SELECT - contracts.id, contracts.deleted, contracts.version, contracts.created_at, contracts.last_updated, contracts.name, contracts.symbol, contracts.address, contracts.creator_address, contracts.chain, contracts.latest_block, contracts.owner_address, contracts.owner_method -FROM contracts -WHERE contracts.deleted = false -AND (contracts.owner_address IS NULL OR contracts.owner_address = '' OR contracts.creator_address IS NULL OR contracts.creator_address = '') -AND contracts.id >= $1::text AND contracts.id < $2::text -ORDER BY contracts.id -` - -type GetContractsByIDRangeParams struct { - StartID string - EndID string -} - -func (q *Queries) GetContractsByIDRange(ctx context.Context, arg GetContractsByIDRangeParams) ([]Contract, error) { - rows, err := q.db.Query(ctx, getContractsByIDRange, arg.StartID, arg.EndID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []Contract - for rows.Next() { - var i Contract - if err := rows.Scan( - &i.ID, - &i.Deleted, - &i.Version, - &i.CreatedAt, - &i.LastUpdated, - &i.Name, - &i.Symbol, - &i.Address, - &i.CreatorAddress, - &i.Chain, - &i.LatestBlock, - &i.OwnerAddress, - &i.OwnerMethod, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const getReprocessJobRangeByID = `-- name: GetReprocessJobRangeByID :one -select id, start_id, end_id from reprocess_jobs where id = $1 -` - -func (q *Queries) GetReprocessJobRangeByID(ctx context.Context, id int) (ReprocessJob, error) { - row := q.db.QueryRow(ctx, getReprocessJobRangeByID, id) - var i ReprocessJob - err := row.Scan(&i.ID, &i.StartID, &i.EndID) - return i, err -} - -const insertStatistic = `-- name: InsertStatistic :one -insert into blockchain_statistics (id, block_start, block_end) values ($1, $2, $3) on conflict do nothing returning id -` - -type InsertStatisticParams struct { - ID persist.DBID - BlockStart persist.BlockNumber - BlockEnd persist.BlockNumber -} - -func (q *Queries) InsertStatistic(ctx context.Context, arg InsertStatisticParams) (persist.DBID, error) { - row := q.db.QueryRow(ctx, insertStatistic, arg.ID, arg.BlockStart, arg.BlockEnd) - var id persist.DBID - err := row.Scan(&id) - return id, err -} - -const updateContractOwnerByID = `-- name: UpdateContractOwnerByID :exec -update contracts set owner_address = $1, creator_address = $2, owner_method = $3 where id = $4 and deleted = false -` - -type UpdateContractOwnerByIDParams struct { - OwnerAddress persist.EthereumAddress - CreatorAddress persist.EthereumAddress - OwnerMethod persist.ContractOwnerMethod - ID persist.DBID -} - -func (q *Queries) UpdateContractOwnerByID(ctx context.Context, arg UpdateContractOwnerByIDParams) error { - _, err := q.db.Exec(ctx, updateContractOwnerByID, - arg.OwnerAddress, - arg.CreatorAddress, - arg.OwnerMethod, - arg.ID, - ) - return err -} - -const updateStatisticContractStats = `-- name: UpdateStatisticContractStats :exec -update blockchain_statistics set contract_stats = $1 where id = $2 -` - -type UpdateStatisticContractStatsParams struct { - ContractStats pgtype.JSONB - ID persist.DBID -} - -func (q *Queries) UpdateStatisticContractStats(ctx context.Context, arg UpdateStatisticContractStatsParams) error { - _, err := q.db.Exec(ctx, updateStatisticContractStats, arg.ContractStats, arg.ID) - return err -} - -const updateStatisticSuccess = `-- name: UpdateStatisticSuccess :exec -update blockchain_statistics set success = $1, processing_time_seconds = $2 where id = $3 -` - -type UpdateStatisticSuccessParams struct { - Success bool - ProcessingTimeSeconds sql.NullInt64 - ID persist.DBID -} - -func (q *Queries) UpdateStatisticSuccess(ctx context.Context, arg UpdateStatisticSuccessParams) error { - _, err := q.db.Exec(ctx, updateStatisticSuccess, arg.Success, arg.ProcessingTimeSeconds, arg.ID) - return err -} - -const updateStatisticTokenStats = `-- name: UpdateStatisticTokenStats :exec -update blockchain_statistics set token_stats = $1 where id = $2 -` - -type UpdateStatisticTokenStatsParams struct { - TokenStats pgtype.JSONB - ID persist.DBID -} - -func (q *Queries) UpdateStatisticTokenStats(ctx context.Context, arg UpdateStatisticTokenStatsParams) error { - _, err := q.db.Exec(ctx, updateStatisticTokenStats, arg.TokenStats, arg.ID) - return err -} - -const updateStatisticTotalLogs = `-- name: UpdateStatisticTotalLogs :exec -update blockchain_statistics set total_logs = $1 where id = $2 -` - -type UpdateStatisticTotalLogsParams struct { - TotalLogs sql.NullInt64 - ID persist.DBID -} - -func (q *Queries) UpdateStatisticTotalLogs(ctx context.Context, arg UpdateStatisticTotalLogsParams) error { - _, err := q.db.Exec(ctx, updateStatisticTotalLogs, arg.TotalLogs, arg.ID) - return err -} - -const updateStatisticTotalTokensAndContracts = `-- name: UpdateStatisticTotalTokensAndContracts :exec -update blockchain_statistics set total_tokens = $1, total_contracts = $2 where id = $3 -` - -type UpdateStatisticTotalTokensAndContractsParams struct { - TotalTokens sql.NullInt64 - TotalContracts sql.NullInt64 - ID persist.DBID -} - -func (q *Queries) UpdateStatisticTotalTokensAndContracts(ctx context.Context, arg UpdateStatisticTotalTokensAndContractsParams) error { - _, err := q.db.Exec(ctx, updateStatisticTotalTokensAndContracts, arg.TotalTokens, arg.TotalContracts, arg.ID) - return err -} - -const updateStatisticTotalTransfers = `-- name: UpdateStatisticTotalTransfers :exec -update blockchain_statistics set total_transfers = $1 where id = $2 -` - -type UpdateStatisticTotalTransfersParams struct { - TotalTransfers sql.NullInt64 - ID persist.DBID -} - -func (q *Queries) UpdateStatisticTotalTransfers(ctx context.Context, arg UpdateStatisticTotalTransfersParams) error { - _, err := q.db.Exec(ctx, updateStatisticTotalTransfers, arg.TotalTransfers, arg.ID) - return err -} diff --git a/db/migrations/indexer/000001_create_core_tables.down.sql b/db/migrations/indexer/000001_create_core_tables.down.sql deleted file mode 100644 index 5b515885a..000000000 --- a/db/migrations/indexer/000001_create_core_tables.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS contracts; -DROP TABLE IF EXISTS tokens; \ No newline at end of file diff --git a/db/migrations/indexer/000001_create_core_tables.up.sql b/db/migrations/indexer/000001_create_core_tables.up.sql deleted file mode 100644 index 3857d24e3..000000000 --- a/db/migrations/indexer/000001_create_core_tables.up.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE TABLE IF NOT EXISTS contracts ( - id character varying(255) PRIMARY KEY, - deleted boolean DEFAULT false NOT NULL, - version integer, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - last_updated timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - name character varying, - symbol character varying, - address character varying(255), - creator_address character varying(255), - chain integer, - latest_block bigint -); - -CREATE TABLE IF NOT EXISTS tokens ( - id character varying(255) PRIMARY KEY, - deleted boolean DEFAULT false NOT NULL, - version integer, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - last_updated timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - name character varying, - description character varying, - contract_address character varying(255), - media jsonb, - owner_address character varying(255), - token_uri character varying, - token_type character varying, - token_id character varying, - quantity character varying, - ownership_history jsonb[], - token_metadata jsonb, - external_url character varying, - block_number bigint, - chain integer, - is_spam boolean -); - -CREATE UNIQUE INDEX IF NOT EXISTS address_idx ON contracts USING btree (address); -CREATE INDEX IF NOT EXISTS block_number_idx ON tokens USING btree (block_number); -CREATE INDEX IF NOT EXISTS contract_address_idx ON tokens USING btree (contract_address); -CREATE UNIQUE INDEX IF NOT EXISTS erc1155_idx ON tokens USING btree (token_id, contract_address, owner_address) WHERE ((token_type)::text = 'ERC-1155'::text); -CREATE UNIQUE INDEX IF NOT EXISTS erc721_idx ON tokens USING btree (token_id, contract_address) WHERE ((token_type)::text = 'ERC-721'::text); -CREATE INDEX IF NOT EXISTS owner_address_idx ON tokens USING btree (owner_address); -CREATE INDEX IF NOT EXISTS token_id_contract_address_idx ON tokens USING btree (token_id, contract_address); \ No newline at end of file diff --git a/db/migrations/indexer/000002_contract_owner.up.sql b/db/migrations/indexer/000002_contract_owner.up.sql deleted file mode 100644 index b5d34bccb..000000000 --- a/db/migrations/indexer/000002_contract_owner.up.sql +++ /dev/null @@ -1 +0,0 @@ -alter table contracts add column owner_address character varying(255); \ No newline at end of file diff --git a/db/migrations/indexer/000003_stats.up.sql b/db/migrations/indexer/000003_stats.up.sql deleted file mode 100644 index ee0d35d55..000000000 --- a/db/migrations/indexer/000003_stats.up.sql +++ /dev/null @@ -1,19 +0,0 @@ -create table if not exists blockchain_statistics ( - id character varying(255) PRIMARY KEY, - deleted boolean default false not null, - version integer default 0 not null, - created_at timestamp with time zone default CURRENT_TIMESTAMP not null, - last_updated timestamp with time zone default CURRENT_TIMESTAMP not null, - block_start bigint not null, - block_end bigint not null, - total_logs bigint, - total_transfers bigint, - total_tokens bigint, - total_contracts bigint, - success boolean default false not null, - contract_stats jsonb, - token_stats jsonb, - processing_time_seconds bigint -); - -create unique index if not exists blockchain_statistics_blocks_idx on blockchain_statistics (block_start, block_end) where deleted = false; diff --git a/db/migrations/indexer/000004_backfill_contract_owner.up.sql b/db/migrations/indexer/000004_backfill_contract_owner.up.sql deleted file mode 100644 index 6f9cb18cc..000000000 --- a/db/migrations/indexer/000004_backfill_contract_owner.up.sql +++ /dev/null @@ -1,7 +0,0 @@ -create table if not exists reprocess_jobs ( - id int primary key, - start_id varchar(255) not null, - end_id varchar(255) not null -); - -alter table contracts add column if not exists owner_method int; \ No newline at end of file diff --git a/db/queries/indexer/query.sql b/db/queries/indexer/query.sql deleted file mode 100644 index cf1cfce1e..000000000 --- a/db/queries/indexer/query.sql +++ /dev/null @@ -1,39 +0,0 @@ --- name: FirstContract :one --- sqlc needs at least one query in order to generate the models. -SELECT * FROM contracts LIMIT 1; - --- name: InsertStatistic :one -insert into blockchain_statistics (id, block_start, block_end) values ($1, $2, $3) on conflict do nothing returning id; - --- name: UpdateStatisticTotalLogs :exec -update blockchain_statistics set total_logs = $1 where id = $2; - --- name: UpdateStatisticTotalTransfers :exec -update blockchain_statistics set total_transfers = $1 where id = $2; - --- name: UpdateStatisticTotalTokensAndContracts :exec -update blockchain_statistics set total_tokens = $1, total_contracts = $2 where id = $3; - --- name: UpdateStatisticSuccess :exec -update blockchain_statistics set success = $1, processing_time_seconds = $2 where id = $3; - --- name: UpdateStatisticContractStats :exec -update blockchain_statistics set contract_stats = $1 where id = $2; - --- name: UpdateStatisticTokenStats :exec -update blockchain_statistics set token_stats = $1 where id = $2; - --- name: GetContractsByIDRange :many -SELECT - contracts.* -FROM contracts -WHERE contracts.deleted = false -AND (contracts.owner_address IS NULL OR contracts.owner_address = '' OR contracts.creator_address IS NULL OR contracts.creator_address = '') -AND contracts.id >= @start_id::text AND contracts.id < @end_id::text -ORDER BY contracts.id; - --- name: UpdateContractOwnerByID :exec -update contracts set owner_address = @owner_address, creator_address = @creator_address, owner_method = @owner_method where id = @id and deleted = false; - --- name: GetReprocessJobRangeByID :one -select * from reprocess_jobs where id = $1; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 9c4ac28d8..e570bbb0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,23 +24,6 @@ services: volumes: - '${PWD}/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf' command: ['-c', 'config-file=/etc/postgresql/postgresql.conf'] - postgres_indexer: - build: - context: '.' - dockerfile: 'docker/postgres_indexer/DOCKERFILE' - args: - - PGHOST=${PGHOST} - - PGPORT=${PGPORT} - - PGUSER=${PGUSER} - - PGPASSWORD=${PGPASSWORD} - - PGDATABASE=${PGDATABASE} - - PGTESTUSER=${PGTESTUSER} - ports: - - '5433:5432' - environment: - - POSTGRES_HOST_AUTH_METHOD=trust - - POSTGRES_USER=postgres - - POSTGRES_DB=postgres task-emulator: image: ghcr.io/aertje/cloud-tasks-emulator:latest ports: @@ -60,8 +43,6 @@ services: '-queue', 'projects/gallery-local/locations/here/queues/token-processing', '-queue', - 'projects/gallery-local/locations/here/queues/indexer-refreshes', - '-queue', 'projects/gallery-local/locations/here/queues/wallet-validate', '-queue', 'projects/gallery-local/locations/here/queues/push-notifications', @@ -102,27 +83,3 @@ services: # depends_on: # - postgres # - redis - # Uncomment if you want to run indexer-api locally as a container - # indexer-api: - # build: - # context: "." - # dockerfile: "docker/indexer_api/Dockerfile" - # ports: - # - "6000:6000" - # volumes: - # - ${PWD}/_local:/app/_local - # - ${PWD}/_deploy:/app/_deploy - # depends_on: - # - postgres - # - postgres_indexer - # - redis - # Uncomment if you want to run pushnotifications locally as a container - # pushnotifications: - # build: - # context: "." - # dockerfile: "docker/pushnotifications/Dockerfile" - # ports: - # - "6600:6600" - # depends_on: - # - postgres - # - redis diff --git a/docker/cloud_sql_proxy/docker-compose.yml b/docker/cloud_sql_proxy/docker-compose.yml index c6b144f7a..cea58f8c1 100644 --- a/docker/cloud_sql_proxy/docker-compose.yml +++ b/docker/cloud_sql_proxy/docker-compose.yml @@ -3,7 +3,6 @@ # Prod DB: 127.0.0.1:6543 # Prod DB Replica: 127.0.0.1:6544 -# Prod Indexer DB: 127.0.0.1:6545 # Dev DB: 127.0.0.1:6643 version: "3.9" @@ -19,7 +18,6 @@ services: command: > --address=0.0.0.0 --credentials-file=/config '${SQL_CONNECTION_NAME_PROD_DB}?port=6543' - '${SQL_CONNECTION_NAME_PROD_ETH_INDEXER}?port=6545' cloud-sql-proxy-dev: image: gcr.io/cloud-sql-connectors/cloud-sql-proxy diff --git a/docker/docker.go b/docker/docker.go index 0fcba2300..392dbd19c 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -81,28 +81,6 @@ func StartPostgres() (resource *dockertest.Resource, err error) { return r, nil } -func StartPostgresIndexer() (resource *dockertest.Resource, err error) { - pool, err := newPool(time.Minute * 3) - if err != nil { - return nil, err - } - - r, err := startService(pool, "postgres_indexer") - if err != nil { - return nil, err - } - - hostAndPort := strings.Split(r.GetHostPort("5432/tcp"), ":") - host := hostAndPort[0] - port := hostAndPort[1] - - if err = pool.Retry(waitOnDB(host, port, "postgres", "", "postgres")); err != nil { - return nil, err - } - - return r, nil -} - func StartRedis() (*dockertest.Resource, error) { pool, err := newPool(time.Minute * 3) if err != nil { diff --git a/docker/indexer/Dockerfile b/docker/indexer/Dockerfile deleted file mode 100644 index 6b5846077..000000000 --- a/docker/indexer/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM golang:1.19-bullseye - -ARG VERSION -ARG FFMPEG_VERSION - -RUN apt-get update -RUN apt-get install -y ffmpeg=$FFMPEG_VERSION && rm -rf /var/lib/apt/lists/* - -# Install deps -WORKDIR /app -COPY go.mod go.sum /app/ -RUN go mod download - -# Install certs -COPY root-certs/sectigo.crt /usr/local/share/ca-certificates/sectigo.crt -RUN update-ca-certificates - -COPY . /app -RUN go build -o ./bin/indexer ./cmd/indexer/main.go - -ENV VERSION=$VERSION - -EXPOSE 4000 -USER nobody -ENTRYPOINT ["./bin/indexer"] diff --git a/docker/indexer_api/Dockerfile b/docker/indexer_api/Dockerfile deleted file mode 100644 index e95908977..000000000 --- a/docker/indexer_api/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM golang:1.19-bullseye - -ARG VERSION -ARG FFMPEG_VERSION - -RUN apt-get update -RUN apt-get install -y ffmpeg=$FFMPEG_VERSION && rm -rf /var/lib/apt/lists/* - -# Install deps -WORKDIR /app -COPY go.mod go.sum /app/ -RUN go mod download - -# Install certs -COPY root-certs/sectigo.crt /usr/local/share/ca-certificates/sectigo.crt -RUN update-ca-certificates - -COPY . /app -RUN go build -o ./bin/indexer ./cmd/indexer/main.go - -ENV VERSION=$VERSION - -EXPOSE 6000 -USER nobody -ENTRYPOINT ["./bin/indexer", "server", "--enable-rpc"] diff --git a/docker/postgres_indexer/01_init.sql b/docker/postgres_indexer/01_init.sql deleted file mode 100644 index 5fe768af9..000000000 --- a/docker/postgres_indexer/01_init.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE ROLE cloudsqlsuperuser WITH LOGIN CREATEDB SUPERUSER; - -CREATE ROLE cloudsqladmin WITH LOGIN CREATEDB; - -DROP SCHEMA public CASCADE; \ No newline at end of file diff --git a/docker/postgres_indexer/02_init.sql b/docker/postgres_indexer/02_init.sql deleted file mode 100644 index f5d431b9f..000000000 --- a/docker/postgres_indexer/02_init.sql +++ /dev/null @@ -1,52 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS public; - -CREATE TABLE IF NOT EXISTS tokens ( - ID varchar(255) PRIMARY KEY, - DELETED boolean NOT NULL DEFAULT false, - VERSION int, - CREATED_AT timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, - LAST_UPDATED timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, - NAME varchar, - DESCRIPTION varchar, - CONTRACT_ADDRESS varchar(255), - MEDIA jsonb, - CHAIN int, - OWNER_ADDRESS varchar(255), - TOKEN_URI varchar, - TOKEN_TYPE varchar, - TOKEN_ID varchar, - QUANTITY varchar, - OWNERSHIP_HISTORY jsonb [], - TOKEN_METADATA jsonb, - EXTERNAL_URL varchar, - BLOCK_NUMBER bigint, - IS_SPAM boolean -); - -CREATE UNIQUE INDEX IF NOT EXISTS erc1155_idx ON tokens (TOKEN_ID, CONTRACT_ADDRESS, OWNER_ADDRESS) WHERE TOKEN_TYPE = 'ERC-1155'; - -CREATE UNIQUE INDEX IF NOT EXISTS erc721_idx ON tokens (TOKEN_ID, CONTRACT_ADDRESS) WHERE TOKEN_TYPE = 'ERC-721'; - -CREATE INDEX IF NOT EXISTS token_id_contract_address_idx ON tokens (TOKEN_ID, CONTRACT_ADDRESS); - -CREATE INDEX IF NOT EXISTS owner_address_idx ON tokens (OWNER_ADDRESS); - -CREATE INDEX IF NOT EXISTS contract_address_idx ON tokens (CONTRACT_ADDRESS); - -CREATE INDEX IF NOT EXISTS block_number_idx ON tokens (BLOCK_NUMBER); - -CREATE TABLE IF NOT EXISTS contracts ( - ID varchar(255) PRIMARY KEY, - DELETED boolean NOT NULL DEFAULT false, - VERSION int, - CREATED_AT timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, - LAST_UPDATED timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, - CHAIN int, - NAME varchar, - SYMBOL varchar, - ADDRESS varchar(255), - OWNER_ADDRESS varchar(255), - LATEST_BLOCK bigint -); - -CREATE UNIQUE INDEX IF NOT EXISTS address_idx ON contracts (ADDRESS); diff --git a/docker/postgres_indexer/DOCKERFILE b/docker/postgres_indexer/DOCKERFILE deleted file mode 100644 index f2a30fe45..000000000 --- a/docker/postgres_indexer/DOCKERFILE +++ /dev/null @@ -1,14 +0,0 @@ -FROM postgres:14 - -ENV POSTGRES_HOST_AUTH_METHOD trust -ENV POSTGRES_USER postgres -ENV POSTGRES_DB postgres -ARG PGHOST="localhost" -ARG PGPORT=5432 -ARG PGUSER=postgres -ARG PGPASSWORD="" -ARG PGDATABASE=postgres -ARG PGTESTUSER=benny -ARG PGDATABASE=postgres -COPY docker/postgres_indexer/01_init.sql /docker-entrypoint-initdb.d/01_init.sql -COPY docker/postgres_indexer/02_init.sql /docker-entrypoint-initdb.d/02_init.sql \ No newline at end of file diff --git a/indexer/cmd/root.go b/indexer/cmd/root.go deleted file mode 100644 index 9661cfa06..000000000 --- a/indexer/cmd/root.go +++ /dev/null @@ -1,106 +0,0 @@ -package cmd - -import ( - "fmt" - "net/http" - - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/indexer" - "github.com/mikeydub/go-gallery/service/logger" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" - "github.com/sirupsen/logrus" - "google.golang.org/appengine" - - "github.com/spf13/cobra" -) - -var ( - port uint64 - fromBlock uint64 - toBlock uint64 - enableRPC bool - quietLogs bool - manualEnv string -) - -func init() { - cobra.OnInitialize(indexer.SetDefaults) - - rootCmd.PersistentFlags().BoolVarP(&enableRPC, "enable-rpc", "r", false, "always enable RPC calls") - rootCmd.PersistentFlags().BoolVarP(&quietLogs, "quiet", "q", false, "hide debug logs") - rootCmd.Flags().Uint64VarP(&fromBlock, "from-block", "f", 0, "first block to process") - rootCmd.Flags().Uint64VarP(&toBlock, "to-block", "t", 0, "last block to process") - rootCmd.MarkFlagsRequiredTogether("from-block", "to-block") - rootCmd.PersistentFlags().StringVarP(&manualEnv, "env", "e", "local", "env to run with") - - rootCmd.AddCommand(serverCmd) - serverCmd.Flags().Uint64VarP(&port, "port", "p", 6000, "port to serve on") -} - -var rootCmd = &cobra.Command{ - Use: "indexer", - Short: "Retrieve all NFTs that have ever been minted", - Long: `An NFT indexer lovingly built by your friends at Gallery. - Source code is available at https://github.com/gallery-so/go-gallery.`, - Args: func(cmd *cobra.Command, args []string) error { - indexer.LoadConfigFile("indexer", manualEnv) - indexer.ValidateEnv() - - if toBlock < fromBlock { - return fmt.Errorf("[from-block] must be less than [to-block]") - } - - if !cmd.Flags().Lookup("to-block").Changed && (!enableRPC && env.GetString("ENV") != "production") { - return fmt.Errorf("`flags in group [from-block, to-block] must all be set when [enable-rpc] is not set") - } - - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - defer sentryutil.RecoverAndRaise(nil) - - var from *uint64 - if cmd.Flags().Lookup("from-block").Changed { - from = &fromBlock - } - - var to *uint64 - if cmd.Flags().Lookup("to-block").Changed { - to = &toBlock - } - - indexer.Init(from, to, quietLogs, enableRPC) - fmt.Println("Running in Default Mode on port 4000") - http.ListenAndServe(":4000", nil) - - }, -} - -var serverCmd = &cobra.Command{ - Use: "server", - Short: "Run the indexer server", - Args: func(cmd *cobra.Command, args []string) error { - indexer.LoadConfigFile("indexer-server", manualEnv) - indexer.ValidateEnv() - - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - defer sentryutil.RecoverAndRaise(nil) - - indexer.InitServer(quietLogs, enableRPC) - - logger.For(nil).WithFields(logrus.Fields{"port": port}).Info("Starting indexer server") - if appengine.IsAppEngine() { - logger.For(nil).Info("Running in App Engine Mode") - appengine.Main() - } else { - logger.For(nil).Info("Running in Default Mode") - http.ListenAndServe(fmt.Sprintf(":%d", port), nil) - } - }, -} - -func Execute() { - rootCmd.Execute() -} diff --git a/indexer/contracts.go b/indexer/contracts.go deleted file mode 100644 index 7566597a8..000000000 --- a/indexer/contracts.go +++ /dev/null @@ -1,176 +0,0 @@ -package indexer - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/ethereum/go-ethereum/ethclient" - "github.com/gin-gonic/gin" - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/multichain/alchemy" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/rpc" - "github.com/mikeydub/go-gallery/util" -) - -func init() { - env.RegisterValidation("ALCHEMY_API_URL", "required") -} - -// GetContractOutput is the response for getting a single smart contract -type GetContractOutput struct { - Contract persist.Contract `json:"contract"` -} - -// GetContractsOutput is the response for getting multiple smart contracts -type GetContractsOutput struct { - Contracts []persist.Contract `json:"contracts"` -} - -// GetContractInput is the input to the Get Contract endpoint -type GetContractInput struct { - Address persist.EthereumAddress `form:"address"` - Owner persist.EthereumAddress `form:"owner"` -} - -// UpdateContractMetadataInput is used to refresh metadata for a given contract -type UpdateContractMetadataInput struct { - Address persist.EthereumAddress `json:"address" binding:"required"` -} - -func getContract(contractsRepo persist.ContractRepository) gin.HandlerFunc { - return func(c *gin.Context) { - var input GetContractInput - if err := c.ShouldBindQuery(&input); err != nil { - util.ErrResponse(c, http.StatusBadRequest, err) - return - } - - if input.Address != "" { - contract, err := contractsRepo.GetByAddress(c, input.Address) - if err != nil { - util.ErrResponse(c, http.StatusInternalServerError, err) - return - } - c.JSON(http.StatusOK, GetContractOutput{Contract: contract}) - return - } else if input.Owner != "" { - contracts, err := contractsRepo.GetContractsOwnedByAddress(c, input.Owner) - if err != nil { - util.ErrResponse(c, http.StatusInternalServerError, err) - return - } - - c.JSON(http.StatusOK, GetContractsOutput{Contracts: contracts}) - return - } - - err := util.ErrInvalidInput{Reason: "must specify 'address' or 'owner' field"} - util.ErrResponse(c, http.StatusBadRequest, err) - } -} - -func updateContractMetadata(contractsRepo persist.ContractRepository, ethClient *ethclient.Client, httpClient *http.Client) gin.HandlerFunc { - return func(c *gin.Context) { - var input UpdateContractMetadataInput - if err := c.ShouldBindJSON(&input); err != nil { - err = util.ErrInvalidInput{Reason: fmt.Sprintf("must specify 'address' field: %v", err)} - util.ErrResponse(c, http.StatusBadRequest, err) - return - } - - err := updateMetadataForContract(c, input, ethClient, httpClient, contractsRepo) - if err != nil { - util.ErrResponse(c, http.StatusInternalServerError, err) - return - } - - c.JSON(http.StatusOK, util.SuccessResponse{Success: true}) - } -} - -func updateMetadataForContract(c context.Context, input UpdateContractMetadataInput, ethClient *ethclient.Client, httpClient *http.Client, contractsRepo persist.ContractRepository) error { - newMetadata, err := rpc.GetTokenContractMetadata(c, input.Address, ethClient) - if err != nil || newMetadata.Name == "" { - alchMetadata, err := getAlchemyContractMetadata(c, httpClient, input.Address) - if err != nil { - return err - } - if newMetadata == nil { - newMetadata = new(rpc.TokenContractMetadata) - } - newMetadata.Name = util.FirstNonEmptyString(alchMetadata.Name, alchMetadata.OpenseaCollection.CollectionName) - newMetadata.Symbol = util.FirstNonEmptyString(alchMetadata.Symbol, newMetadata.Symbol) - } - - latestBlock, err := ethClient.BlockNumber(c) - if err != nil { - return err - } - - up := persist.ContractUpdateInput{ - Name: persist.NullString(newMetadata.Name), - Symbol: persist.NullString(newMetadata.Symbol), - LatestBlock: persist.BlockNumber(latestBlock), - } - - timedContext, cancel := context.WithTimeout(c, time.Second*10) - defer cancel() - - owner, _, err := GetContractOwner(timedContext, input.Address, ethClient, httpClient) - if err != nil { - logger.For(c).WithError(err).Errorf("error finding creator address") - } else { - up.OwnerAddress = owner - } - - // TODO creator address - - return contractsRepo.UpdateByAddress(c, input.Address, up) -} - -func getAlchemyContractMetadata(ctx context.Context, httpClient *http.Client, addr persist.EthereumAddress) (alchemy.ContractMetadata, error) { - - u := fmt.Sprintf("%s/getContractMetadata", env.GetString("ALCHEMY_API_URL")) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return alchemy.ContractMetadata{}, err - } - - q := req.URL.Query() - q.Add("contractAddress", string(addr)) - req.URL.RawQuery = q.Encode() - - resp, err := httpClient.Do(req) - if err != nil { - return alchemy.ContractMetadata{}, err - } - - var metadata alchemy.GetContractMetadataResponse - err = json.NewDecoder(resp.Body).Decode(&metadata) - if err != nil { - return alchemy.ContractMetadata{}, err - } - - return metadata.ContractMetadata, nil -} - -func GetContractOwner(ctx context.Context, address persist.EthereumAddress, ethClient *ethclient.Client, httpClient *http.Client) (persist.EthereumAddress, persist.ContractOwnerMethod, error) { - - owner, err := rpc.GetContractOwner(ctx, address, ethClient) - if err == nil { - return owner, persist.ContractOwnerMethodOwnable, nil - } - logger.For(ctx).WithError(err).Error("error finding owner address through ownable interface") - - creator, err := rpc.GetContractCreator(ctx, address, ethClient) - if err != nil { - logger.For(ctx).WithError(err).Error("error finding creator address with binary search") - return "", persist.ContractOwnerMethodFailed, err - } - return creator, persist.ContractOwnerBinarySearch, nil -} diff --git a/indexer/core.go b/indexer/core.go deleted file mode 100644 index 61c0eb38e..000000000 --- a/indexer/core.go +++ /dev/null @@ -1,202 +0,0 @@ -package indexer - -import ( - "context" - "net/http" - "time" - - "github.com/getsentry/sentry-go" - "github.com/gin-gonic/gin" - "github.com/mikeydub/go-gallery/db/gen/coredb" - "github.com/mikeydub/go-gallery/db/gen/indexerdb" - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/middleware" - "github.com/mikeydub/go-gallery/service/auth" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/persist/postgres" - "github.com/mikeydub/go-gallery/service/rpc" - "github.com/mikeydub/go-gallery/service/rpc/arweave" - "github.com/mikeydub/go-gallery/service/rpc/ipfs" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" - "github.com/mikeydub/go-gallery/service/task" - "github.com/mikeydub/go-gallery/service/tracing" - "github.com/mikeydub/go-gallery/util" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" -) - -// Init initializes the indexer -func Init(fromBlock, toBlock *uint64, quietLogs, enableRPC bool) { - router, i := coreInit(fromBlock, toBlock, quietLogs, enableRPC) - logger.For(nil).Info("Starting indexer...") - go i.Start(sentry.SetHubOnContext(context.Background(), sentry.CurrentHub())) - http.Handle("/", router) -} - -// InitServer initializes the indexer server -func InitServer(quietLogs, enableRPC bool) { - router := coreInitServer(quietLogs, enableRPC) - logger.For(nil).Info("Starting indexer server...") - http.Handle("/", router) -} - -func coreInit(fromBlock, toBlock *uint64, quietLogs, enableRPC bool) (*gin.Engine, *indexer) { - InitSentry() - logger.InitWithGCPDefaults() - logger.SetLoggerOptions(func(logger *logrus.Logger) { - logger.AddHook(sentryutil.SentryLoggerHook) - logger.SetLevel(logrus.InfoLevel) - if env.GetString("ENV") != "production" && !quietLogs { - logger.SetLevel(logrus.DebugLevel) - } - }) - - s := rpc.NewStorageClient(context.Background()) - contractRepo := postgres.NewContractRepository(postgres.MustCreateClient()) - iQueries := indexerdb.New(postgres.NewPgxClient()) - bQueries := coredb.New(postgres.NewPgxClient(postgres.WithHost(env.GetString("BACKEND_POSTGRES_HOST")), postgres.WithUser(env.GetString("BACKEND_POSTGRES_USER")), postgres.WithPassword(env.GetString("BACKEND_POSTGRES_PASSWORD")), postgres.WithPort(env.GetInt("BACKEND_POSTGRES_PORT")))) - tClient := task.NewClient(context.Background()) - ethClient := rpc.NewEthSocketClient() - ipfsClient := ipfs.NewShell() - arweaveClient := arweave.NewClient() - - if env.GetString("ENV") == "production" || enableRPC { - rpcEnabled = true - } - - i := newIndexer(ethClient, &http.Client{Timeout: 10 * time.Minute}, ipfsClient, arweaveClient, s, iQueries, bQueries, tClient, contractRepo, persist.Chain(env.GetInt("CHAIN")), defaultTransferEvents, nil, fromBlock, toBlock) - - router := gin.Default() - - router.Use(middleware.GinContextToContext(), middleware.Sentry(true), middleware.Tracing(), middleware.HandleCORS(), middleware.ErrLogger()) - - if env.GetString("ENV") != "production" { - gin.SetMode(gin.DebugMode) - } - - logger.For(nil).Info("Registering handlers...") - return handlersInit(router, i, contractRepo, ethClient, ipfsClient, arweaveClient, s), i -} - -func coreInitServer(quietLogs, enableRPC bool) *gin.Engine { - ctx := sentry.SetHubOnContext(context.Background(), sentry.CurrentHub()) - InitSentry() - logger.InitWithGCPDefaults() - logger.SetLoggerOptions(func(logger *logrus.Logger) { - logger.SetLevel(logrus.InfoLevel) - if env.GetString("ENV") != "production" && !quietLogs { - logger.SetLevel(logrus.DebugLevel) - } - }) - - s := rpc.NewStorageClient(context.Background()) - contractRepo := postgres.NewContractRepository(postgres.MustCreateClient()) - iQueries := indexerdb.New(postgres.NewPgxClient()) - bQueries := coredb.New(postgres.NewPgxClient(postgres.WithHost(env.GetString("BACKEND_POSTGRES_HOST")), postgres.WithUser(env.GetString("BACKEND_POSTGRES_USER")), postgres.WithPassword(env.GetString("BACKEND_POSTGRES_PASSWORD")), postgres.WithPort(env.GetInt("BACKEND_POSTGRES_PORT")))) - tClient := task.NewClient(context.Background()) - ethClient := rpc.NewEthSocketClient() - ipfsClient := ipfs.NewShell() - arweaveClient := arweave.NewClient() - - if env.GetString("ENV") == "production" || enableRPC { - rpcEnabled = true - } - - router := gin.Default() - - router.Use(middleware.GinContextToContext(), middleware.Sentry(true), middleware.Tracing(), middleware.HandleCORS(), middleware.ErrLogger()) - - if env.GetString("ENV") != "production" { - gin.SetMode(gin.DebugMode) - logrus.SetLevel(logrus.DebugLevel) - } - - logger.For(ctx).Info("Registering handlers...") - - httpClient := &http.Client{Timeout: 10 * time.Minute, Transport: tracing.NewTracingTransport(http.DefaultTransport, false)} - - i := newIndexer(ethClient, httpClient, ipfsClient, arweaveClient, s, iQueries, bQueries, tClient, contractRepo, persist.Chain(env.GetInt("CHAIN")), defaultTransferEvents, nil, nil, nil) - return handlersInitServer(router, contractRepo, ethClient, httpClient, ipfsClient, arweaveClient, s, i) -} - -func SetDefaults() { - viper.SetDefault("RPC_URL", "") - viper.SetDefault("IPFS_URL", "https://gallery.infura-ipfs.io") - viper.SetDefault("IPFS_API_URL", "https://ipfs.infura.io:5001") - viper.SetDefault("FALLBACK_IPFS_URL", "https://ipfs.io") - viper.SetDefault("IPFS_PROJECT_ID", "") - viper.SetDefault("IPFS_PROJECT_SECRET", "") - viper.SetDefault("CHAIN", 0) - viper.SetDefault("ENV", "local") - viper.SetDefault("GCLOUD_TOKEN_LOGS_BUCKET", "dev-eth-token-logs") - viper.SetDefault("GCLOUD_TOKEN_CONTENT_BUCKET", "dev-token-content") - viper.SetDefault("POSTGRES_HOST", "0.0.0.0") - viper.SetDefault("POSTGRES_PORT", 5433) - viper.SetDefault("POSTGRES_USER", "postgres") - viper.SetDefault("POSTGRES_PASSWORD", "") - viper.SetDefault("POSTGRES_DB", "postgres") - viper.SetDefault("BACKEND_POSTGRES_HOST", "0.0.0.0") - viper.SetDefault("BACKEND_POSTGRES_PORT", 5432) - viper.SetDefault("BACKEND_POSTGRES_USER", "postgres") - viper.SetDefault("BACKEND_POSTGRES_PASSWORD", "") - viper.SetDefault("BACKEND_POSTGRES_DB", "postgres") - viper.SetDefault("ALLOWED_ORIGINS", "http://localhost:3000") - viper.SetDefault("REDIS_URL", "localhost:6379") - viper.SetDefault("SENTRY_DSN", "") - viper.SetDefault("IMGIX_API_KEY", "") - viper.SetDefault("VERSION", "") - viper.SetDefault("ALCHEMY_API_URL", "") - viper.SetDefault("ALCHEMY_NFT_API_URL", "") - viper.SetDefault("TASK_QUEUE_HOST", "localhost:8123") - viper.SetDefault("TOKEN_PROCESSING_QUEUE", "projects/gallery-local/locations/here/queues/token-processing") - viper.SetDefault("TOKEN_PROCESSING_URL", "http://localhost:6500") - viper.AutomaticEnv() -} - -func LoadConfigFile(service string, manualEnv string) { - if env.GetString("ENV") != "local" { - logger.For(nil).Info("running in non-local environment, skipping environment configuration") - return - } - util.LoadEncryptedEnvFile(util.ResolveEnvFile(service, manualEnv)) -} - -func ValidateEnv() { - util.VarNotSetTo("RPC_URL", "") - if env.GetString("ENV") != "local" { - util.VarNotSetTo("SENTRY_DSN", "") - } -} - -func InitSentry() { - if env.GetString("ENV") == "local" { - logger.For(nil).Info("skipping sentry init") - return - } - - logger.For(nil).Info("initializing sentry...") - - err := sentry.Init(sentry.ClientOptions{ - Dsn: env.GetString("SENTRY_DSN"), - Environment: env.GetString("ENV"), - TracesSampler: sentry.TracesSamplerFunc(func(ctx sentry.SamplingContext) sentry.Sampled { - if ctx.Span.Op == rpc.GethSocketOpName { - return sentry.UniformTracesSampler(0.01).Sample(ctx) - } - return sentry.UniformTracesSampler(env.GetFloat64("SENTRY_TRACES_SAMPLE_RATE")).Sample(ctx) - }), - Release: env.GetString("VERSION"), - AttachStacktrace: true, - BeforeSend: func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { - event = auth.ScrubEventCookies(event, hint) - event = sentryutil.UpdateErrorFingerprints(event, hint) - event = sentryutil.UpdateLogErrorEvent(event, hint) - return event - }, - }) - - if err != nil { - logger.For(nil).Fatalf("failed to start sentry: %s", err) - } -} diff --git a/indexer/handler.go b/indexer/handler.go deleted file mode 100644 index 595675c69..000000000 --- a/indexer/handler.go +++ /dev/null @@ -1,26 +0,0 @@ -package indexer - -import ( - "net/http" - - "cloud.google.com/go/storage" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/everFinance/goar" - "github.com/gin-gonic/gin" - shell "github.com/ipfs/go-ipfs-api" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/util" -) - -func handlersInit(router *gin.Engine, i *indexer, contractRepository persist.ContractRepository, ethClient *ethclient.Client, ipfsClient *shell.Shell, arweaveClient *goar.Client, storageClient *storage.Client) *gin.Engine { - router.GET("/status", getStatus(i, contractRepository)) - router.GET("/alive", util.HealthCheckHandler()) - return router -} - -func handlersInitServer(router *gin.Engine, contractRepository persist.ContractRepository, ethClient *ethclient.Client, httpClient *http.Client, ipfsClient *shell.Shell, arweaveClient *goar.Client, storageClient *storage.Client, idxer *indexer) *gin.Engine { - contractsGroup := router.Group("/contracts") - contractsGroup.GET("/get", getContract(contractRepository)) - contractsGroup.POST("/refresh", updateContractMetadata(contractRepository, ethClient, httpClient)) - return router -} diff --git a/indexer/hook.go b/indexer/hook.go deleted file mode 100644 index 9f858399d..000000000 --- a/indexer/hook.go +++ /dev/null @@ -1,90 +0,0 @@ -package indexer - -import ( - "context" - "net/http" - "sync" - - "github.com/ethereum/go-ethereum/ethclient" - "github.com/mikeydub/go-gallery/db/gen/coredb" - "github.com/mikeydub/go-gallery/db/gen/indexerdb" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/task" - "github.com/mikeydub/go-gallery/util" - "github.com/sourcegraph/conc/pool" -) - -type DBHook[T any] func(ctx context.Context, it []T, statsID persist.DBID) error - -func newContractHooks(queries *indexerdb.Queries, repo persist.ContractRepository, ethClient *ethclient.Client, httpClient *http.Client, ownerStats *sync.Map) []DBHook[persist.Contract] { - return []DBHook[persist.Contract]{ - func(ctx context.Context, it []persist.Contract, statsID persist.DBID) error { - upChan := make(chan []persist.Contract) - go fillContractFields(ctx, it, queries, repo, httpClient, ethClient, ownerStats, upChan, statsID) - p := pool.New().WithErrors().WithContext(ctx).WithMaxGoroutines(10) - for up := range upChan { - up := up - p.Go(func(ctx context.Context) error { - logger.For(ctx).Info("bulk upserting contracts") - if err := repo.BulkUpsert(ctx, up); err != nil { - return err - } - return nil - }) - } - return p.Wait() - }, - } -} - -func newTokenHooks(tasks *task.Client, bQueries *coredb.Queries) []DBHook[persist.Token] { - return []DBHook[persist.Token]{ - func(ctx context.Context, it []persist.Token, statsID persist.DBID) error { - - wallets, _ := util.Map(it, func(t persist.Token) (string, error) { - return t.OwnerAddress.String(), nil - }) - chains, _ := util.Map(it, func(t persist.Token) (int32, error) { - return int32(t.Chain), nil - }) - - // get all gallery users associated with any of the tokens - users, err := bQueries.GetUsersByWalletAddressesAndChains(ctx, coredb.GetUsersByWalletAddressesAndChainsParams{ - WalletAddresses: wallets, - Chains: chains, - }) - if err != nil { - return err - } - - // map the chain address to the user id - addressToUser := make(map[persist.ChainAddress]persist.DBID) - for _, u := range users { - addressToUser[persist.NewChainAddress(u.Wallet.Address, u.Wallet.Chain)] = u.User.ID - } - - tokensForUser := make(map[persist.DBID]map[persist.TokenUniqueIdentifiers]persist.HexString) - for _, t := range it { - ca := persist.NewChainAddress(persist.Address(t.OwnerAddress.String()), t.Chain) - // check if the token corresponds to a user - if u, ok := addressToUser[ca]; ok { - if _, ok := tokensForUser[u]; !ok { - tokensForUser[u] = make(map[persist.TokenUniqueIdentifiers]persist.HexString) - } - cur := tokensForUser[u] - cur[persist.TokenUniqueIdentifiers{ - Chain: t.Chain, - ContractAddress: persist.Address(t.ContractAddress.String()), - TokenID: t.TokenID, - OwnerAddress: persist.Address(t.OwnerAddress.String()), - }] = t.Quantity - - tokensForUser[u] = cur - } - } - - return nil - }, - } -} diff --git a/indexer/indexer.go b/indexer/indexer.go deleted file mode 100644 index 5c6f9af49..000000000 --- a/indexer/indexer.go +++ /dev/null @@ -1,1061 +0,0 @@ -package indexer - -import ( - "bytes" - "context" - "database/sql" - "encoding/json" - "fmt" - "github.com/mikeydub/go-gallery/service/task" - "io" - "math/big" - "net/http" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "cloud.google.com/go/storage" - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" - gethrpc "github.com/ethereum/go-ethereum/rpc" - "github.com/everFinance/goar" - "github.com/gammazero/workerpool" - "github.com/getsentry/sentry-go" - shell "github.com/ipfs/go-ipfs-api" - "github.com/jackc/pgtype" - "github.com/mikeydub/go-gallery/contracts" - "github.com/mikeydub/go-gallery/db/gen/coredb" - "github.com/mikeydub/go-gallery/db/gen/indexerdb" - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/multichain/alchemy" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/rpc" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" - "github.com/mikeydub/go-gallery/service/tracing" - "github.com/mikeydub/go-gallery/util" - "github.com/sirupsen/logrus" - "github.com/sourcegraph/conc/iter" -) - -func init() { - env.RegisterValidation("GCLOUD_TOKEN_CONTENT_BUCKET", "required") - env.RegisterValidation("ALCHEMY_API_URL", "required") -} - -const ( - // transferEventHash represents the keccak256 hash of Transfer(address,address,uint256) - transferEventHash eventHash = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" - // transferSingleEventHash represents the keccak256 hash of TransferSingle(address,address,address,uint256,uint256) - transferSingleEventHash eventHash = "0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62" - // transferBatchEventHash represents the keccak256 hash of TransferBatch(address,address,address,uint256[],uint256[]) - transferBatchEventHash eventHash = "0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb" - - defaultWorkerPoolSize = 3 - defaultWorkerPoolWaitSize = 10 - blocksPerLogsCall = 50 -) - -var ( - rpcEnabled bool = false // Enables external RPC calls - erc1155ABI, _ = contracts.IERC1155MetaData.GetAbi() - defaultTransferEvents = []eventHash{ - transferBatchEventHash, - transferEventHash, - transferSingleEventHash, - } -) - -// eventHash represents an event keccak256 hash -type eventHash string - -type transfersAtBlock struct { - block persist.BlockNumber - transfers []rpc.Transfer -} - -type contractAtBlock struct { - ti persist.EthereumTokenIdentifiers - boi blockchainOrderInfo - contract persist.Contract -} - -func (o contractAtBlock) TokenIdentifiers() persist.EthereumTokenIdentifiers { - return o.ti -} - -func (o contractAtBlock) OrderInfo() blockchainOrderInfo { - return o.boi -} - -type tokenAtBlock struct { - ti persist.EthereumTokenIdentifiers - boi blockchainOrderInfo - token persist.Token -} - -func (o tokenAtBlock) TokenIdentifiers() persist.EthereumTokenIdentifiers { - return o.ti -} - -func (o tokenAtBlock) OrderInfo() blockchainOrderInfo { - return o.boi -} - -type getLogsFunc func(ctx context.Context, curBlock, nextBlock *big.Int, topics [][]common.Hash) ([]types.Log, error) - -// indexer is the indexer for the blockchain that uses JSON RPC to scan through logs and process them -// into a format used by the application -type indexer struct { - ethClient *ethclient.Client - httpClient *http.Client - ipfsClient *shell.Shell - arweaveClient *goar.Client - storageClient *storage.Client - queries *indexerdb.Queries - contractRepo persist.ContractRepository - dbMu *sync.Mutex // Manages writes to the db - stateMu *sync.Mutex // Manages updates to the indexer's state - memoryMu *sync.Mutex // Manages large memory operations - - tokenBucket string - - chain persist.Chain - - eventHashes []eventHash - - mostRecentBlock uint64 // Current height of the blockchain - lastSyncedChunk uint64 // Start block of the last chunk handled by the indexer - maxBlock *uint64 // If provided, the indexer will only index up to maxBlock - - contractOwnerStats *sync.Map // map[contractOwnerMethod]int - Used to track the number of times a contract owner method is used - - isListening bool // Indicates if the indexer is waiting for new blocks - - getLogsFunc getLogsFunc - - contractDBHooks []DBHook[persist.Contract] - tokenDBHooks []DBHook[persist.Token] -} - -// newIndexer sets up an indexer for retrieving the specified events that will process tokens -func newIndexer(ethClient *ethclient.Client, httpClient *http.Client, ipfsClient *shell.Shell, arweaveClient *goar.Client, storageClient *storage.Client, iQueries *indexerdb.Queries, bQueries *coredb.Queries, taskClient *task.Client, contractRepo persist.ContractRepository, pChain persist.Chain, pEvents []eventHash, getLogsFunc getLogsFunc, startingBlock, maxBlock *uint64) *indexer { - if rpcEnabled && ethClient == nil { - panic("RPC is enabled but an ethClient wasn't provided!") - } - - ownerStats := &sync.Map{} - - i := &indexer{ - ethClient: ethClient, - httpClient: httpClient, - ipfsClient: ipfsClient, - arweaveClient: arweaveClient, - storageClient: storageClient, - contractRepo: contractRepo, - queries: iQueries, - dbMu: &sync.Mutex{}, - stateMu: &sync.Mutex{}, - memoryMu: &sync.Mutex{}, - - tokenBucket: env.GetString("GCLOUD_TOKEN_CONTENT_BUCKET"), - - chain: pChain, - - maxBlock: maxBlock, - - contractOwnerStats: ownerStats, - - eventHashes: pEvents, - - getLogsFunc: getLogsFunc, - - contractDBHooks: newContractHooks(iQueries, contractRepo, ethClient, httpClient, ownerStats), - tokenDBHooks: newTokenHooks(taskClient, bQueries), - - mostRecentBlock: 0, - lastSyncedChunk: 0, - isListening: false, - } - - if startingBlock != nil { - i.lastSyncedChunk = *startingBlock - i.lastSyncedChunk -= i.lastSyncedChunk % blocksPerLogsCall - } else { - recentDBBlock, err := contractRepo.MostRecentBlock(context.Background()) - if err != nil { - panic(err) - } - i.lastSyncedChunk = recentDBBlock.Uint64() - - safeSub, overflowed := math.SafeSub(i.lastSyncedChunk, (i.lastSyncedChunk%blocksPerLogsCall)+(blocksPerLogsCall*defaultWorkerPoolSize)) - - if overflowed { - i.lastSyncedChunk = 0 - } else { - i.lastSyncedChunk = safeSub - } - - } - - if maxBlock != nil { - i.mostRecentBlock = *maxBlock - } else if rpcEnabled { - mostRecentBlock, err := ethClient.BlockNumber(context.Background()) - if err != nil { - panic(err) - } - i.mostRecentBlock = mostRecentBlock - } - - if i.lastSyncedChunk > i.mostRecentBlock { - panic(fmt.Sprintf("last handled chunk=%d is greater than the height=%d!", i.lastSyncedChunk, i.mostRecentBlock)) - } - - if i.getLogsFunc == nil { - i.getLogsFunc = i.defaultGetLogs - } - - logger.For(nil).Infof("starting indexer at block=%d until block=%d with rpc enabled: %t", i.lastSyncedChunk, i.mostRecentBlock, rpcEnabled) - return i -} - -// INITIALIZATION FUNCS --------------------------------------------------------- - -// Start begins indexing events from the blockchain -func (i *indexer) Start(ctx context.Context) { - if rpcEnabled && i.maxBlock == nil { - go i.listenForNewBlocks(sentryutil.NewSentryHubContext(ctx)) - } - - topics := eventsToTopics(i.eventHashes) - - logger.For(ctx).Info("Catching up to latest block") - i.isListening = false - i.catchUp(ctx, topics) - - if !rpcEnabled { - logger.For(ctx).Info("Running in cached logs only mode, not listening for new logs") - return - } - - logger.For(ctx).Info("Subscribing to new logs") - i.isListening = true - i.waitForBlocks(ctx, topics) -} - -// catchUp processes logs up to the most recent block. -func (i *indexer) catchUp(ctx context.Context, topics [][]common.Hash) { - wp := workerpool.New(defaultWorkerPoolSize) - defer wp.StopWait() - - go func() { - time.Sleep(10 * time.Second) - for wp.WaitingQueueSize() > 0 { - logger.For(ctx).Infof("Catching up: waiting for %d workers to finish", wp.WaitingQueueSize()) - time.Sleep(10 * time.Second) - } - }() - - from := i.lastSyncedChunk - for ; from < atomic.LoadUint64(&i.mostRecentBlock); from += blocksPerLogsCall { - input := from - toQueue := func() { - workerCtx := sentryutil.NewSentryHubContext(ctx) - defer recoverAndWait(workerCtx) - defer sentryutil.RecoverAndRaise(workerCtx) - logger.For(workerCtx).Infof("Indexing block range starting at %d", input) - i.startPipeline(workerCtx, persist.BlockNumber(input), topics) - i.updateLastSynced(input) - logger.For(workerCtx).Infof("Finished indexing block range starting at %d", input) - } - if wp.WaitingQueueSize() > defaultWorkerPoolWaitSize { - wp.SubmitWait(toQueue) - } else { - wp.Submit(toQueue) - } - } -} - -func (i *indexer) updateLastSynced(block uint64) { - i.stateMu.Lock() - if i.lastSyncedChunk < block { - i.lastSyncedChunk = block - } - i.stateMu.Unlock() -} - -// waitForBlocks polls for new blocks. -func (i *indexer) waitForBlocks(ctx context.Context, topics [][]common.Hash) { - for { - timeAfterWait := <-time.After(time.Minute * 3) - i.startNewBlocksPipeline(ctx, topics) - logger.For(ctx).Infof("Waiting for new blocks... Finished recent blocks in %s", time.Since(timeAfterWait)) - } -} - -func (i *indexer) startPipeline(ctx context.Context, start persist.BlockNumber, topics [][]common.Hash) { - span, ctx := tracing.StartSpan(ctx, "indexer.pipeline", "catchup", sentry.TransactionName("indexer-main:catchup")) - tracing.AddEventDataToSpan(span, map[string]interface{}{"block": start}) - defer tracing.FinishSpan(span) - - startTime := time.Now() - transfers := make(chan []transfersAtBlock) - plugins := NewTransferPlugins(ctx) - enabledPlugins := []chan<- TransferPluginMsg{plugins.contracts.in, plugins.tokens.in} - - statsID, err := i.queries.InsertStatistic(ctx, indexerdb.InsertStatisticParams{ID: persist.GenerateID(), BlockStart: start, BlockEnd: start + blocksPerLogsCall}) - if err != nil { - panic(err) - } - - go func() { - ctx := sentryutil.NewSentryHubContext(ctx) - span, ctx := tracing.StartSpan(ctx, "indexer.logs", "processLogs") - defer tracing.FinishSpan(span) - - logs := i.fetchLogs(ctx, start, topics, statsID) - i.processLogs(ctx, transfers, logs) - }() - go i.processAllTransfers(sentryutil.NewSentryHubContext(ctx), transfers, enabledPlugins, statsID) - i.processTokens(ctx, plugins.tokens.out, plugins.contracts.out, statsID) - - err = i.queries.UpdateStatisticSuccess(ctx, indexerdb.UpdateStatisticSuccessParams{ID: statsID, Success: true, ProcessingTimeSeconds: sql.NullInt64{Int64: int64(time.Since(startTime) / time.Second), Valid: true}}) - if err != nil { - panic(err) - } - - logger.For(ctx).Warnf("Finished processing %d blocks from block %d in %s", blocksPerLogsCall, start.Uint64(), time.Since(startTime)) -} - -func (i *indexer) startNewBlocksPipeline(ctx context.Context, topics [][]common.Hash) { - span, ctx := tracing.StartSpan(ctx, "indexer.pipeline", "polling", sentry.TransactionName("indexer-main:polling")) - defer tracing.FinishSpan(span) - - transfers := make(chan []transfersAtBlock) - plugins := NewTransferPlugins(ctx) - enabledPlugins := []chan<- TransferPluginMsg{plugins.contracts.in, plugins.tokens.in} - - mostRecentBlock, err := rpc.RetryGetBlockNumber(ctx, i.ethClient) - if err != nil { - panic(err) - } - - if i.lastSyncedChunk+blocksPerLogsCall > mostRecentBlock { - logger.For(ctx).Infof("No new blocks to process. Last synced block: %d, most recent block: %d", i.lastSyncedChunk, mostRecentBlock) - return - } - - statsID, err := i.queries.InsertStatistic(ctx, indexerdb.InsertStatisticParams{ID: persist.GenerateID(), BlockStart: persist.BlockNumber(i.lastSyncedChunk), BlockEnd: persist.BlockNumber(mostRecentBlock - (i.mostRecentBlock % blocksPerLogsCall))}) - if err != nil { - panic(err) - } - - go i.pollNewLogs(sentryutil.NewSentryHubContext(ctx), transfers, topics, mostRecentBlock, statsID) - go i.processAllTransfers(sentryutil.NewSentryHubContext(ctx), transfers, enabledPlugins, statsID) - i.processTokens(ctx, plugins.tokens.out, plugins.contracts.out, statsID) - -} - -func (i *indexer) listenForNewBlocks(ctx context.Context) { - defer sentryutil.RecoverAndRaise(ctx) - - for { - <-time.After(time.Second*12*time.Duration(blocksPerLogsCall) + time.Minute) - finalBlockUint, err := rpc.RetryGetBlockNumber(ctx, i.ethClient) - if err != nil { - panic(fmt.Sprintf("error getting block number: %s", err)) - } - atomic.StoreUint64(&i.mostRecentBlock, finalBlockUint) - logger.For(ctx).Debugf("final block number: %v", finalBlockUint) - } -} - -// LOGS FUNCS --------------------------------------------------------------- - -func (i *indexer) fetchLogs(ctx context.Context, startingBlock persist.BlockNumber, topics [][]common.Hash, statsID persist.DBID) []types.Log { - curBlock := startingBlock.BigInt() - nextBlock := new(big.Int).Add(curBlock, big.NewInt(int64(blocksPerLogsCall))) - - logger.For(ctx).Infof("Getting logs from %d to %d", curBlock, nextBlock) - - logsTo, err := i.getLogsFunc(ctx, curBlock, nextBlock, topics) - if err != nil { - panic(fmt.Sprintf("error getting logs: %s", err)) - } - - err = i.queries.UpdateStatisticTotalLogs(ctx, indexerdb.UpdateStatisticTotalLogsParams{ - ID: statsID, - TotalLogs: sql.NullInt64{Int64: int64(len(logsTo)), Valid: true}, - }) - if err != nil { - panic(err) - } - - logger.For(ctx).Infof("Found %d logs at block %d", len(logsTo), curBlock.Uint64()) - - return logsTo -} - -func (i *indexer) defaultGetLogs(ctx context.Context, curBlock, nextBlock *big.Int, topics [][]common.Hash) ([]types.Log, error) { - var logsTo []types.Log - reader, err := i.storageClient.Bucket(env.GetString("GCLOUD_TOKEN_LOGS_BUCKET")).Object(fmt.Sprintf("%d-%d", curBlock, nextBlock)).NewReader(ctx) - if err != nil { - logger.For(ctx).WithError(err).Warn("error getting logs from GCP") - } else { - func() { - logger.For(ctx).Infof("Reading logs from GCP") - i.memoryMu.Lock() - defer i.memoryMu.Unlock() - defer reader.Close() - err = json.NewDecoder(reader).Decode(&logsTo) - if err != nil { - panic(err) - } - }() - } - - if len(logsTo) > 0 { - lastLog := logsTo[len(logsTo)-1] - if nextBlock.Uint64()-lastLog.BlockNumber > (blocksPerLogsCall / 5) { - logger.For(ctx).Warnf("Last log is %d blocks old, skipping", nextBlock.Uint64()-lastLog.BlockNumber) - logsTo = []types.Log{} - } - } - - rpcCtx, cancel := context.WithTimeout(ctx, time.Second*30) - defer cancel() - - if len(logsTo) == 0 && rpcEnabled { - logger.For(ctx).Infof("Reading logs from Blockchain") - logsTo, err = rpc.RetryGetLogs(rpcCtx, i.ethClient, ethereum.FilterQuery{ - FromBlock: curBlock, - ToBlock: nextBlock, - Topics: topics, - }) - if err != nil { - logEntry := logger.For(ctx).WithError(err).WithFields(logrus.Fields{ - "fromBlock": curBlock.String(), - "toBlock": nextBlock.String(), - "rpcCall": "eth_getFilterLogs", - }) - if rpcErr, ok := err.(gethrpc.Error); ok { - logEntry = logEntry.WithFields(logrus.Fields{"rpcErrorCode": strconv.Itoa(rpcErr.ErrorCode())}) - } - logEntry.Error("failed to fetch logs") - return []types.Log{}, nil - } - go saveLogsInBlockRange(ctx, curBlock.String(), nextBlock.String(), logsTo, i.storageClient, i.memoryMu) - } - logger.For(ctx).Infof("Found %d logs at block %d", len(logsTo), curBlock.Uint64()) - return logsTo, nil -} - -func (i *indexer) processLogs(ctx context.Context, transfersChan chan<- []transfersAtBlock, logsTo []types.Log) { - defer close(transfersChan) - defer recoverAndWait(ctx) - defer sentryutil.RecoverAndRaise(ctx) - - transfers := logsToTransfers(ctx, logsTo) - - logger.For(ctx).Infof("Processed %d logs into %d transfers", len(logsTo), len(transfers)) - - transfersChan <- transfersToTransfersAtBlock(transfers) -} - -func logsToTransfers(ctx context.Context, pLogs []types.Log) []rpc.Transfer { - - result := make([]rpc.Transfer, 0, len(pLogs)*2) - for _, pLog := range pLogs { - - switch { - case strings.EqualFold(pLog.Topics[0].Hex(), string(transferEventHash)): - - if len(pLog.Topics) < 4 { - continue - } - - result = append(result, rpc.Transfer{ - From: persist.EthereumAddress(pLog.Topics[1].Hex()), - To: persist.EthereumAddress(pLog.Topics[2].Hex()), - TokenID: persist.HexTokenID(pLog.Topics[3].Hex()), - Amount: 1, - BlockNumber: persist.BlockNumber(pLog.BlockNumber), - ContractAddress: persist.EthereumAddress(pLog.Address.Hex()), - TokenType: persist.TokenTypeERC721, - TxHash: pLog.TxHash, - BlockHash: pLog.BlockHash, - TxIndex: pLog.TxIndex, - }) - - case strings.EqualFold(pLog.Topics[0].Hex(), string(transferSingleEventHash)): - if len(pLog.Topics) < 4 { - continue - } - - eventData := map[string]interface{}{} - err := erc1155ABI.UnpackIntoMap(eventData, "TransferSingle", pLog.Data) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to unpack TransferSingle event") - panic(err) - } - - id, ok := eventData["id"].(*big.Int) - if !ok { - panic("Failed to unpack TransferSingle event, id not found") - } - - value, ok := eventData["value"].(*big.Int) - if !ok { - panic("Failed to unpack TransferSingle event, value not found") - } - - result = append(result, rpc.Transfer{ - From: persist.EthereumAddress(pLog.Topics[2].Hex()), - To: persist.EthereumAddress(pLog.Topics[3].Hex()), - TokenID: persist.HexTokenID(id.Text(16)), - Amount: value.Uint64(), - BlockNumber: persist.BlockNumber(pLog.BlockNumber), - ContractAddress: persist.EthereumAddress(pLog.Address.Hex()), - TokenType: persist.TokenTypeERC1155, - TxHash: pLog.TxHash, - BlockHash: pLog.BlockHash, - TxIndex: pLog.TxIndex, - }) - - case strings.EqualFold(pLog.Topics[0].Hex(), string(transferBatchEventHash)): - if len(pLog.Topics) < 4 { - continue - } - - eventData := map[string]interface{}{} - err := erc1155ABI.UnpackIntoMap(eventData, "TransferBatch", pLog.Data) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to unpack TransferBatch event") - panic(err) - } - - ids, ok := eventData["ids"].([]*big.Int) - if !ok { - panic("Failed to unpack TransferBatch event, ids not found") - } - - values, ok := eventData["values"].([]*big.Int) - if !ok { - panic("Failed to unpack TransferBatch event, values not found") - } - - for j := 0; j < len(ids); j++ { - - result = append(result, rpc.Transfer{ - From: persist.EthereumAddress(pLog.Topics[2].Hex()), - To: persist.EthereumAddress(pLog.Topics[3].Hex()), - TokenID: persist.HexTokenID(ids[j].Text(16)), - Amount: values[j].Uint64(), - ContractAddress: persist.EthereumAddress(pLog.Address.Hex()), - TokenType: persist.TokenTypeERC1155, - BlockNumber: persist.BlockNumber(pLog.BlockNumber), - TxHash: pLog.TxHash, - BlockHash: pLog.BlockHash, - TxIndex: pLog.TxIndex, - }) - } - - default: - logger.For(ctx).WithFields(logrus.Fields{ - "address": pLog.Address, - "block": pLog.BlockNumber, - "eventType": pLog.Topics[0]}, - ).Warn("unknown event") - } - } - return result -} - -func (i *indexer) pollNewLogs(ctx context.Context, transfersChan chan<- []transfersAtBlock, topics [][]common.Hash, mostRecentBlock uint64, statsID persist.DBID) { - span, ctx := tracing.StartSpan(ctx, "indexer.logs", "pollLogs") - defer tracing.FinishSpan(span) - defer close(transfersChan) - defer recoverAndWait(ctx) - defer sentryutil.RecoverAndRaise(ctx) - - logger.For(ctx).Infof("Subscribing to new logs from block %d starting with block %d", mostRecentBlock, i.lastSyncedChunk) - - totalLogs := &atomic.Int64{} - - wp := workerpool.New(10) - // starting at the last chunk that we synced, poll for logs in chunks of blocksPerLogsCall - for j := i.lastSyncedChunk; j+blocksPerLogsCall <= mostRecentBlock; j += blocksPerLogsCall { - curBlock := j - wp.Submit(func() { - ctx := sentryutil.NewSentryHubContext(ctx) - defer sentryutil.RecoverAndRaise(ctx) - - nextBlock := curBlock + blocksPerLogsCall - - rpcCtx, cancel := context.WithTimeout(ctx, time.Second*30) - defer cancel() - - logsTo, err := rpc.RetryGetLogs(rpcCtx, i.ethClient, ethereum.FilterQuery{ - FromBlock: persist.BlockNumber(curBlock).BigInt(), - ToBlock: persist.BlockNumber(nextBlock).BigInt(), - Topics: topics, - }) - if err != nil { - errData := map[string]interface{}{ - "from": curBlock, - "to": nextBlock, - "err": err.Error(), - } - logger.For(ctx).WithError(err).Error(errData) - return - } - - totalLogs.Add(int64(len(logsTo))) - - go saveLogsInBlockRange(ctx, strconv.Itoa(int(curBlock)), strconv.Itoa(int(nextBlock)), logsTo, i.storageClient, i.memoryMu) - - logger.For(ctx).Infof("Found %d logs at block %d", len(logsTo), curBlock) - - transfers := logsToTransfers(ctx, logsTo) - - logger.For(ctx).Infof("Processed %d logs into %d transfers", len(logsTo), len(transfers)) - - logger.For(ctx).Debugf("Sending %d total transfers to transfers channel", len(transfers)) - transfersChan <- transfersToTransfersAtBlock(transfers) - - }) - } - - wp.StopWait() - - total := totalLogs.Load() - - err := i.queries.UpdateStatisticTotalLogs(ctx, indexerdb.UpdateStatisticTotalLogsParams{ - TotalLogs: sql.NullInt64{Int64: total, Valid: true}, - ID: statsID, - }) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to update total logs") - panic(err) - } - - logger.For(ctx).Infof("Processed %d logs from %d to %d.", total, i.lastSyncedChunk, mostRecentBlock) - - i.updateLastSynced(mostRecentBlock - (mostRecentBlock % blocksPerLogsCall)) - -} - -// TRANSFERS FUNCS ------------------------------------------------------------- - -func (i *indexer) processAllTransfers(ctx context.Context, incomingTransfers <-chan []transfersAtBlock, plugins []chan<- TransferPluginMsg, statsID persist.DBID) { - span, ctx := tracing.StartSpan(ctx, "indexer.transfers", "processTransfers") - defer tracing.FinishSpan(span) - defer sentryutil.RecoverAndRaise(ctx) - for _, plugin := range plugins { - defer close(plugin) - } - - wp := workerpool.New(5) - - logger.For(ctx).Info("Starting to process transfers...") - var totatTransfers int64 - for transfers := range incomingTransfers { - if len(transfers) == 0 { - continue - } - - totatTransfers += int64(len(transfers)) - - submit := transfers - wp.Submit(func() { - ctx := sentryutil.NewSentryHubContext(ctx) - timeStart := time.Now() - logger.For(ctx).Infof("Processing %d transfers", len(submit)) - i.processTransfers(ctx, submit, plugins) - logger.For(ctx).Infof("Processed %d transfers in %s", len(submit), time.Since(timeStart)) - }) - } - - logger.For(ctx).Info("Waiting for transfers to finish...") - wp.StopWait() - - err := i.queries.UpdateStatisticTotalTransfers(ctx, indexerdb.UpdateStatisticTotalTransfersParams{ - TotalTransfers: sql.NullInt64{Int64: totatTransfers, Valid: true}, - ID: statsID, - }) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to update total transfers") - panic(err) - } - - logger.For(ctx).Info("Closing field channels...") -} - -func (i *indexer) processTransfers(ctx context.Context, transfers []transfersAtBlock, plugins []chan<- TransferPluginMsg) { - - for _, transferAtBlock := range transfers { - for _, transfer := range transferAtBlock.transfers { - - contractAddress := persist.EthereumAddress(transfer.ContractAddress.String()) - - tokenID := transfer.TokenID - - key := persist.NewEthereumTokenIdentifiers(contractAddress, tokenID) - - RunTransferPlugins(ctx, transfer, key, plugins) - - } - - } - -} - -// TOKENS FUNCS --------------------------------------------------------------- - -func (i *indexer) processTokens(ctx context.Context, tokensOut <-chan tokenAtBlock, contractsOut <-chan contractAtBlock, statsID persist.DBID) { - - wg := &sync.WaitGroup{} - mu := &sync.Mutex{} - - contractsMap := make(map[persist.EthereumTokenIdentifiers]contractAtBlock) - tokensMap := make(map[persist.EthereumTokenIdentifiers]tokenAtBlock) - - RunTransferPluginReceiver(ctx, wg, mu, contractsPluginReceiver, contractsOut, contractsMap) - RunTransferPluginReceiver(ctx, wg, mu, tokensPluginReceiver, tokensOut, tokensMap) - - wg.Wait() - - contracts := contractsAtBlockToContracts(contractsMap) - tokens := tokensAtBlockToTokens(tokensMap) - - i.queries.UpdateStatisticTotalTokensAndContracts(ctx, indexerdb.UpdateStatisticTotalTokensAndContractsParams{ - TotalContracts: sql.NullInt64{Int64: int64(len(contracts)), Valid: true}, - ID: statsID, - }) - - i.runDBHooks(ctx, contracts, tokens, statsID) -} - -func contractsAtBlockToContracts(contractsAtBlock map[persist.EthereumTokenIdentifiers]contractAtBlock) []persist.Contract { - contracts := make([]persist.Contract, 0, len(contractsAtBlock)) - seen := make(map[persist.EthereumAddress]bool) - for _, contract := range contractsAtBlock { - if seen[contract.contract.Address] { - continue - } - contracts = append(contracts, contract.contract) - seen[contract.contract.Address] = true - } - return contracts -} - -func tokensAtBlockToTokens(tokensAtBlock map[persist.EthereumTokenIdentifiers]tokenAtBlock) []persist.Token { - tokens := make([]persist.Token, 0, len(tokensAtBlock)) - seen := make(map[persist.TokenUniqueIdentifiers]bool) - for _, token := range tokensAtBlock { - tids := persist.TokenUniqueIdentifiers{ - Chain: token.token.Chain, - ContractAddress: persist.Address(token.token.ContractAddress), - TokenID: token.token.TokenID, - OwnerAddress: persist.Address(token.token.OwnerAddress), - } - if seen[tids] { - continue - } - tokens = append(tokens, token.token) - seen[tids] = true - } - return tokens -} - -func (i *indexer) runDBHooks(ctx context.Context, contracts []persist.Contract, tokens []persist.Token, statsID persist.DBID) { - defer recoverAndWait(ctx) - - wp := workerpool.New(10) - - for _, hook := range i.contractDBHooks { - hook := hook - wp.Submit(func() { - err := hook(ctx, contracts, statsID) - if err != nil { - logger.For(ctx).WithError(err).Errorf("Failed to run contract db hook %s", err) - } - }) - } - - for _, hook := range i.tokenDBHooks { - hook := hook - wp.Submit(func() { - err := hook(ctx, tokens, statsID) - if err != nil { - logger.For(ctx).WithError(err).Errorf("Failed to run token db hook %s", err) - } - }) - } - - wp.StopWait() -} - -func contractsPluginReceiver(cur contractAtBlock, inc contractAtBlock) contractAtBlock { - return inc -} - -func tokensPluginReceiver(cur tokenAtBlock, inc tokenAtBlock) tokenAtBlock { - if cur.token.TokenType == persist.TokenTypeERC1155 { - inc.token.Quantity = cur.token.Quantity.Add(inc.token.Quantity) - } - return inc -} - -type AlchemyContract struct { - Address persist.EthereumAddress `json:"address"` - ContractMetadata AlchemyContractMetadata `json:"contractMetadata"` -} - -type AlchemyContractMetadata struct { - Address persist.EthereumAddress `json:"address"` - Metadata alchemy.ContractMetadata `json:"contractMetadata"` - ContractDeployer persist.EthereumAddress `json:"contractDeployer"` -} - -func fillContractFields(ctx context.Context, contracts []persist.Contract, queries *indexerdb.Queries, contractRepo persist.ContractRepository, httpClient *http.Client, ethClient *ethclient.Client, contractOwnerStats *sync.Map, upChan chan<- []persist.Contract, statsID persist.DBID) { - defer close(upChan) - - innerPipelineStats := map[persist.ContractOwnerMethod]any{} - - contractsNotInDB := make(chan persist.Contract) - - batched := make(chan []persist.Contract) - - go func() { - defer close(contractsNotInDB) - - iter.ForEach(contracts, func(c *persist.Contract) { - _, err := contractRepo.GetByAddress(ctx, c.Address) - if err == nil { - return - } - contractsNotInDB <- *c - }) - }() - - go func() { - defer close(batched) - var curBatch []persist.Contract - for contract := range contractsNotInDB { - curBatch = append(curBatch, contract) - if len(curBatch) == 100 { - logger.For(ctx).Infof("Batching %d contracts for metadata", len(curBatch)) - batched <- curBatch - curBatch = []persist.Contract{} - } - } - if len(curBatch) > 0 { - logger.For(ctx).Infof("Batching %d contracts for metadata", len(curBatch)) - batched <- curBatch - } - }() - - // process contracts in batches of 100 - for batch := range batched { - // get contract metadata - toUp, _ := GetContractMetadatas(ctx, batch, httpClient, ethClient) - - for _, c := range toUp { - it, ok := contractOwnerStats.LoadOrStore(c.OwnerMethod, 1) - if ok { - total := it.(int) - total++ - contractOwnerStats.Store(c.OwnerMethod, total) - } - - it, ok = innerPipelineStats[c.OwnerMethod] - if ok { - total := it.(int) - total++ - innerPipelineStats[c.OwnerMethod] = total - } else { - innerPipelineStats[c.OwnerMethod] = 1 - } - } - - logger.For(ctx).Infof("Fetched metadata for %d contracts", len(toUp)) - - asContracts, _ := util.Map(toUp, func(c ContractOwnerResult) (persist.Contract, error) { - return c.Contract, nil - }) - upChan <- asContracts - } - - marshalled, err := json.Marshal(innerPipelineStats) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to marshal inner pipeline stats") - panic(err) - } - - err = queries.UpdateStatisticContractStats(ctx, indexerdb.UpdateStatisticContractStatsParams{ - ContractStats: pgtype.JSONB{Bytes: marshalled, Status: pgtype.Present}, - ID: statsID, - }) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to update contract stats") - panic(err) - } - - logger.For(ctx).Infof("Fetched metadata for total %d contracts", len(contracts)) - -} - -type ContractOwnerResult struct { - Contract persist.Contract `json:"contract"` - OwnerMethod persist.ContractOwnerMethod `json:"ownerMethod"` -} - -func GetContractMetadatas(ctx context.Context, batch []persist.Contract, httpClient *http.Client, ethClient *ethclient.Client) ([]ContractOwnerResult, error) { - toUp := make([]ContractOwnerResult, 0, 100) - - cToAddr := make(map[string]persist.Contract) - for _, c := range batch { - cToAddr[c.Address.String()] = c - } - - addresses := util.MapKeys(cToAddr) - - u := fmt.Sprintf("%s/getContractMetadataBatch", env.GetString("ALCHEMY_API_URL")) - - in := map[string][]string{"contractAddresses": addresses} - inAsJSON, err := json.Marshal(in) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to marshal contract metadata request") - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewBuffer(inAsJSON)) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to create contract metadata request") - return nil, err - } - - req.Header.Add("accept", "application/json") - req.Header.Add("content-type", "application/json") - - resp, err := httpClient.Do(req) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to execute contract metadata request") - return nil, err - } - - if resp.StatusCode != http.StatusOK { - bodyAsBytes, _ := io.ReadAll(resp.Body) - logger.For(ctx).Errorf("Failed to execute contract metadata request: %s status: %s (url: %s) (input: %s) ", string(bodyAsBytes), resp.Status, u, string(inAsJSON)) - return nil, err - } - - var out []AlchemyContract - err = json.NewDecoder(resp.Body).Decode(&out) - if err != nil { - logger.For(ctx).WithError(err).Error("Failed to decode contract metadata response") - return nil, err - } - - for _, c := range out { - - result := ContractOwnerResult{} - contract := cToAddr[c.Address.String()] - contract.Name = persist.NullString(c.ContractMetadata.Metadata.Name) - contract.Symbol = persist.NullString(c.ContractMetadata.Metadata.Symbol) - - var method = persist.ContractOwnerMethodAlchemy - cOwner, err := rpc.GetContractOwner(ctx, c.Address, ethClient) - if err != nil { - logger.For(ctx).WithError(err).WithFields(logrus.Fields{ - "contractAddress": c.Address, - }).Error("error getting contract owner") - contract.OwnerAddress = c.ContractMetadata.ContractDeployer - } else { - contract.OwnerAddress = cOwner - method = persist.ContractOwnerMethodOwnable - } - - if contract.OwnerAddress == "" { - method = persist.ContractOwnerMethodFailed - } - - contract.CreatorAddress = c.ContractMetadata.ContractDeployer - - result.OwnerMethod = method - result.Contract = contract - - toUp = append(toUp, result) - } - return toUp, nil -} - -// HELPER FUNCS --------------------------------------------------------------- - -func transfersToTransfersAtBlock(transfers []rpc.Transfer) []transfersAtBlock { - transfersMap := map[persist.BlockNumber]transfersAtBlock{} - - for _, transfer := range transfers { - if tab, ok := transfersMap[transfer.BlockNumber]; !ok { - transfers := make([]rpc.Transfer, 0, 10) - transfers = append(transfers, transfer) - transfersMap[transfer.BlockNumber] = transfersAtBlock{ - block: transfer.BlockNumber, - transfers: transfers, - } - } else { - tab.transfers = append(tab.transfers, transfer) - transfersMap[transfer.BlockNumber] = tab - } - } - - allTransfersAtBlock := make([]transfersAtBlock, len(transfersMap)) - i := 0 - for _, transfersAtBlock := range transfersMap { - allTransfersAtBlock[i] = transfersAtBlock - i++ - } - sort.Slice(allTransfersAtBlock, func(i, j int) bool { - return allTransfersAtBlock[i].block < allTransfersAtBlock[j].block - }) - return allTransfersAtBlock -} - -func saveLogsInBlockRange(ctx context.Context, curBlock, nextBlock string, logsTo []types.Log, storageClient *storage.Client, memoryMu *sync.Mutex) { - memoryMu.Lock() - defer memoryMu.Unlock() - logger.For(ctx).Infof("Saving logs in block range %s to %s", curBlock, nextBlock) - obj := storageClient.Bucket(env.GetString("GCLOUD_TOKEN_LOGS_BUCKET")).Object(fmt.Sprintf("%s-%s", curBlock, nextBlock)) - obj.Delete(ctx) - storageWriter := obj.NewWriter(ctx) - - if err := json.NewEncoder(storageWriter).Encode(logsTo); err != nil { - panic(err) - } - if err := storageWriter.Close(); err != nil { - panic(err) - } -} - -func recoverAndWait(ctx context.Context) { - if err := recover(); err != nil { - logger.For(ctx).Errorf("Error in indexer: %v", err) - time.Sleep(time.Second * 10) - } -} - -func eventsToTopics(hashes []eventHash) [][]common.Hash { - events := make([]common.Hash, len(hashes)) - for i, event := range hashes { - events[i] = common.HexToHash(string(event)) - } - return [][]common.Hash{events} -} diff --git a/indexer/plugin.go b/indexer/plugin.go deleted file mode 100644 index d4fef8ea0..000000000 --- a/indexer/plugin.go +++ /dev/null @@ -1,257 +0,0 @@ -package indexer - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/gammazero/workerpool" - "github.com/getsentry/sentry-go" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/rpc" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" - "github.com/mikeydub/go-gallery/service/tracing" - "github.com/sirupsen/logrus" -) - -const ( - pluginPoolSize = 32 - pluginTimeout = 2 * time.Minute -) - -// TransferPluginMsg is used to communicate to a plugin. -type TransferPluginMsg struct { - transfer rpc.Transfer - key persist.EthereumTokenIdentifiers -} - -// TransferPlugins are plugins that add contextual data to a transfer. -type TransferPlugins struct { - contracts contractTransfersPlugin - tokens tokenTransfersPlugin -} - -type blockchainOrderInfo struct { - blockNumber persist.BlockNumber - txIndex uint -} - -// Less returns true if the current block number and tx index are less than the other block number and tx index. -func (b blockchainOrderInfo) Less(other blockchainOrderInfo) bool { - if b.blockNumber < other.blockNumber { - return true - } - if b.blockNumber > other.blockNumber { - return false - } - return b.txIndex < other.txIndex -} - -type orderedBlockChainData interface { - TokenIdentifiers() persist.EthereumTokenIdentifiers - OrderInfo() blockchainOrderInfo -} - -// TransferPluginReceiver receives the results of a plugin. -type TransferPluginReceiver[T, V orderedBlockChainData] func(cur V, inc T) V - -func startSpan(ctx context.Context, plugin, op string) (*sentry.Span, context.Context) { - return tracing.StartSpan(ctx, "indexer.plugin", fmt.Sprintf("%s:%s", plugin, op)) -} - -// NewTransferPlugins returns a set of transfer plugins. Plugins have an `in` and an optional `out` channel that are handles to the service. -// The `in` channel is used to submit a transfer to a plugin, and the `out` channel is used to receive results from a plugin, if any. -// A plugin can be stopped by closing its `in` channel, which finishes the plugin and lets receivers know that its done. -func NewTransferPlugins(ctx context.Context) TransferPlugins { - ctx = sentryutil.NewSentryHubContext(ctx) - return TransferPlugins{ - contracts: newContractsPlugin(ctx), - tokens: newTokensPlugin(ctx), - } -} - -// RunTransferPlugins returns when all plugins have received the message. Every plugin recieves the same message. -func RunTransferPlugins(ctx context.Context, transfer rpc.Transfer, key persist.EthereumTokenIdentifiers, plugins []chan<- TransferPluginMsg) { - span, _ := tracing.StartSpan(ctx, "indexer.plugin", "submitMessage") - defer tracing.FinishSpan(span) - - msg := TransferPluginMsg{ - transfer: transfer, - key: key, - } - for _, plugin := range plugins { - plugin <- msg - } -} - -// RunTransferPluginReceiver runs a plugin receiver and will update the out map with the results of the receiver, ensuring that the most recent data is kept. -// If the incoming channel is nil, the function will return immediately. -func RunTransferPluginReceiver[T, V orderedBlockChainData](ctx context.Context, wg *sync.WaitGroup, mu *sync.Mutex, receiver TransferPluginReceiver[T, V], incoming <-chan T, out map[persist.EthereumTokenIdentifiers]V) { - span, _ := tracing.StartSpan(ctx, "indexer.plugin", "runPluginReceiver") - defer tracing.FinishSpan(span) - - if incoming == nil { - return - } - - if out == nil { - panic("out map must not be nil") - } - - wg.Add(1) - - go func() { - defer wg.Done() - - for it := range incoming { - func() { - processed := receiver(out[it.TokenIdentifiers()], it) - cur, ok := out[processed.TokenIdentifiers()] - if !ok || cur.OrderInfo().Less(processed.OrderInfo()) { - mu.Lock() - defer mu.Unlock() - out[processed.TokenIdentifiers()] = processed - } - }() - } - - logger.For(ctx).WithFields(logrus.Fields{"incoming_type": fmt.Sprintf("%T", *new(T)), "outgoing_type": fmt.Sprintf("%T", *new(V))}).Info("plugin finished receiving") - }() - -} - -// contractTransfersPlugin retrieves ownership information for a token. -type contractTransfersPlugin struct { - in chan TransferPluginMsg - out chan contractAtBlock -} - -func newContractsPlugin(ctx context.Context) contractTransfersPlugin { - in := make(chan TransferPluginMsg) - out := make(chan contractAtBlock) - - go func() { - span, _ := startSpan(ctx, "contractTransfersPlugin", "handleBatch") - defer tracing.FinishSpan(span) - defer close(out) - - wp := workerpool.New(pluginPoolSize) - - seenContracts := map[persist.EthereumAddress]bool{} - - for msg := range in { - - if seenContracts[msg.transfer.ContractAddress] { - continue - } - - msg := msg - wp.Submit(func() { - - child := span.StartChild("plugin.contractTransfersPlugin") - child.Description = "handleMessage" - - out <- contractAtBlock{ - ti: msg.key, - boi: blockchainOrderInfo{ - blockNumber: msg.transfer.BlockNumber, - txIndex: msg.transfer.TxIndex, - }, - contract: persist.Contract{ - Address: msg.transfer.ContractAddress, - LatestBlock: msg.transfer.BlockNumber, - }, - } - - tracing.FinishSpan(child) - }) - - seenContracts[msg.transfer.ContractAddress] = true - } - - wp.StopWait() - logger.For(ctx).Info("contracts plugin finished sending") - }() - - return contractTransfersPlugin{ - in: in, - out: out, - } -} - -type tokenTransfersPlugin struct { - in chan TransferPluginMsg - out chan tokenAtBlock -} - -func newTokensPlugin(ctx context.Context) tokenTransfersPlugin { - in := make(chan TransferPluginMsg) - out := make(chan tokenAtBlock) - - go func() { - span, _ := startSpan(ctx, "tokenTransfersPlugin", "handleBatch") - defer tracing.FinishSpan(span) - defer close(out) - - wp := workerpool.New(pluginPoolSize) - - seenTokens := map[persist.TokenUniqueIdentifiers]bool{} - - for msg := range in { - - contract, tokenID, err := msg.key.GetParts() - if err != nil { - panic(err) - } - ids := persist.TokenUniqueIdentifiers{ - // TODO currently this can only be ETH but we should transition away from persist.EthereumAddress and hard coded ETH on the indexer if we could see ourselves indexing other EVMs - Chain: persist.ChainETH, - ContractAddress: persist.Address(contract), - TokenID: tokenID, - OwnerAddress: persist.Address(msg.transfer.To.String()), - } - - if seenTokens[ids] && msg.transfer.TokenType != persist.TokenTypeERC1155 { - continue - } - - msg := msg - wp.Submit(func() { - - child := span.StartChild("plugin.tokenTransfersPlugin") - child.Description = "handleMessage" - - out <- tokenAtBlock{ - ti: msg.key, - boi: blockchainOrderInfo{ - blockNumber: msg.transfer.BlockNumber, - txIndex: msg.transfer.TxIndex, - }, - token: persist.Token{ - TokenType: msg.transfer.TokenType, - Chain: persist.ChainETH, - TokenID: tokenID, - OwnerAddress: msg.transfer.To, - BlockNumber: msg.transfer.BlockNumber, - ContractAddress: contract, - Quantity: "1", - }, - } - - tracing.FinishSpan(child) - }) - - seenTokens[ids] = true - } - - wp.StopWait() - logger.For(ctx).Info("tokens plugin finished sending") - }() - - return tokenTransfersPlugin{ - in: in, - out: out, - } -} diff --git a/indexer/server.go b/indexer/server.go deleted file mode 100644 index 55f1e3bde..000000000 --- a/indexer/server.go +++ /dev/null @@ -1,34 +0,0 @@ -package indexer - -import ( - "context" - "net/http" - "time" - - "github.com/gin-gonic/gin" - "github.com/mikeydub/go-gallery/service/persist" -) - -func getStatus(i *indexer, contractRepo persist.ContractRepository) gin.HandlerFunc { - return func(c *gin.Context) { - ctx, cancel := context.WithTimeout(c, 10*time.Second) - defer cancel() - - mostRecent, _ := contractRepo.MostRecentBlock(ctx) - - readableSyncMap := make(map[persist.ContractOwnerMethod]int) - - i.contractOwnerStats.Range(func(key, value interface{}) bool { - readableSyncMap[key.(persist.ContractOwnerMethod)] = value.(int) - return true - }) - - c.JSON(http.StatusOK, gin.H{ - "most_recent_blockchain": i.mostRecentBlock, - "most_recent_db": mostRecent, - "last_synced_chunk": i.lastSyncedChunk, - "is_listening": i.isListening, - "contract_owner_stats": readableSyncMap, - }) - } -} diff --git a/indexer/t__integration_test.go b/indexer/t__integration_test.go deleted file mode 100644 index c88db3931..000000000 --- a/indexer/t__integration_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package indexer - -import ( - "context" - "testing" - "time" - - "github.com/getsentry/sentry-go" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/stretchr/testify/assert" -) - -func TestIndexLogs_Success(t *testing.T) { - a, db, pgx, pgx2 := setupTest(t) - i := newMockIndexer(db, pgx, pgx2) - - // Run the Indexer - i.catchUp(sentry.SetHubOnContext(context.Background(), sentry.CurrentHub()), eventsToTopics(i.eventHashes)) - - // ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - // defer cancel() - - // ethClient := rpc.NewEthClient() - - t.Run("it updates its state", func(t *testing.T) { - a.EqualValues(testBlockTo-blocksPerLogsCall, i.lastSyncedChunk) - }) - - t.Run("it saves contracts to the db", func(t *testing.T) { - for _, address := range expectedContracts() { - contract := contractExistsInDB(t, a, i.contractRepo, address) - a.NotEmpty(contract.ID) - a.Equal(address, contract.Address) - } - }) - -} - -func contractExistsInDB(t *testing.T, a *assert.Assertions, contractRepo persist.ContractRepository, address persist.EthereumAddress) persist.Contract { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - contract, err := contractRepo.GetByAddress(ctx, address) - a.NoError(err) - return contract -} diff --git a/indexer/t__utils_test.go b/indexer/t__utils_test.go deleted file mode 100644 index ab3d9182f..000000000 --- a/indexer/t__utils_test.go +++ /dev/null @@ -1,279 +0,0 @@ -package indexer - -import ( - "context" - "database/sql" - "math/big" - "net/http" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/jackc/pgx/v4/pgxpool" - migrate "github.com/mikeydub/go-gallery/db" - "github.com/mikeydub/go-gallery/db/gen/coredb" - "github.com/mikeydub/go-gallery/db/gen/indexerdb" - "github.com/mikeydub/go-gallery/docker" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/service/persist/postgres" - "github.com/mikeydub/go-gallery/service/rpc" - "github.com/mikeydub/go-gallery/service/task" - "github.com/stretchr/testify/assert" -) - -var ( - testBlockFrom = 0 - testBlockTo = 100 - testAddress = "0x9a3f9764b21adaf3c6fdf6f947e6d3340a3f8ac5" - galleryMembershipAddress = "0xe01569ca9b39e55bc7c0dfa09f05fa15cb4c7698" - ensAddress = "0x57f1887a8bf19b14fc0df6fd9b2acc9af147ea85" - contribAddress = "0xda3845b44736b57e05ee80fc011a52a9c777423a" // Jarrel's address with a contributor card in it -) - -var allLogs = func() []types.Log { - logs := htmlLogs - logs = append(logs, ipfsLogs...) - logs = append(logs, customHandlerLogs...) - logs = append(logs, svgLogs...) - logs = append(logs, erc1155Logs...) - logs = append(logs, ensLogs...) - return logs -}() - -func setupTest(t *testing.T) (*assert.Assertions, *sql.DB, *pgxpool.Pool, *pgxpool.Pool) { - SetDefaults() - LoadConfigFile("indexer-server", "local") - ValidateEnv() - - r, err := docker.StartPostgresIndexer() - if err != nil { - t.Fatal(err) - } - - r2, err := docker.StartPostgres() - if err != nil { - t.Fatal(err) - } - - hostAndPort := strings.Split(r.GetHostPort("5432/tcp"), ":") - t.Setenv("POSTGRES_HOST", hostAndPort[0]) - t.Setenv("POSTGRES_PORT", hostAndPort[1]) - - hostAndPort2 := strings.Split(r2.GetHostPort("5432/tcp"), ":") - - db := postgres.MustCreateClient() - pgx := postgres.NewPgxClient() - pgx2 := postgres.NewPgxClient(postgres.WithHost(hostAndPort2[0]), postgres.WithPort(5432)) - migrate, err := migrate.RunMigration(db, "./db/migrations/indexer") - if err != nil { - t.Fatalf("failed to seed db: %s", err) - } - t.Cleanup(func() { - migrate.Close() - r.Close() - }) - - return assert.New(t), db, pgx, pgx2 -} - -func newMockIndexer(db *sql.DB, pool, pool2 *pgxpool.Pool) *indexer { - start := uint64(testBlockFrom) - end := uint64(testBlockTo) - rpcEnabled = true - ethClient := rpc.NewEthSocketClient() - iQueries := indexerdb.New(pool) - bQueries := coredb.New(pool2) - - i := newIndexer(ethClient, &http.Client{Timeout: 10 * time.Minute}, nil, nil, nil, iQueries, bQueries, task.NewClient(context.Background()), postgres.NewContractRepository(db), persist.ChainETH, defaultTransferEvents, func(ctx context.Context, curBlock, nextBlock *big.Int, topics [][]common.Hash) ([]types.Log, error) { - transferAgainLogs := []types.Log{{ - Address: common.HexToAddress("0x0c2ee19b2a89943066c2dc7f1bddcc907f614033"), - Topics: []common.Hash{common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), common.HexToHash(testAddress), common.HexToHash("0x0000000000000000000000008914496dc01efcc49a2fa340331fb90969b6f1d2"), common.HexToHash("0x00000000000000000000000000000000000000000000000000000000000000d9")}, - Data: []byte{}, - BlockNumber: 51, - TxIndex: 1, - }} - if curBlock.Uint64() == 0 { - return allLogs, nil - } - return transferAgainLogs, nil - }, &start, &end) - return i -} - -var htmlLogs = []types.Log{ - { - Address: common.HexToAddress("0x0c2ee19b2a89943066c2dc7f1bddcc907f614033"), - Topics: []common.Hash{ - common.HexToHash(string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - common.HexToHash("0x00000000000000000000000000000000000000000000000000000000000000d9"), - }, - BlockNumber: 1, - }, - { - Address: common.HexToAddress("0x059edd72cd353df5106d2b9cc5ab83a52287ac3a"), - Topics: []common.Hash{ - common.HexToHash(string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"), - }, - BlockNumber: 1, - }, -} -var ipfsLogs = []types.Log{ - { - Address: common.HexToAddress("0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d"), - Topics: []common.Hash{common.HexToHash( - string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"), - }, - BlockNumber: 2, - }, -} -var customHandlerLogs = []types.Log{ - { - Address: common.HexToAddress("0xd4e4078ca3495de5b1d4db434bebc5a986197782"), - Topics: []common.Hash{ - common.HexToHash(string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"), - }, - BlockNumber: 22, - }, -} -var svgLogs = []types.Log{ - { - Address: common.HexToAddress("0x69c40e500b84660cb2ab09cb9614fa2387f95f64"), - Topics: []common.Hash{ - common.HexToHash(string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"), - }, - BlockNumber: 3, - }, -} -var erc1155Logs = []types.Log{ - { - Address: common.HexToAddress(galleryMembershipAddress), - Topics: []common.Hash{ - common.HexToHash(string(transferSingleEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(contribAddress), - }, - Data: common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), - BlockNumber: 5, - }, -} -var ensLogs = []types.Log{ - { - Address: common.HexToAddress(ensAddress), - Topics: []common.Hash{ - common.HexToHash(string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - common.HexToHash("0xc1cb7903f69821967b365cce775cd62d694cd7ae7cfe00efe1917a55fdae2bb7"), - }, - BlockNumber: 42, - }, - { - Address: common.HexToAddress(ensAddress), - Topics: []common.Hash{ - common.HexToHash(string(transferEventHash)), - common.HexToHash(persist.ZeroAddress.String()), - common.HexToHash(testAddress), - // Leading zero in token ID - common.HexToHash("0x08c111a4e7c31becd720bde47f538417068e102d45b7732f24cfeda9e2b22a45"), - }, - BlockNumber: 42, - }, -} - -type expectedTokenResults map[persist.TokenIdentifiers]persist.Token - -var expectedResults expectedTokenResults = expectedTokenResults{ - persist.NewTokenIdentifiers("0x059edd72cd353df5106d2b9cc5ab83a52287ac3a", "1", 0): { - BlockNumber: 1, - OwnerAddress: persist.EthereumAddress(testAddress), - ContractAddress: persist.EthereumAddress("0x059edd72cd353df5106d2b9cc5ab83a52287ac3a"), - TokenType: persist.TokenTypeERC721, - TokenID: "1", - Quantity: "1", - }, - persist.NewTokenIdentifiers("0x69c40e500b84660cb2ab09cb9614fa2387f95f64", "1", 0): persist.Token{ - BlockNumber: 3, - OwnerAddress: persist.EthereumAddress(testAddress), - ContractAddress: persist.EthereumAddress("0x69c40e500b84660cb2ab09cb9614fa2387f95f64"), - TokenType: persist.TokenTypeERC721, - TokenID: "1", - Quantity: "1", - }, - persist.NewTokenIdentifiers("0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d", "1", 0): persist.Token{ - BlockNumber: 2, - OwnerAddress: persist.EthereumAddress(testAddress), - ContractAddress: persist.EthereumAddress("0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d"), - TokenType: persist.TokenTypeERC721, - TokenID: "1", - Quantity: "1", - }, - persist.NewTokenIdentifiers("0xd4e4078ca3495de5b1d4db434bebc5a986197782", "1", 0): persist.Token{ - BlockNumber: 22, - OwnerAddress: persist.EthereumAddress(testAddress), - ContractAddress: persist.EthereumAddress("0xd4e4078ca3495de5b1d4db434bebc5a986197782"), - TokenType: persist.TokenTypeERC721, - TokenID: "1", - Quantity: "1", - }, - persist.NewTokenIdentifiers("0x0c2ee19b2a89943066c2dc7f1bddcc907f614033", "d9", 0): persist.Token{ - BlockNumber: 51, - OwnerAddress: persist.EthereumAddress("0x8914496dc01efcc49a2fa340331fb90969b6f1d2"), - ContractAddress: persist.EthereumAddress("0x0c2ee19b2a89943066c2dc7f1bddcc907f614033"), - TokenType: persist.TokenTypeERC721, - TokenID: "d9", - Quantity: "1", - }, - persist.NewTokenIdentifiers(persist.Address(galleryMembershipAddress), "0", 0): persist.Token{ - BlockNumber: 5, - OwnerAddress: persist.EthereumAddress(contribAddress), - ContractAddress: persist.EthereumAddress(galleryMembershipAddress), - TokenType: persist.TokenTypeERC1155, - TokenID: "0", - Quantity: "1", - }, - persist.NewTokenIdentifiers(persist.Address(ensAddress), "c1cb7903f69821967b365cce775cd62d694cd7ae7cfe00efe1917a55fdae2bb7", 0): persist.Token{ - BlockNumber: 42, - OwnerAddress: persist.EthereumAddress(testAddress), - ContractAddress: persist.EthereumAddress(ensAddress), - TokenType: persist.TokenTypeERC721, - TokenID: "c1cb7903f69821967b365cce775cd62d694cd7ae7cfe00efe1917a55fdae2bb7", - Quantity: "1", - }, - persist.NewTokenIdentifiers(persist.Address(ensAddress), "8c111a4e7c31becd720bde47f538417068e102d45b7732f24cfeda9e2b22a45", 0): persist.Token{ - BlockNumber: 42, - OwnerAddress: persist.EthereumAddress(testAddress), - ContractAddress: persist.EthereumAddress(ensAddress), - TokenType: persist.TokenTypeERC721, - TokenID: "8c111a4e7c31becd720bde47f538417068e102d45b7732f24cfeda9e2b22a45", - Quantity: "1", - }, -} - -func expectedContracts() []persist.EthereumAddress { - contracts := make([]persist.EthereumAddress, 0, len(expectedResults)) - seen := map[persist.EthereumAddress]struct{}{} - for _, token := range expectedResults { - if _, ok := seen[token.ContractAddress]; !ok { - seen[token.ContractAddress] = struct{}{} - contracts = append(contracts, token.ContractAddress) - } - } - return contracts -} diff --git a/secrets/dev/app-dev-indexer.yaml b/secrets/dev/app-dev-indexer.yaml deleted file mode 100644 index ad132ada1..000000000 --- a/secrets/dev/app-dev-indexer.yaml +++ /dev/null @@ -1,40 +0,0 @@ -#ENC[AES256_GCM,data:zrZjDI2DW7jJDUh8oR5MmlyIttOiyxQFeb8bBMsQwDZzl8abuyjad7aGUG5I2+9hINq40nx+KjQGqTCw5YNM2gwUT97TT7fcOnL88IJ4DaMzTVa0pv/AkdDkLl627A==,iv:IhaHQ2+0l9bfrl+r/KY1dwRmmMoxlDPARWSGW2f/KFg=,tag:FkpKMKkBXuseNStb7N15Eg==,type:comment] -runtime: ENC[AES256_GCM,data:AtX3Lw0=,iv:duJELTIfreBe26HcpsVQpOlj99/GXj4Lyn2q2sDKozA=,tag:FU8DAWDabh36mbuMh1uejQ==,type:str] -service: ENC[AES256_GCM,data:8txdYJ7daw==,iv:8pewpYZQUPKanWOW9VZ2A+XVogzlxjRRHHc6Tj6XOmg=,tag:naX0ZWw0aHaUR2+6G7AuFA==,type:str] -main: ENC[AES256_GCM,data:hpmfxwY01DrPeK/+vg==,iv:KFtQoezSB4PHnlr5JuFyneffM73wwjM4byf/FT4J1Vs=,tag:g9s9nmNjvpi2CUaFPmV95w==,type:str] -app_engine_apis: ENC[AES256_GCM,data:xB7LWQ==,iv:reWLPa9dFv5O9sjhajTBtn3wljkEuSV7LuNxklx7EPE=,tag:m6J1AHRJ/IVhgw9Xl6fNvw==,type:bool] -env_variables: - ENV: ENC[AES256_GCM,data:ulM1rjztAmP6AOo=,iv:/DGxmgkgLlu9chna1zB+yS4PVyqd1ItXRQp5vXnG0Kk=,tag:NMOz7SJQdJavZZjpS/PfnQ==,type:str] - IPFS_URL: ENC[AES256_GCM,data:JvdexeGrVfEmPPSI2YQl8VBX90/9zULesEUo,iv:XIOpQeUmTMchX91/qTv6QGFcNphjt3wsZbpOSjP4itU=,tag:d6AMc6CA6VTSDc6R7xw1gQ==,type:str] - RPC_URL: ENC[AES256_GCM,data:y/e1e5c4GbLo9RgnVce2Tg5LRU9vkGl7obmuykZMT1NiTT0LD2S80aAOak1oGqK77ECita8cU0Ye426Y54nj/KCPCs+u,iv:yMo+gcTT/Z1kz6lVfvyIVXnh6m1Z/XDosh9KjM/f2pU=,tag:txD0CJRWYkmptFnvMpP8/g==,type:str] - GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:LwAgUfOv7D3bH28AHJgymFHD,iv:BZ3MYWFP+wrSZxMjBa56nU4wxQ8bUpCqPxgIxWmaphk=,tag:CnBPNTJ9PFvvM4F/HD6dMA==,type:str] - GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:iSxIkZOWYE+uOIqTxI2pjU8=,iv:90Nt2dSMwDRSaCqBFbrtUn4VHdQjIsFwjLou7/nJYtQ=,tag:ztLnLRXxKnpuIan0vEGG0w==,type:str] - CHAIN: ENC[AES256_GCM,data:3Fv/,iv:WeCO1zzrX8qnFTM6FVuxMLQh7OLiu/Cqrr+YPAZtdcg=,tag:Cef6/C/K0/RJukuhlpER6A==,type:str] - POSTGRES_HOST: ENC[AES256_GCM,data:wlD0fT+zf7KzHKwkkXKaPwj6RnIcyLE2PfQxcEZyC6hMVjmmCbzxCONe8X1ixsdE6VY7ag==,iv:cWk73QbbYUVp7Abn5uxF3Oa2s66mBn8RKjmQDq+oPfk=,tag:620gkN1yz4vL/qc1VPn/Pw==,type:str] - POSTGRES_PORT: ENC[AES256_GCM,data:lzuwOA==,iv:0hO1nb0bF8QS0Acv/HNJmKTOLiVOCB1GrswvRRMI6YI=,tag:apSmzRp+EeAYiCV9nc8C3w==,type:str] - POSTGRES_USER: ENC[AES256_GCM,data:9vCl5H7qLUA=,iv:RWmZWWLC2Flnf6geanGDx+6rgzv1rwxNw4uvNj+4EtM=,tag:anRabICVfKJehtzWzF4mNw==,type:str] - POSTGRES_DB: ENC[AES256_GCM,data:dWb536SluwU=,iv:IxitJRhNi9W9Nao4YALgR8kvUosOnzwonmckisVuzGk=,tag:vUxctaIaae5/0r5qPCS8vg==,type:str] - POSTGRES_PASSWORD: ENC[AES256_GCM,data:OMC9hzQfJnJRJd0uyCzxrA==,iv:2ZJGAmSGCSj9GYCqeD5Y02nRldDDdOMmliVQASofmfE=,tag:ZRjJLhThrgrylSc2rQYUvQ==,type:str] - ALCHEMY_API_URL: ENC[AES256_GCM,data:51faEKVs7pzccffdsNAg/6i3y/O2mFJ6FDr9H00uGUB0GveIRC+v98St8Y3YwxLayXIiH1OgwJbG8Lm5w419bWuIufpR,iv:NOJjnuRJZYG+YNd4W1NFbTLBD8s5qiZARVEMJh0IkbI=,tag:WNhnOvCJwruVy3XX6sxo/g==,type:str] - SENTRY_DSN: ENC[AES256_GCM,data:uGQVPI389g603X93/85W6bAT/tNouPFaGBxz+MFOpYndsCWvZR+OpVQerab1GtZu1XWihQ65LvLsmsspRi4hStIDBX/WjAm+apU=,iv:Hlr1uKacyVy3InRWTsVNqT2O42RH7yjuZtf8E22lyEw=,tag:UrXEtFzC8hZ71JguAIoHaA==,type:str] - SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:MQ==,iv:vHMPv6X0DPRcre/lmdek0Rnqw9Yk9jQ/DX7Azh65eiI=,tag:y2pAPIDq5wrU957uPqYvfg==,type:int] - FALLBACK_IPFS_URL: ENC[AES256_GCM,data:J2geibWIdS8xo2ZO35Z5jeo1Re5UlwStzLikcmuXJm0y2b2o,iv:r6LPKONiVTtz0DAK0cRzxcFPr5Gs5JYqULewPnlbBOk=,tag:UQHROMKdI7AkVHYPNu8xyw==,type:str] -instance_class: ENC[AES256_GCM,data:YR4=,iv:0/asoZfhCondwnpHmL7D5eo2yjrExozqNIG3UdBCGEI=,tag:o40nnZXU+AP2Hde3YlW/8Q==,type:str] -manual_scaling: - instances: ENC[AES256_GCM,data:yw==,iv:026s3tZVak7NMBJibe4tc1oNx9J4lnWtx/2m4GjE0i8=,tag:NacoUp/bJISWBzz111xd/A==,type:int] -inbound_services: - - ENC[AES256_GCM,data:VlAau7uO,iv:t5RtCh3qncJv6wyZoGWUEIWA/2gW7w9Qn3KRroxyRjQ=,tag:Il+uOTZxBGZhL96cwB2Zbg==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-dev/cryptoKeys/sops-key - created_at: "2023-01-19T14:21:44Z" - enc: CiQAtzbCKaPMuhJBgtgPnQ2hdY1WEMjl6ZcYlVU3YjIvj03pJGASSACwF7kcJlwwzJCW5eUPK1krkc10uu1Y4w3v+qqDA5uyFIxUQb5lw6nXH8/FOIV27C/OoOavJCHqFlhdqTVCCL55Qspn8Uy59w== - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2023-04-10T19:38:26Z" - mac: ENC[AES256_GCM,data:2yrAYgi10wm1KepGujvDIXjNAlupCfjaj7DStvZop08L+zhPArjCOioVZHM/8sR85vIrO3GyOV8i6fZZSsUleHlhWBfDk9rVVzV6/EUAtGz240md+ntk4liFQ3BvfSJvB1Fw6kJkIp3w0Up9s0+d+NJEwGcZOqIWG3xw7i2htz4=,iv:wEePxcPYkC3WdnF2Fx8PuZV1D9LJisYrSI3X7AZhfYo=,tag:Q9fasIGdwoNChQImEnmaJA==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/dev/backend-env.yaml b/secrets/dev/backend-env.yaml index a9c483a5f..7bfb146fd 100644 --- a/secrets/dev/backend-env.yaml +++ b/secrets/dev/backend-env.yaml @@ -20,7 +20,6 @@ POSTGRES_PORT: ENC[AES256_GCM,data:V+bUkQ==,iv:hrdiiSzEulSlOtw8Wscr2bwXu2XHYedEp POSTGRES_DB: ENC[AES256_GCM,data:m6lAorFYhm8=,iv:H5cLzyKzqFETvo7Sr9em3z3fGd7xcHqdwAH/RXE/ySU=,tag:nWWqKqo0YcH1LZ5dRk6LqA==,type:str] POSTGRES_USER: ENC[AES256_GCM,data:jQ+jpjKfgSwg13pUFg/N,iv:RZMCf5kZDLXfj/sd53esfp5x4IHYkOH8aDlYEuFlYKo=,tag:0BhyIRVbGAiz2KHcSMKuUg==,type:str] POSTGRES_PASSWORD: ENC[AES256_GCM,data:wfHa8VDDIMv8DGIoUesjHObxC29r2EXvWiCm3VMPleg=,iv:LqjjvAfn0FwRUONu2/hcI9W+Z56L9gICvuq1InZIqpg=,tag:3tvUnmc8uyMrT/OpRtggFQ==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:BrT3Kb+V3sJb+2cAoehyOvUqyflW70cA2/OHNya4sEXUpR791WBR0h7LTA==,iv:1fcPi83JCUmjN4J6DhY0X3vNKIXsegf1bkwsmgv5AJY=,tag:T5G3zhcSq7CBzsui4V4Shg==,type:str] OPENSEA_API_KEY: ENC[AES256_GCM,data:I9EHo7Q2I5YngagAJ8NqofIudE7kkky20ZyPaNQUGH4=,iv:Q8qLQP0R6Zk/cbiHj88TnNZ1qHsWb7jAjnz/2Ew9Y2w=,tag:18ozU2ifgq+rQ6ZY/LN2nw==,type:str] SENTRY_DSN: ENC[AES256_GCM,data:Dcm7Tj3K3m48J9SbuXxstwov+Kp8HnQ2Cnwrvoxivz6bYqUF+zjwbefyLQ6oc6QdXlsYr0zPZd570LfG3mUw88gcFtmPFmMbCR8=,iv:pLhze2nsnN4S5VFDiSAPtVznPFnBAeqoWeSaViy1mLk=,tag:PnLEguTRE6PXkJvDWdT6kA==,type:str] SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:rhjZ,iv:2Aee0klYjmi7NRScwessVXjgwfarIpErVaWiY+cNJ+c=,tag:UbYxReAjO5hpAAo38VKByQ==,type:str] @@ -98,8 +97,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-10T22:54:26Z" - mac: ENC[AES256_GCM,data:nwM2OwfuLUUTWNLp04goXWtNSWdIKJcIsmkOZoASIrMHT/8P/rDN+iKKwnsICQ6u1ezdnlqDhKE3YH1EINzR+qrvlQuXiL2m3iqJmyCFYf9IPD8wTzlli2ymP5C9EFDIC3AyGyvA087oxzAJuw1ixBxmtuemsx8qTLU0hUx815s=,iv:TTyZPGJHPygM8guxbgnEsKy27xZznu+2Fs5COjP/FMY=,tag:VakhF/X2hmsOpDQGQYsUAA==,type:str] + lastmodified: "2024-04-19T18:58:08Z" + mac: ENC[AES256_GCM,data:pCNQfZ0TFETseAwyXzLDgyLM/8cyL7ecp6IuS9dvqAOOW8oX6RihGRBQPSgaEwMDCdVEkr1+bwgiFxI7ZcNpg9UmipeKtNX2WErnnSiTikElD9L1vYbqDnduwYWR8Ee/G6plu2eti2azGZfteFR0Xstg26ELXmjSgCZb7BeiUj4=,iv:Um5xSV2C8tnIv3fBwe76p1L1zC8z3K5p3jzNlGEqYYo=,tag:JSdOQ5b4fvSy9j6m69sTZQ==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/dev/backend-sandbox-env.yaml b/secrets/dev/backend-sandbox-env.yaml index dc848beee..68a6da3f6 100644 --- a/secrets/dev/backend-sandbox-env.yaml +++ b/secrets/dev/backend-sandbox-env.yaml @@ -20,7 +20,6 @@ POSTGRES_PORT: ENC[AES256_GCM,data:6IO4AA==,iv:ON9OIsYdRFnUs01GQQ32fNOxhUzgNlSLf POSTGRES_DB: ENC[AES256_GCM,data:VRYMXfoMrW8=,iv:hP5OPkv8l7NxN8rORVd8eW/8sEePC20mV235o2Y1BbM=,tag:3r1pcewPEF1KKqa4KD1PAA==,type:str] POSTGRES_USER: ENC[AES256_GCM,data:YzU1iMRcAK/dsfJx5J2j,iv:DRrps69Pun2Utx8sq0YBy0PNUe4L5fY8juq1RqCYB0g=,tag:O3y0Z+XiOIUL3KoH80JwxA==,type:str] POSTGRES_PASSWORD: ENC[AES256_GCM,data:8h+HbK9ntx2Glt/ml3gXfDcN3n+uhMXu7FUrzK8RHdg=,iv:TbGsCOiIe0cZqDSre2CxTUxQ/naiokc93belnVL4zbo=,tag:dePWkKMtDZ5z29drK1eeuQ==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:DnuW8RW1i+37RbRUcElQuxSYHsP430Ez+NTob8Q0aXM8A+6Uo+G7oNRNKQ==,iv:glWQvZY06dyXK19UVTewwXkF4vURgsaPIZhDaf+TglQ=,tag:T+Qj5kYv9MQURPlEN0EaGQ==,type:str] OPENSEA_API_KEY: ENC[AES256_GCM,data:eFQo8DIqNVwMlylCMJWEFN8TgNZf7j7oaPYJm5t7NQI=,iv:xRa4vc1PcGQEWHnyR8puP99au67JHI+uHY2BmnDAZtI=,tag:CQr1Xu0lMljH4seGw/Tmmg==,type:str] SENTRY_DSN: ENC[AES256_GCM,data:vxeO/GyItJzahNZUYirKJ/CAFNsJzAR2225MlbWGmG80rXUsfqoUjYydGbd5T4OPKmSMzLvnMfH7Y4FisND3Dzr1iNKZkQqWmMA=,iv:4DRZ6a76wTcOO9DiZLiZQel2aUfJrYu1AqNmTahm64E=,tag:sNaNtI5hIEtwypL/JEgruA==,type:str] SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:mHXj,iv:0aV3RyVHvV006JYTtyxnu8SbR7IMfJf4hDSRYIhL0ZA=,tag:upziwWB+PKxFs8KwdCR5Gg==,type:str] @@ -96,8 +95,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-10T22:56:04Z" - mac: ENC[AES256_GCM,data:u2RGmpD4ZdN+9J2X/JPb6FoIqnkiO8DBiDpot1ATMZNSUlDG9r8nqN/hI5azjt+kBtn/NWWbsyYAWXf81uW5uhgBzsO5vmLNxi5DVbW3ueys2CCPJ39roAy04zAasvu+XQIM+mrYRPc60gPu+PFSd5eWHKpNV+3U1+vQsLo43DU=,iv:ZmNIhj/03iz1CGjS5BY2C8x1lXKMX4c3kX+y+DhRTiU=,tag:VK7753a7ZtVqPVDKENKnOg==,type:str] + lastmodified: "2024-04-19T18:58:35Z" + mac: ENC[AES256_GCM,data:qgf3qBwr2nThg6xSGBO6Wxkb3mElC4UQIVRvxhmrlUSOjHkcBI4oC5Px7LlSmUkwPE/llQVjWs3OYdZDbrPb2N3PS2zWasDoRglZqJ5qzudk3PtQeR5wf7+zNHXqQJEzJ/p9UYeLMw8jOI8mwSyRVrEvdl2FSIN2cuJmiIwos3s=,iv:2caPkBfpqeh2qcRLqp09FYMpCbT3F/NBdxDkD/rnE1g=,tag:Z316/29lhUd/xw3pUx7BzA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/dev/indexer-server-env.yaml b/secrets/dev/indexer-server-env.yaml deleted file mode 100644 index 54433ae6e..000000000 --- a/secrets/dev/indexer-server-env.yaml +++ /dev/null @@ -1,30 +0,0 @@ -ENV: ENC[AES256_GCM,data:x713o6h1W53W62Q=,iv:8tS16ZsZI/rhHV1UJESQRNCT9leIorIAaBunRiAQCfE=,tag:G3yqms65Mv3J0iY3MsHhNA==,type:str] -IPFS_URL: ENC[AES256_GCM,data:nAiCoX1RERpSIw08uHiYoojN9bzyTh9rafzf,iv:RQZua2UkkZ9/z9O4LhRJGfRxRa4RtdmYgcvjF36DulU=,tag:LnKbKBK00a8TAKyvJesMCQ==,type:str] -RPC_URL: ENC[AES256_GCM,data:ka/0/kreSXdZlXjfz3y8yBLqNpR4FMHfFGewedAIrpIa2ilvc602M1Buat6r0VTzpgfkpVyQMjXPkoSNsrCY+wp+2A==,iv:UaQK4dxwjiewKNgLKfh/JSw38CyPC3cUFvi/V7ev2JA=,tag:7Jo7QEr2rnu5T5TNvqnIMw==,type:str] -GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:PTq0HcFzonrWUx2Qepzstrjk,iv:gIquB5nRScEeqr9+Wowzru8P4pxQbjen7iEsPjCWywY=,tag:6BsQ88/AxP61y8ZcCCi7VA==,type:str] -GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:TNP78x4eK1y919a0wDtlobw=,iv:YeeC4ZLWSC6wdf52tDphDwveMaNsM5TrWTqqiqOZzoU=,tag:yw//lkuGyWl7t8bCjm4anQ==,type:str] -CHAIN: ENC[AES256_GCM,data:0i/R,iv:wFUiCafxuCT/gT4rtYGOg64Xq6MYvL3CG0XuuGGqO6I=,tag:91ZG/pzHpg/P5OaUy6TZ/g==,type:str] -POSTGRES_HOST: ENC[AES256_GCM,data:jZZIdjdElAyRx7ZJCUC4cx+RTnfw6KD/0vg+YGF3xaDD3wV23Wr0hKBV5SDWfLjdMYJag9Iy8MZnHwXK2XOHyw==,iv:gUqRrd50HwQeuWb0otEoM5Zep6yfw7zYBhna/SVy5Ug=,tag:yGKPmbYalm9BCJxsDrG2rA==,type:str] -POSTGRES_PORT: ENC[AES256_GCM,data:22J90A==,iv:VkStC9zFtClk51lyBMS5WvUesBI8HPAECMSEaBDFRl4=,tag:mSO5gwCcF+5QN6mfVQ1GzQ==,type:str] -POSTGRES_USER: ENC[AES256_GCM,data:z0bO8RgY1T0=,iv:MnIYO6Xif9EaRVUJwGPxaSnSVEVcAkbY1BHozN2Hyu8=,tag:Y23l8zQ/3oxt8h2DINdWdA==,type:str] -POSTGRES_DB: ENC[AES256_GCM,data:XkLjLNuRbBc=,iv:E38x7066fwUoOjoZV3z0JyuIR5Z+b48/C0SG++9wXjM=,tag:5nJWLerX5gqGYtmuzeUoSQ==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:hRjw6aRQ+OMMUZUa,iv:EGJ0K12zPsms9LV4UQquMDaR2/VA6Ecc/0m3WcK889c=,tag:ChCN8wkhxahCn1CoEk7uyw==,type:str] -SENTRY_DSN: ENC[AES256_GCM,data:2kBK7h9UAkx7eEZ1VpKC5eJAPwt8xuUooFzr151zqDRCCtxiEUV0gifN6jt2jJarJpwqN4hlcLUGr9Kc3VfkNriANWYAUp0qAGY=,iv:X+Je/pe9CgP4pnJykNthhGcpvVt9GDwh9Q+RKQ61FQw=,tag:AW9+n9iwoJ54VMaf1zK7iA==,type:str] -SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:yaHi,iv:xgfUd+WMRc/96Oa+e9DbFqGAfs6x4bTVbo479moDjS8=,tag:I+Fn/76jmDCSfOetLg+0Yg==,type:str] -IMGIX_API_KEY: ENC[AES256_GCM,data:GXar5/UmOXOk7JGs/uQcEQuKh40uzrgwJyfQrIMsmR+3XxwyW6N/uXyw+f5xoeI5Cl4RaHL7D3w3I11gXgVbgPckqg==,iv:49qfGkgwTRzSHFZGoJDsiMDR3gO+oK61kc06357LLVg=,tag:wOl91Fea9QB7NfZfsNMm1A==,type:str] -REDIS_URL: ENC[AES256_GCM,data:TGJn5EnwOGy6rhs+Rwi/PG4hGA==,iv:D6KRqnha2vZlMD8TVgNZq/09LMrlAALq3/pAxpHyKSg=,tag:jVghWAUESkRRT3dxGzfdmg==,type:str] -OPENSEA_API_KEY: ENC[AES256_GCM,data:H/X4HZjH0x/IAC9VXv1K3mjfKYqvRNZnAf9wHnREsGY=,iv:QssqPTwc0TKxzDQi95eLj8rePQ9ubfqfP+LQBn9mtYE=,tag:ky329b7xGowaFxJ50Df0cA==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-dev/cryptoKeys/sops-key - created_at: "2023-01-19T14:22:14Z" - enc: CiQAtzbCKWL5wPQU0zF9tcZVzdOZi1D+nzVEL6AXyrmT6k4FYVsSSQCwF7kcGDUPN4MwPItycOGmlBV7wRIuHiun4SeeD22MGNQn6HCYDNqcPdM2Y4sYy/BWv6MkKAyvjwMu6SrP+w1tuPKOJBsaGgI= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2023-09-28T14:43:40Z" - mac: ENC[AES256_GCM,data:p4E3eOSw55PX2H5DODJ3HfB91E7XHCCokEgwlRRS/T5x4CpAxon7KXCAmEkQ5nzvQNQlJXtWmXJfih0ZGCIyuqul0n5dH515rQFOQ+LCvu9UFXw36b4dek+dtT4oQwsRWkBDmdcVDme953x8hiD5QTtgpP7eTpq6BpkeKc/ZB18=,iv:UfaaJ55LPQr+ks1a3DBPTo7nLd/2bPF0PQQdSZx8zk0=,tag:F4w2egZg6YWnJPpOG1QovA==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/dev/local/app-dev-backend.yaml b/secrets/dev/local/app-dev-backend.yaml index b0c372d87..c0420f9cd 100644 --- a/secrets/dev/local/app-dev-backend.yaml +++ b/secrets/dev/local/app-dev-backend.yaml @@ -10,8 +10,6 @@ OPENSEA_API_KEY: ENC[AES256_GCM,data:7iuBV6YRagS0PIz2nPsNDU80es3bCFX5oLYt2qJ49l8 TEZOS_API_URL: ENC[AES256_GCM,data:rRX5EW2nMUI+Dc+cH/dKLIqDGQ==,iv:x+I39Zw2MuliHE/XXWEpIsXrcPALl3FGeBKlTnQWb1o=,tag:BwINanQuhCI2mwaNcYjVKw==,type:str] TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:254mQ1xI3t6JUzPthzps9jZbwVBGZ0zGnHJpRqG5QCo=,iv:Jr8gE0ke5fX6XEyT226Y7wJko/x2Ng4t5jq0Hd7A6l8=,tag:w3aG95+2MXhg+nXvANNr5Q==,type:str] #ENC[AES256_GCM,data:zwgPg5o8EPv7KbOu5EqFHugDrfOLowyKmaUEK8pP+aGvGbk4gkNJlA53e5aMbVYvGf4ZiVjCaP5xhKn2S+4sn3tIqfh8maOT0A==,iv:CDILhjibyqjLV2a8CX7CR/S6yv4Ged0RNxqPVOIfZX8=,tag:jrHScI7Ia64NEPRWi8reqg==,type:comment] -INDEXER_HOST: ENC[AES256_GCM,data:AMSoNna+FClTDLT2p+9jVnH2qH9M,iv:vywl4L+b3B1WSIIYPwKzRMSRn67boCM531P5dLKw1Ls=,tag:y4erUfhOns07F4RqDLafGQ==,type:str] -#ENC[AES256_GCM,data:kxDk5XyPsjJvjeJCrLoxZ8W52B7Wk4NgVQ9aJfOM3MBvOkumXBstjLTrZZCd/zbg3PwVRLeF/ZYISA==,iv:Ik9VWuJvIgVobthlV1mKTfCBqwqCVDi4UOIJ/4cDNz0=,tag:Xe49oB+2mypS+DbEGh9ErQ==,type:comment] TOKEN_PROCESSING_QUEUE: ENC[AES256_GCM,data:L7EdI/IY0tirUA4L1oUy/IB3WUx9LcYfyoi2+yAQuOwBvdAZJnXgwgqRXQ9KdmOMJJFaXVPcbLXk68bwACkbYMAq6A==,iv:lTE4K7RYhqoj8pMK6QWnk/ZH0agOmU5xFo7F29OZxDU=,tag:mE4hi42g2mmnyjxhJzUvJQ==,type:str] GOOGLE_CLOUD_PROJECT: ENC[AES256_GCM,data:qhY7T2h7ua2oKtjsrg6aJ07h,iv:17NFGUjLPt7rH3TQ+4JM0Y91Y7ZiWuBRDjNkLeIP/3o=,tag:BZOB9GoHI/wzCdisXOlZJw==,type:str] PUBSUB_TOPIC_NEW_NOTIFICATIONS: ENC[AES256_GCM,data:HVIKdVNcmD57/1ELkCs6Z9LU7iJh,iv:tNYt+sbEVO+Lq8wYJv2UMMUWyTVau/ScwyhLm6fzQZk=,tag:0x9fGGkN9Je8kj9ERvWJIg==,type:str] @@ -72,8 +70,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-10T22:55:03Z" - mac: ENC[AES256_GCM,data:BaFlWcxxL7NhkqTE2Ghc9OCpF+hMDEFTsNX9qbT3E7h4yWDwzYShrURg9TYefPeOfOfo3TfRcqtuG20yR8JGuqvLRZM5ADQIbVn2svDnniW9VJAPTn1HOG0oXtgrxFkNjNykgIVr3RabKpgmOqXHSIOPQ3Awg8KXrVQB8OKWaGo=,iv:BcoPjuVVcoUnJiwOYBLLRAMRxNhvJ1+sUZtd43Fp/kQ=,tag:pw/idTljMg3eUBnVlMCTFQ==,type:str] + lastmodified: "2024-04-19T18:59:39Z" + mac: ENC[AES256_GCM,data:xZmjN4F29GW6KVkDNxTRiOptusWgXVksBKxsFYUd2K77+v4pbTs/6X5fYkPEVdTuhL4pbBfntthYfPAoQqTnueeCQXdoOYfgfAVpqc1A1pHOd/Fvxy+coOI+5aaV0ze2bfU/0Mb9NRxNIyACYQ/2bem4bxPLJkOEIoWoxaudEXM=,iv:5haTAntY5rm3OakEWtbxpUc7xvkJ5Clqf+Yn1rKMrew=,tag:ge/lMCfq/GCbzv4jnfgHIA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/dev/local/app-dev-tokenprocessing.yaml b/secrets/dev/local/app-dev-tokenprocessing.yaml index 82634a94d..f73728390 100644 --- a/secrets/dev/local/app-dev-tokenprocessing.yaml +++ b/secrets/dev/local/app-dev-tokenprocessing.yaml @@ -10,7 +10,6 @@ POSTGRES_DB: ENC[AES256_GCM,data:nEf+pFTAhlk=,iv:Tz6K3k8rqAMV7mFJyiHBmjjPVA839FV POSTGRES_USER: ENC[AES256_GCM,data:eQ/HqIZjKx9y1QGwUZAU,iv:h3mEWFmfiZY0vfcE6Xba//s/h2FsyujIl5G9T9+FXmY=,tag:uZKVRPxCTlCULOOd4s7iaQ==,type:str] POSTGRES_PASSWORD: ENC[AES256_GCM,data:Usi4/eSc889GPD3Vk69HuBM317G9SMG5hq5VOwO2sIU=,iv:3ZrWp3eJBlwkVtXrWCxwKybtxfBzRoGrlZLx9nKZn5E=,tag:T/ixt0+WcmLKbw1aAmsD6g==,type:str] IMGIX_API_KEY: ENC[AES256_GCM,data:jVsVr44tCUYGW2X+gqmJ+u4KM4jQTvEB0t5EloiIo4Oo71SEZwN3XWnJjL1p6O8IWe1qTYPihMwL5+BHPp9LWgShvw==,iv:ddBBDWwxN9ZxrTjU8WQWX4nCse4ImpE+87YorRU/Ups=,tag:Nvf/lsWiWnXQKLCHHR1UdQ==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:eTXaGaO2mqUl3Q++8ExkgrQI/sny,iv:Pd+y3QpPMytHeZ9pqw9YvWeX6PBvE23FG6RxLC0MuTY=,tag:l/Aq4N+L7TkCfLU8BEL/pA==,type:str] TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:mSrwzJYhLF+4I9DCsonspBVTgwuU,iv:FNZofq3XqM13ojMhd/v1n5+nmJK6BTvyVd0hwN/los0=,tag:V6Ybjw7J6DLIKjqmwBI/Xg==,type:str] OPENSEA_API_KEY: ENC[AES256_GCM,data:A+ch3Yfy6gf1IdlFvuejdFoA8dlKXIy8GO+wEsYJ0YA=,iv:OgITHJtuWLOlugmbkWEe+jxtSBcMis/1kdtM8CKxwfM=,tag:jDhz/vbp4AHSfU14ANQfng==,type:str] ALCHEMY_API_URL: ENC[AES256_GCM,data:9/m8cwYlAeu2pb+PtMMr+pwNrXwmOTJ22nx7/Rxx9bj9YPTHw9VRGxHRkJYoWt/DakEftgvpN2QdBKCVp1V8gJL4de+W,iv:/FQast/kphRXjUNZlUK1JcLD1T6/ibsfzDR5RqX40ug=,tag:eycuy3SaSVTr+t6cpQLhew==,type:str] @@ -43,8 +42,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-10T22:55:15Z" - mac: ENC[AES256_GCM,data:gQeX0IGNsdKDTQxe2y/orai4EBU2TTd3KIB0W9g1GnO7kWiFljDBccfEVF0DYv3qN0xZnT/8oswu3KsNurH+nndi8heG1Sw/EnU1Ao8aS9MLXSS2UutiknrUPRr4H4lCnRhcpwb4EzDHQZGnKef2KDEBehzxXUbcTyeGgt3xLK4=,iv:G366S7XUOa8Gvaqsc8l6CEw8yhb5Yp7JQiznxhGnhMk=,tag:FgTKiny5fW2MjAkwVF5D4w==,type:str] + lastmodified: "2024-04-19T18:58:56Z" + mac: ENC[AES256_GCM,data:tvfCarzGYIVKgL7PiYwxtIYrss84bDdeXd9zqwOThMK4IMR/p4YBNy32Y0av1/RaMVTA3sSTk1buJSKGsvdsuPjwJrpwPvKs9hqV9achvy5zG0rgMsMo201h5VuXeHvB9LQE2Ex99yOZ6OBQtRYpSiNzj1hd7udv8Bt5iAWEC6I=,iv:U12anhVQ9UQ/6kPga1AsDMIjOHPsL++4YfKn/U1gHTM=,tag:6wapXjJikyXWxV9yq5OeLw==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/dev/makevars.yaml b/secrets/dev/makevars.yaml index 243103069..3d8867f5d 100644 --- a/secrets/dev/makevars.yaml +++ b/secrets/dev/makevars.yaml @@ -24,10 +24,6 @@ ACTIVITYSTATS_TIMEOUT: ENC[AES256_GCM,data:AolMyJ8=,iv:Bi5tjosXYQDRox358v8kjF7Jo ACTIVITYSTATS_CPU: ENC[AES256_GCM,data:DyDiERw=,iv:LsxlPKjQOnIBGEy2f2cp+JqZTlkygIB5Nnotv3O0iNw=,tag:bmvITqwBuF1pPPd0ZVQdQw==,type:str] ACTIVITYSTATS_MEMORY: ENC[AES256_GCM,data:Kb5b,iv:kDjeQeSuSP1tCUF1RzyN0aXr28ccu1B0xMqZcRsJHM4=,tag:tBT8C8DHdGeYtioW4vG3zQ==,type:str] ACTIVITYSTATS_CONCURRENCY: ENC[AES256_GCM,data:pQ==,iv:aIhTkijpX5UmdDafVvuSnwakinx/+VllI/SBPmyayLc=,tag:kPH3GZ+BKcYC4U1x6qrgaA==,type:str] -INDEXER_SERVER_TIMEOUT: ENC[AES256_GCM,data:9eUyA6Y=,iv:+GyPL4oJdxtT8Gmses/gNqEDUPG2m8ungWfAVLN0kd0=,tag:harxeciywOKr9uJC65quTg==,type:str] -INDEXER_SERVER_CPU: ENC[AES256_GCM,data:Kr5wzQA=,iv:EFNf4m/lsLyY78YTxa9NU6VqI7W63C9ibt1KotvEZvw=,tag:Yj4tUz/ltg3/ON/dXNLtrQ==,type:str] -INDEXER_SERVER_MEMORY: ENC[AES256_GCM,data:/IA4cQ==,iv:E6Qdox68WpFC9P02MiQiHOVDBVlCX0+YaNpzwgAlKUY=,tag:wjoqDLupFsoVQ/h75mEroA==,type:str] -INDEXER_SERVER_CONCURRENCY: ENC[AES256_GCM,data:+fYbUcOxPg==,iv:zpUb2RMMA9krSy1VBVDo3XPCDdhqaYui3iIplYcMMN8=,tag:F4OaBBE+cpNt+az+xuIXzg==,type:str] EMAILS_TIMEOUT: ENC[AES256_GCM,data:hCc7iMs=,iv:dh7styhihwl21kCPVyCmvByrl5mOZICxxLjV6IOm9u8=,tag:6DldJF1ye2FyZeQS+rIBTA==,type:str] EMAILS_CPU: ENC[AES256_GCM,data:pPK9xsE=,iv:ontNMyPg63PPerExj1dKnG6WkxCjxChSCDxiUbsh+cY=,tag:Sm/HQ4xGld0Dc2AOt88P6A==,type:str] EMAILS_MEMORY: ENC[AES256_GCM,data:1gOB,iv:lUQ9j00ooIYEX0+uwsRnFpBlN44g9E1syogpmnIH2v4=,tag:yZTzqRGeaHnnLJwCDl9WHQ==,type:str] @@ -66,8 +62,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-01-04T23:07:54Z" - mac: ENC[AES256_GCM,data:gvVGxLwuFJMJin7JxZlxQdNQ4j1R0AcE7QAT/MFuGxtZU9bau5zAoJik4TbklSnMdB+OwcEJFdpac/zYw7rnnfNOJXLvOigvuSYYu+GkS/G3yCLstXUUpFeSPDzcG0JtkygCxN1aJDjUc12mV6MKRVyKlA3FTkfyKFnc5TWf9QE=,iv:zX+sz2Owxsgce8xSiuDAuokFzAND0gieS2eFGnjNDlc=,tag:Hk1CxWV1zgM4GZSN4KMQiw==,type:str] + lastmodified: "2024-04-19T18:55:47Z" + mac: ENC[AES256_GCM,data:3tAwCl0bZNkMx5cUq1JZ/yO4ltMUk4bD40/8mUi9vHenWlaSgWSjjUeACECldLD/6H7xlZ8xONRMLTMNlAeRtcY3cbI9CSNCA3tr8jqoG6L8KDJcVTSrQHn0Tx+Sl7IONFTUh3HgEwUAr2sYAU9Fgw1OyojLavekxD+jzblPjBM=,iv:E7lmQz6ezp/q7qyxJ19aBsqzvoT8ehHzi2N5/cvtlWw=,tag:0Et4qX2bj0HVHxqmjvr5Ew==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/dev/tokenprocessing-env.yaml b/secrets/dev/tokenprocessing-env.yaml index 7b3b199b7..7e9db7cd6 100644 --- a/secrets/dev/tokenprocessing-env.yaml +++ b/secrets/dev/tokenprocessing-env.yaml @@ -15,7 +15,6 @@ REDIS_URL: ENC[AES256_GCM,data:hQVjQxv50Xwdh3fYVL5e8w==,iv:tU18ROT11rds8eOvL05n0 SENTRY_DSN: ENC[AES256_GCM,data:MPS5mhBz9v8xfyNhnoONetzMKHeh1bnH7hGj7r20qQ7ide9a0kzvooRSsHbOZCUZ5L3v/KHbsyEDQ85yGgJJOfJXDxh9VnyzW+s=,iv:psKPFKjelPNA17GNXbVaLIxk7URbCJF8sk0duo8uxts=,tag:CozHS0m1rbOAN/f2mxNqTg==,type:str] SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:IALU,iv:ajKrV6AqJcskOF6xBOo9KFoIs5cI0AQynzSxk4xPyEA=,tag:PTKlrgz/190sN65b5fffGg==,type:str] IMGIX_API_KEY: ENC[AES256_GCM,data:ct2c7OpEutkpfe36Ur6VQWFqJgpciT3C/rbQwpgM5K3LC9DArjXOYgQBWG+CO8Cmv9m4BwCUlCFtUIxVdPkRI/gEbg==,iv:AC0vYViGzmLsAUCZTKT40L+hJC4JdyxP6dsKDpKkZq4=,tag:Py86Xq2hGhR0qjS2acTDSg==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:rePy4ucQ3SXtBHRv7VjIcYQQQID/npWwk4PlZ4cI90P9Ts7yDTkJYFgyuw==,iv:zdaSdV5dud+qfPMnbGagk9CwSSrd2pvUkm1Z+H3PmHQ=,tag:R+V+rt5NI6Mc6pD4ndJM/w==,type:str] RPC_URL: ENC[AES256_GCM,data:JGMPzx13UEcREf9jaZxKtLxUrcggEgeLMzSddyxgL93JxJEQm49YjU7i2WP7Ss4D+Sgs9GHUvy8e0xHUE+zzdrrbs1iI,iv:ZSTkhjM+2CqlsEEQuBTAdNspbiiMA9Bjz4RYIRu3Gb0=,tag:lYbHmPyVOoG6adw5coqsQA==,type:str] TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:xNlTxbycxlaN64eIL88GP7IxGOMBs8OZFbbV21dGZj16fTM8riEc6/GyQImQkl/LoDU=,iv:eDmycWCBzVevvTLLLmXbchjwmbqpil5iJIm9vvH+vpE=,tag:4JbFH2PJnK8Xw1mqGYnxMQ==,type:str] ALCHEMY_API_URL: ENC[AES256_GCM,data:YzV7FSm4kgHkHa9HFx5QvO4gexwHD9z33Rbyd0vV/YOVOUf17rTyxZN8NpErFEW5FSAVaDG5pfMX3WDIGbBVToYEWT8E,iv:8RpradJ0s1n6ZvgkTo1WFhvmzNEkbckJqvoGNGz1Y28=,tag:rXbIPlS9gCm0ak/TynRfgQ==,type:str] @@ -60,8 +59,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-10T22:54:52Z" - mac: ENC[AES256_GCM,data:sR/umYk+iXJm3yFTHTThGIRjJT2G8SPPQ6zO8mC7KnC/r3QxPznl/rRm+TaLMTKQGJh5ZYAC5+Q2YdqjdTekX3AumHFm9m9amiJKwgclBm1rS/r+boeu4twgJraUY1swvfdoG57oREMK9dRLPV34PfzwYU0plGJfplhawLNgdlY=,iv:N8rNi3OjjBlUiISRU2r2c0N5syRDZiDme0MAN+wSv0I=,tag:G5YMrtUMSu1Z7tkodYzJJQ==,type:str] + lastmodified: "2024-04-19T18:58:18Z" + mac: ENC[AES256_GCM,data:veg+DHFxQL+AuKOUzG9LrDuVt9xi5PHj4ARLJJDOTQRwWWow7dTcrpMHoYvjaVOPMeGKKpHGdJ1KxwIKPqFrfVQohRP1jDzgf7QoD3ZOeU70mlkcm6vPQFup9WkxQEtUh6jVSN6aQOxecObC4+4jeTLyfSwI+n7zXTXgQ/cLs6Q=,iv:sLOg8FM3Yvv80W4Zk/uGeNhwV4WIU9plbpJmxwWVj7o=,tag:4xAGmrPwS9balD8PPHA2Lw==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/local/local/app-local-backend.yaml b/secrets/local/local/app-local-backend.yaml index 5567b1136..d195050c6 100644 --- a/secrets/local/local/app-local-backend.yaml +++ b/secrets/local/local/app-local-backend.yaml @@ -6,8 +6,6 @@ OPENSEA_API_KEY: ENC[AES256_GCM,data:8PgqNL99yJI876DPJafZP5eZ7ePdZ0XFrg/Bro7spJA TEZOS_API_URL: ENC[AES256_GCM,data:tcxjJaYkJJg8kpeyk16hmsm8Hg==,iv:/c1kkO+f9MoZ/R9f2/eIjws43DZrZuUacdlAXoLyxPE=,tag:3RKW+9UTn+w7jskX1XVnFw==,type:str] #ENC[AES256_GCM,data:bmAjlTFOCONaZM1Rwf1pHK/sRU5PD5q2gXpFNeljvMoqb2EWOoCdWSl82ho=,iv:xPyGzHyhNFllbvIAF2/Hp5t2gxrYM/MhH9R1oByNOo0=,tag:X9IRmvJzeYTRom0rd3q1Qw==,type:comment] TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:V2wxNORvssYs/WFCNfX5f9SUagO6syWcKJeUZOckGlQ=,iv:sxCjbyQxpHCIEGsx+iv5lyBMuLGaOvb4BMsNMnjx//Q=,tag:jsxgxIt6kP+2jJgrNFB8Ag==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:HEtCfrFvXbRgV95IGqLbZUltcIYP,iv:XTuPlrO8kiJHXL6lLSNA314RFuTqrv6XkmKDq4A7LFA=,tag:HlMh0zbYk3B6reG4xZeJqw==,type:str] -#ENC[AES256_GCM,data:2HqipHNtZWtjh/sh3QDkUJqyvlPlmlnBRwcZ518E/Jjr0kfYeOXkD+CRsUQz6dqH7HB7g8yGkV+oIA==,iv:Dc3m8Jh5Y1zZwxmCPSEGAMGqHoiOI6a08uDZrnpe2DU=,tag:8GCJVzR04NJRN6KuZCxX6g==,type:comment] TOKEN_PROCESSING_QUEUE: ENC[AES256_GCM,data:lvnKsrbGNn0DBYKYAMFZsgTNfyhJvqWteOt5d3AptFZ18hgEsP3azXlwZokI/1ZjK99/msM+I6uAlP3OUg==,iv:myY5ogTZVsQ8ymwXt5vb38fcdvuIMrdd4CoJ7z1pLpU=,tag:yiCyVuGTiffS07R6pKeaAg==,type:str] GOOGLE_CLOUD_PROJECT: ENC[AES256_GCM,data:bE370XUNzn1jNtIipaikC8Ng,iv:U8E8nXrvnuyUU/rDO62I9EHjtI1Hj3KULXdckhViyc8=,tag:I/Jskhzabb7M1S3hx3HGSQ==,type:str] TASK_QUEUE_HOST: ENC[AES256_GCM,data:dSDrAnHltaGLZk3yMdQ=,iv:IRiP5XJyHW5uBWOTOcadlxTxIZik7NIxhnhIAx2lMtQ=,tag:oSvNPdu+QQXQo6wuCz986A==,type:str] @@ -64,8 +62,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-02T04:31:58Z" - mac: ENC[AES256_GCM,data:l4Sn+n/6gwoH1+V2bK77XcznxwSuHiRzsz5luJ6whUmgVDrJrwDkij2snH0VBiMv4l70erKVZACLC69CW8YEn7AMzxw6FY/GOa0ZMLoN3xTKa2FTVmpb7qDX9qiuJcMEnXU4ew3qEgL9CkV5DPZOAwmEdG0RPZcgtCrvXYk4RyE=,iv:+R+5mV4w1wJ85WDpeuG8TMPvbK4W6r8F2o60u06rLp0=,tag:5WzrdzHSKmGGMHeqv/Ccjg==,type:str] + lastmodified: "2024-04-19T18:59:16Z" + mac: ENC[AES256_GCM,data:AvB7PEftNvl8BqswIeBLAv/cLFt+V+nuA7MTrfi6vlCOqKSr3Szl7Za5G1ZZX1QaFczszab6/hLzM22lY0/0sWvGqSw6dNPkwfrI+MR5jx9WnA/O8PkU0ZD1pgXBhyuG5HHLMDRZ5dUoJ1b9D3uGkWTs3tKP6B7NzZ/Y0Q1DRHY=,iv:JKRVWUuuc/ovWf6lYPI3VG4fHljyux2tgePvM0sIpaE=,tag:PAcer+A1t558Rj+EX19JbA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/local/local/app-local-indexer-server.yaml b/secrets/local/local/app-local-indexer-server.yaml deleted file mode 100644 index 0e94b9e11..000000000 --- a/secrets/local/local/app-local-indexer-server.yaml +++ /dev/null @@ -1,20 +0,0 @@ -#ENC[AES256_GCM,data:8zgEEHBLWgSSWwbZjxNVCM3FvmLR9gPdgA0WefIPzIsoWioeTUoVVILBDkAMPtF/aw==,iv:E2rtefZaRAEkqiJZN3cc0hr/cUk1HF79IINmR+O73Gc=,tag:Ufi4/EEqzkxbvq8ksTLYig==,type:comment] -RPC_URL: ENC[AES256_GCM,data:5wvrWfg4kHA/vSHbEBQTPuUJgHu0IMmrHah894Wh0cPfOvmLRo+YhHhf9Z9OeKa+uZYVQY88KBUHFq+CmcRIJu1H0A==,iv:UD/9JWGZgCBISbhmda72gxdUP8ApT7MoeQAu+Lb6eLc=,tag:HV9haJAMTcRpLEG6gnLYow==,type:str] -IMGIX_API_KEY: ENC[AES256_GCM,data:xeBmuCi2hcY6akGUqRyotYmQV4+un1k56GmWRa5pVbQToAzAkHkwEINUeevu02xVbvIzHqaGBPne5KHib1zTuPtd9Q==,iv:pVNrkQtCmPppoxoo8O5jNxMDPpvThpdV3c2ZaIkfbCk=,tag:lFWyC8N+5IN6FL2XhWARuA==,type:str] -IPFS_PROJECT_ID: ENC[AES256_GCM,data:gBkA5TZiWdk59RVjZ8cDi/xTg9DCZ8TIsSBM,iv:o0Oamk/i/CDkxTXxSG5Fp5awZqIPpCgXc2217XBaaGQ=,tag:AQE3sWkHmKKgkQI4fFtang==,type:str] -ALCHEMY_API_URL: ENC[AES256_GCM,data:7TotMkl1896tcF3N6ohmrOC+x2Bu3Lu/cbtzKTjJXg81Nf77wzFrbeNTnO8XWJJI5im32NP/pNHvait9JddsgcTy9WKZ,iv:6Uch/+cGRXs8RJoKrASar1i7rB8/B53NVwWhxDrMcoM=,tag:r3kfdzZZwH6DxUNjt9+7kQ==,type:str] -IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:dqdtL+KL0Kfb1ahqNLECV+Bt6brdHTYsg+krCxKdKU0=,iv:yZCgCU05kJRhqUbv+BpHKu7MWxYH4AIuk1wkSbh/Sxw=,tag:hffGpFBC70NGTaSS6ntSwQ==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-local/cryptoKeys/sops-key - created_at: "2023-01-19T14:50:54Z" - enc: CiQAKcDhKwE5RCeml8LZil+kHjQhYAlglAjIeOL5rcrLbq/zbnMSSQD1eGvVVDLdG+s8BZsXun2QRhW9QOaO7XiQ+ctGtyQZYDdHQtmY2s8a7n5hDU/mqwt8OKszSejLcUVfPg95WhZudQfyjcs9HDo= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2024-03-07T19:11:24Z" - mac: ENC[AES256_GCM,data:Epdln4NfTkvYcr3dk8ULT9fnL0ivQ+b+nXthGWK/ne3/FJU0CwfmEGLKt9G7ctbUG0nU31+X4QuIPV47PE5Qs5FBi7kF6MG5klnOf5GngGkDuNOXpSo1Fw6WOE7lqjRAFT5jabN12K7EX8vWhIcKKzqRlqRQuPuCWQ0VLnJQPDM=,iv:e1GL4ppivXyl4ms+ipQ4IKT6N0TlTJGwOJSQUH2HSa8=,tag:XyqUBW1xwLjO6jXE15+Zcg==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/local/local/app-local-indexer.yaml b/secrets/local/local/app-local-indexer.yaml deleted file mode 100644 index 0eaa2e59a..000000000 --- a/secrets/local/local/app-local-indexer.yaml +++ /dev/null @@ -1,18 +0,0 @@ -#ENC[AES256_GCM,data:D3AQBa661kIGgj01GPvwobbaAp59ckg7drNVJmqrLZY8S90uCGvuI/SNb5/wF2UW8A==,iv:/TaktXubseVaJwnZWBcqOm1ApjWl5i8hDgEohEWdJD0=,tag:jjDK1ln192dbb3x6gl/3kg==,type:comment] -RPC_URL: ENC[AES256_GCM,data:Y2pnbV4J5cP/y32QzjGEatUeOkB5cdlacyzzW4HbhyaIk/gwFXLrG2wKLhL0/BiH33KNJ9Yfny1ZKgjoskrMch+vejDf,iv:aeCqv3cbH1x0GQ0eNKl3oKlrYQUSYqHqYL32kXloOp8=,tag:46GVSXyUhAbVlU6y4If/0w==,type:str] -TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:kw0aDQAhQPFvAPzdrmXZPMkrbMHYSjT/0+DKj1TNHYc=,iv:jPGyMsHEuOnST7W0egTvsjt8YdAxhdk7zYx4D70whkM=,tag:fbX+HPu12fCll74B6zOYyw==,type:str] -ALCHEMY_API_URL: ENC[AES256_GCM,data:r8hM1msbRkBqf+kupSt2bDspJ+mGW7HotF8E/Td/wOlsUhDF2X0cgi+4Ta4sbUDqAfVXlbFV5dc6Q/VGIdVmG2ZDCFDrASiEcQ==,iv:4dOSUYljDKS+1hHEWDkAfK5eJ+laiYumbXFzLAV/Sgw=,tag:xHZPrUXm3MON5csv50cWXA==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-local/cryptoKeys/sops-key - created_at: "2023-01-19T14:51:00Z" - enc: CiQAKcDhKxrapyZXScDibMQXPryWih7zIjEBWEwEDE8OZ5aKhy0SSQD1eGvVHGsTH0kg4+VoiqwP1W4JsJ0c4XxY5OzHK6uHazPWuKo7ZBMTYhNP9EyPVhnuZnm2Qc4meqeOEE1AQAPCOwlD+htEoW8= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2023-07-18T22:51:47Z" - mac: ENC[AES256_GCM,data:hT7n9ChAqhRC8aYKhtaS4x2c+RfHHCnA62ARtk8j6AE7UuHvCh1k5R+WnADes5Ui60xWPmpGY20DQOcqcspPLP5Y/TUwakslKYB/E+zf9iACfouF4rut9X8M6DlY5fvKtHmWKeCCsGW34Ytts5xawoOiY9hTwwV41FdyjUQ/Pow=,iv:olNz2EloiENST9SMLtxAF153TRLJeuTCjh4wm1nqDD4=,tag:ljnoB6fR7I+8uLdT2qxakg==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/local/local/app-local-tokenprocessing.yaml b/secrets/local/local/app-local-tokenprocessing.yaml index 0060db19e..1a084c237 100644 --- a/secrets/local/local/app-local-tokenprocessing.yaml +++ b/secrets/local/local/app-local-tokenprocessing.yaml @@ -5,7 +5,6 @@ IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:YhhIG2LlhYmzDEYELnTsR0zyjWcGm6GN09kC8iZ GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:1//q6sJN++FN3Ycq8TBIVPdD,iv:l/WezEomyf+fjEosQRxjRjc2ztBvvGxcEyUWUiLHIoE=,tag:rwkchSDeu0yVL49eMXRh0A==,type:str] GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:wutTPK740fP4r8WO0PwRKQg=,iv:dyP55Ny3ErUmqrNEbpoltAhDLRfLzBruVsSAEn7jJI8=,tag:Xlw6J6w7J8LgqVtGChYwEQ==,type:str] IMGIX_API_KEY: ENC[AES256_GCM,data:mLypOjQG8WRf6vKtMV6w1VbZn5biLu05vhb4lyMU7pBjGvOJBm+ecbVUQn/ruLqM7SF7mHo62dvNSu/PIuKRAcaMKw==,iv:oV6DzXjb+h2kL/i0TwlfpjvWF/J4HkJpDrG7eaJHI5M=,tag:V5PUKDHqfBgP4LdgCKk6MA==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:5RAAy7ukxWol2ZZ3ZOQits9nkCbD,iv:cxvp/u0iZ6XlSfmg8mTPHeMIDqdHUKlG4QV60kd2OUs=,tag:P3npOvKxh+h2vPk89BG6nA==,type:str] #ENC[AES256_GCM,data:lVXNSgfXWQgbDaSqPRqH0+HmnkFOKCDkvddkuCELF577oIgGxhfgSISYpRQKDihgZvUr/7ufbRXmCsylyabRy0W0LwtnADfmqC3PtL9VI1SwRJlZCy6cYIHi,iv:Wd7vvS2L/xxsQfVodagSr/fmrHuezE4n2sT38IkmGAs=,tag:XET+dpieokoiLESU1ONbMw==,type:comment] REDIS_URL: ENC[AES256_GCM,data:Btl3GsNm70oSLY4yCyE=,iv:jlHZZK7kE0hjgUx+F4M89RsyJOYVVfl63eRJQQkZVE0=,tag:QE+UEUmdge9ySwYtdRH1CA==,type:str] #ENC[AES256_GCM,data:0qYMcBuz2RcyrfSgb3XXMijC/PrtFnA/hdN5Zam1uhAT7jbDqNpQREWJf5sVLQHMncq9Fxn155ObQ/GN0AXjAQorPcQONUNqJ0EjPdlnh5t6S6c=,iv:QcnCDGhi1HM0v50/+ZbnsLM5DhuGkgveHtD6CsbQnQo=,tag:6FA8nMm+YgtMcr+Vsf0mXA==,type:comment] @@ -41,8 +40,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-10T22:54:17Z" - mac: ENC[AES256_GCM,data:t4m47LSKphQr46D8Ki0fo2NOfu9f5VLl8tN0CDX3Pkzh8Z0RkfSQ2pLfd06Tb30HJQstMw4Mb37tgAd3antj16idF/H9jI8omvQ5iEwJRRCP13Tr8DAaeMEXnIC1iu0/RjNwYIatxNNQoaypwRWEXBuv5MY7a0an7ppF2THV3n4=,iv:Esm3toJATd2Gr2Rkdwku/GZ4b954/46TKdLvfZanCEc=,tag:wYpWYgI4rSafzAcFzqruGA==,type:str] + lastmodified: "2024-04-19T19:00:27Z" + mac: ENC[AES256_GCM,data:SF+6ugHPck6j1NIdCTomQLDFPVmkq3mOVztnhzKt8QPd0ypL4dEOfQsyGHeqoz/m2v2jtbDXJRnyveFdywXEpdrHHghbFSHi0BCj83J7ADQ1DZi3oDCpyn5v+km23VCG5egPn8uPfNZQOFWcNCBJeaVFL10esRuwHKAui+4E0S8=,iv:I0PzlXg8829KUDI2uxj9pdIzqlv4cPNyc0ADlaSOo0M=,tag:nQTKxGJYnBh9sXmVAyDMEA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/prod/backend-env.yaml b/secrets/prod/backend-env.yaml index 21a9eca95..3682e1181 100644 --- a/secrets/prod/backend-env.yaml +++ b/secrets/prod/backend-env.yaml @@ -22,7 +22,6 @@ POSTGRES_USER: ENC[AES256_GCM,data:dz5FOk4nIGimMVXpMFf4,iv:S7hc/g/kaOW6RhxNrwZOl POSTGRES_PASSWORD: ENC[AES256_GCM,data:LakdjRumxbCVCgl++ka825gnQfyGz0H9a+r2b627OLM=,iv:iGVdal/DDoGDW54JKG6Dyq48DVVGxGo7a7lHrlCwQFQ=,tag:Wsj3PsSMjx3+qK0DgMvLWQ==,type:str] TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:ZZKo4+YzQ0LHDVLgvR0F9XUCRk8db8msRKEmS94/YxmEk1cG1c5QASjKs/Vw9SQ0U0k=,iv:9QuzcRM/Xmn40rPYlGITYHVBzbrqMbGLP9ot0LoBv94=,tag:dzHKI7I4L1bgzE2x0ZIBRA==,type:str] SNAPSHOT_BUCKET: ENC[AES256_GCM,data:yCTxODsLMxQUsprfsM+JMDnESQuahd6ioF8As9daIQ==,iv:S4wYnyKt3dmuadSIxwjGf43+1Sc56MvGEmNQDIJuLHI=,tag:8gSH9hMqHjsP/f0jc/fsLQ==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:VKRvcNklnL9IF+FM1iYmZi2ANBpmSoTeepGeFzHvcfs6oJOms7Fh/s4lrg==,iv:+goyc8SNZ5Jz6qN4JXtrWZ4QDlHYgJ2vWwwoEeYo0C0=,tag:SJs89vI3ieX//cHznnuJng==,type:str] SENTRY_DSN: ENC[AES256_GCM,data:2pRvfV+xzN44+Yy7tQQTlW6k/r/iXr9e7MLh3zadoHThut+8DwrsKRehV5aqWYPnpfhpW4wPUiMTQqY4dHR5McLn1Mn3jls7FMI=,iv:eo+OOjVeXszf6b2jwfVTji3UCLSRJVTtQyBR8Agm/6w=,tag:Q5KNKppoVlomEC3xCkL6Jg==,type:str] SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:lAPH,iv:83NESZk7/ZU0Wh2p4hPS3Fx2Mxu24mQIvbA/Aurl9DE=,tag:Mok8UHsz3b4Q6MkGujUkWQ==,type:str] IMGIX_SECRET: ENC[AES256_GCM,data:wPj563X6Rbks27wCA3JgoQ==,iv:W/16DoeYEHqc00jZLIkERXd20103mV3+h5ddwj01bf4=,tag:qBYF81eS6dEHkXt06HsMjw==,type:str] @@ -98,8 +97,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-15T23:36:47Z" - mac: ENC[AES256_GCM,data:Z6V3KFYeUw8F6+xfYP8iKUstanxIelIZq/HJ00B0KSz19aIllN4VXEuOD3cH5c2HVnl9cQy7n+Oj/oLu2UjfNHXEFVigpCCz3NZhUk3uRHgw5JRYK8Q7yIxMZsgQOuIvzv3FB/rV1SCQUXMpCjeA49C3VXgCAbqOWn//+Gj0Ul4=,iv:x0Qp+Y7Ya26NQrpPw065lR/EA5YXndhlsJtucay6L6k=,tag:KFSSksdxuDVDozPYzmRBfA==,type:str] + lastmodified: "2024-04-19T18:59:54Z" + mac: ENC[AES256_GCM,data:ItD7VpX/Uyz6IGUWfiSUFEr4GHxiLjwfhTSBt8357WaU4HpXiYGtUW2gK2UqFRnWsNfVlqbtJ2x8oZgkC3j0J6w1ZTJn3F/OOOtmfeg5L0uIy6t49IWqwq4rTzBtuFdtJVan62fwPmccxpLAm7xANuKh9wH6/YatxY62xHFsjdk=,iv:MnKmpNrTSKUE/1Ooi9rFSJ5lwLR0uLNoOe5VSa3BUog=,tag:M85bmMCC8Uam+0k96oLDwg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/prod/indexer-env.yaml b/secrets/prod/indexer-env.yaml deleted file mode 100644 index f292d3928..000000000 --- a/secrets/prod/indexer-env.yaml +++ /dev/null @@ -1,38 +0,0 @@ -ENV: ENC[AES256_GCM,data:q/xg8LHxMqmQFw==,iv:LzJcROpSAkPupEfPzGSSlH9pPIjvB7624JNKAQnNUZQ=,tag:bGq9CPx1v3nb6F7Y2VbVfA==,type:str] -IPFS_URL: ENC[AES256_GCM,data:PbOx+Zf97Flf0p+nr89D5ctfEWb6oCXIIiuLVrlV,iv:YUNrJrj7HQyAW0WciJn1UCP4mVbY7wsNdLXZLY2jkXU=,tag:PjW/5+kGzyjdYMiC1MjybA==,type:str] -IPFS_API_URL: ENC[AES256_GCM,data:vlEeS7diaUj76prL2yKdaW+QPtpfer/vn7yi,iv:qUoScrqlWm0G0VIHJsSfRKyQCFT2vQLrSmOyqchZBEo=,tag:WJfxNJ5mvJNY1/Xlj6viGg==,type:str] -IPFS_PROJECT_ID: ENC[AES256_GCM,data:PgU70ePQcgiiDBjORcab2cwJEIvwiTKh5icb,iv:OZwqlVR/dTvKDc15e+A1dVkJXl5oRackZtDbGsx3GWo=,tag:PJoVEhAUrad+rLDy+ogi7w==,type:str] -IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:zNxIHTd+flkIxzypw7JyVjWHfuOwGt0p1BaFlfzJTuU=,iv:tYD2ch+YrCaHevDHiAVOQ9FIJUOuy3kRMzwlY/+S08M=,tag:1NZEtevEBALn0i21LzJYPg==,type:str] -RPC_URL: ENC[AES256_GCM,data:Gk72APmS5dvHhLa6vjR9KTmk6wJpziF3hISq7quGPpBIlth6GVm5LWGnJ6wAnmc1yAiKa5SUid7iletiOhVf6KN8wddS,iv:eBdksvuTVQx9LHWbMm8QnVeBIk0W3Ea+n6uq8igqiFo=,tag:Hw0gh0Afr0IQALVk3AeDyQ==,type:str] -GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:Eg7BRVCPZawM4OjtdmORMvQvrA==,iv:AvhX6iz/DM1v5alo7EUNPzx0db2ftUMRzIzxzm/tJWg=,tag:/hdCkPydS/eeTIlTB9dBBg==,type:str] -GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:dVK0/1HsJY/tueKTlqoWzWNk,iv:IMYaJODN1ozTvcZDNpZd3x0tkwsFaRH4JrxQLBK2NRE=,tag:ad48T8tULlDmAfnGlT/Y5w==,type:str] -CHAIN: ENC[AES256_GCM,data:rAi0,iv:USjBmy1T0Z5hHcUn4ap+IqPt2a141jVbEHKFTNegbAU=,tag:UUDQNF/6QmxWJBZwd///jA==,type:str] -POSTGRES_HOST: ENC[AES256_GCM,data:5Ej6vGRL1AjTvF/lNBzpsLZYI/IP6Vd7D3nimVmRhTSpjHjJogHw+jikVPrVYlGJTEwUqKH9geuSO7xv+syB,iv:8sQI7tEP2qxrhsAway1votcYpVJo9prrjSmnkaH7uso=,tag:QJOYzxr6ekRowx1lMyGq3A==,type:str] -POSTGRES_PORT: ENC[AES256_GCM,data:tu8GKw==,iv:4tqaPp2QPbesByNzPUrW9jPJpB1BoaklPGlgpbk0OhU=,tag:A9qXe9xJDkrSoqep+BEquA==,type:str] -POSTGRES_USER: ENC[AES256_GCM,data:v9uy8NJWoKQ=,iv:HbhS3zW8311Hk3oEhpqQhq50hloSnQqrzGpkY3CG1qY=,tag:QxfONPXG+KnosOC4tX/oRg==,type:str] -POSTGRES_DB: ENC[AES256_GCM,data:ekVBcOYFZsY=,iv:7HH7WN0rrolk7CiTJHu5XAJrMFkOZ+1GtnpPOACoVbw=,tag:ZnfloMtTiMBbDQGJCkpMcA==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:48Zh4HQzWUXy9xHrCLs=,iv:hBI1zSdlU9RA0SqNhaTAs9/KheCfQ6HmTds9mLy7Fcc=,tag:c2PQIs5n41ej0uPVegQ9EQ==,type:str] -BACKEND_POSTGRES_HOST: ENC[AES256_GCM,data:PwfRJEXXy9Pn1nKLOK1QRTrG46jj+af5y2ewR+RO9BZRYz6OLOltt6BLgDwaJbCyOPxn3FjprBLS,iv:gHTS1iKfnilDutp8159TGWElvzMywWYoj/Xknj4g7Sg=,tag:dmygfYCYuYLlrYXcGboYfA==,type:str] -BACKEND_POSTGRES_USER: ENC[AES256_GCM,data:c11Yn8KG20tUGUTy5xyg,iv:YayPp1b7qUmhk5WFJPhfZVruWGsS/gNhcLCDpu+v9FQ=,tag:92H67fQplU4ya/DmGy/56Q==,type:str] -BACKEND_POSTGRES_PASSWORD: ENC[AES256_GCM,data:9Oqph9u6h0ipCb02+XkaD/B55jRi7MkNMk6q3HV/Q2k=,iv:1bvAN5aOthT28GxVU/wWVDE7lQyNaF3UROKNrzf/kIk=,tag:HmCWcYzXWK1Ux21mq1XgsA==,type:str] -ALLOWED_ORIGINS: ENC[AES256_GCM,data:XmLxOeSIAhJB/RQJETrOyWO1chQZ1g4/Sf4xsUNfpQB/975jM7Jj72lr41G8KA==,iv:QgSMzd+2aNj2BiEjQbkwVZ18M7Hlyrvv0ZBYAoa7ZVU=,tag:ML9c/JBRLoL9YSRTenxw/A==,type:str] -SENTRY_DSN: ENC[AES256_GCM,data:j1YPexNsXFa3RI6g7avbEbRsk1qMdhzB0xCzvL5EjqjYf2yk4o19/1i1g88T4lZYTUDynBkNDkbJzi473h2QYSeXw6CoPAlVG4s=,iv:bxpgv2SW2J3QRbvvhCEzeaO2M0IChqowxPfheE4TgD0=,tag:g7J7oVelBfbf1fnoJhjU5g==,type:str] -SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:Y6j0,iv:VXQTZ8pqNqSYPuUOBzQ84AgXHOCNs8OEIL38lq1IgaE=,tag:O88Hyj22IWlbz+NgbtcQ6w==,type:str] -ALCHEMY_API_URL: ENC[AES256_GCM,data:0c6Yo6RJ1RtdlLNZ9QSBtIAuM/54NPo+H7FMo0VplEaijMZwHEnnO3PdsOMa4myR4TreJ9XER6crYRym+6lRzQLyR3JbB/iRsQ==,iv:b/lc7A3+fBrER1IzG8+/jwpiX3Pa7aD0yya9yICpLX4=,tag:jhVJVbxnF9+LWO5tdK2eMA==,type:str] -TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:Ff+0XzsgJiMOSM1Vf503J+6xOjhC35E+hdvGIStzL5zEeTiywIR9QUZLe0DZlWqaS1w=,iv:qwS5+LhQLORnAFeQirO3kMeOX2c5/z9sxvP4DW5ahxw=,tag:Fw3h8mv/iVlfvZc1aYei8w==,type:str] -TOKEN_PROCESSING_QUEUE: ENC[AES256_GCM,data:MsDZdU/3M+dpaUU5wDo9DEulDAd6m0i7CH5OY0zsaktK6m5PRiAhP3/JKm+ceB98WvZuV1JW/9fg98RiJ3bOMwapdgc=,iv:Sf7NlNIWx74VWTGIPeswTdOUqr3EtnjKAnDTt3bDgdg=,tag:vH12wgpFCiWSsAUmTJdRYA==,type:str] -OPENSEA_API_KEY: ENC[AES256_GCM,data:RV9JQlK5AmWOZc8PJJFHwVWH3Nm1Gv1WCBBEu6eGd0A=,iv:hSJvh1y4O4+YSGmrsDI4n1QV9OPPOEAxcsiHP+x9dtA=,tag:P9PMXMWJBJnkIfWHUPnHPw==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-prod/cryptoKeys/sops-key - created_at: "2023-01-19T14:28:21Z" - enc: CiQA9/UBbo7hMDOTBhM1Wj9nsux1udD7u9Hazgi2RoTPSLhPMSQSSQDWc4myeoMtN1y5HMs6ZB9bGuBDX6KKFBC21N0kk4ZM5ifCxaWMd4t/QEQSHEFsZ7vnT1OoqOISrbLT2rfBLEUXfGLEAhS8PsY= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2024-03-15T18:05:23Z" - mac: ENC[AES256_GCM,data:/yQ1QQej2DIBoypfnV+VMnnxOsjbgosroV45vz7b/gKR7CmqJbmBUXEaCZDf0iHjF/MRWKMYINwHPZ1sQm2nCbZfLHbdu3JAP+EZ3v0XFO69UtLCvGv1W0MucqU939O1KlWuUSQyStDlwcl+O1S/MAg4PbYqiMeF8p3NTaaT9Jo=,iv:XlTHLL7JaAxHXDYwl4yvJ1trwvxkKQTm9dzoOcilFWs=,tag:K40QRX0snD4i4xf+u0tQsA==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/prod/indexer-server-env.yaml b/secrets/prod/indexer-server-env.yaml deleted file mode 100644 index 4e0a35678..000000000 --- a/secrets/prod/indexer-server-env.yaml +++ /dev/null @@ -1,40 +0,0 @@ -ENV: ENC[AES256_GCM,data:41raOeoMVqcBuA==,iv:AZX23YPAUa5+YBF63w2JrBFKJ7YkFuVlGTFe+MNvqAU=,tag:QtLpYBjhGllV/7VSt8iVFg==,type:str] -IPFS_URL: ENC[AES256_GCM,data:7PVutrvucvTwAwZu+YWpRMkCqAHSc8YwFc3VjF7x,iv:WHmFm/jidSHUPYF4/r3LEwkTwGmSvYfx7qJsDwSjzF8=,tag:V60UBrgYD0974t33WoBZ8g==,type:str] -IPFS_API_URL: ENC[AES256_GCM,data:mAuin+5KJN9nM2lSt+E3HMKsILs4ICQxr5x4,iv:jgycRQMIFJkhEoQyWr5+DxvOBdorP2PWbNMpabjzLRQ=,tag:RqYZkxYft8itEhCbZg5+mw==,type:str] -IPFS_PROJECT_ID: ENC[AES256_GCM,data:ZK6Iwzyo0f2aGIeCrBFG44gOZe9hCVUbZhT0,iv:VV/d5z5lTwRPt7nR/Z/39WtlGqdkD39Phe8oldRPxR4=,tag:jw8GmSTsRL0l2C4LC/84Ig==,type:str] -IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:WqEAb5arZqmgMMCq2KDNQmkwihGMs8p7PeNQna53Fww=,iv:4CXpexSQ5kQni27/VHiD9BwFJydLWED2cPddSP1/scc=,tag:yHswHdKPax+97YCkzRawcw==,type:str] -RPC_URL: ENC[AES256_GCM,data:vX9EeTlrGijawGaTmJnG91cdzZTq+8yWChODk1EwO4Y0cymS2Axy/XBEfY71fkrUJgMlOuxttSWKLOef7qrAWU+mdQ==,iv:JnZhOupQW/5EhWDq7m+AXFBEw+Zta5xD2x5MxBG9/ko=,tag:BJbm8oCxp0bDy0pjY9a97g==,type:str] -GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:elhYJNOeX2yRRAT+2nOTQrV6lQ==,iv:PqpoM/PpjwBr4mLvCj+MzoYpLhJaFVHODRtTpQ9UK1I=,tag:Uk4XWHiVuyXaBltzBEn5cw==,type:str] -GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:e8hmNVLREI8PWys4rlOK4rX8,iv:luiJZ2OIeTboc46gLDPJ/aurAqWzdePY+zbd86dZKiY=,tag:VsPGy9IKYFP3yxqgMSoTMA==,type:str] -#ENC[AES256_GCM,data:j1yom4QcBgoPUoeTwdjoQL6kyU9lbWvK6to=,iv:ZVx+ptBFrsKWiZJJb/LkJyTqKqYe9h30ZYv4V/1KvxQ=,tag:D5t/6UfNDErx1iDeuUxhJw==,type:comment] -POSTGRES_HOST: ENC[AES256_GCM,data:kCcOMrnRO05Nn3qlKzUErKkJa0sfGK1Ujc8Iv0fq10ZyQ+g4nThdzSHWs+Kpyj8KYJH0UnAgZhLFgha2ad0L,iv:zfC6xeIhk6Dgilkts8dU+Bak9F3zQl2PuJTQzIK32sM=,tag:4ZoQDGnkw4/dgeNhTfNDsA==,type:str] -POSTGRES_PORT: ENC[AES256_GCM,data:IR/a6A==,iv:drzEQ8wR//R4a7kKyrqLBUMuMjowpQKImhhOuYCm2iM=,tag:F2Pz32tg6wN/WdCh/2KIgQ==,type:str] -POSTGRES_USER: ENC[AES256_GCM,data:PwhVT7KAX3I=,iv:CUJjplFumm3h/MY7H7RJWgyU1+E/xVImhXGTLvk/4CE=,tag:XxVN8Q7ry6RJu6VtIAIEaQ==,type:str] -POSTGRES_DB: ENC[AES256_GCM,data:cIzhwMKxmMU=,iv:tZtYY3bMnM0k7RbyUSf/QRST9PnNpSW3BgRm8MZ5QYY=,tag:hT/JoVjzut664tjku0xQag==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:KxgYgUzlOBrGQoHSkig=,iv:WaPXd3kUwJfUCR8UtbRtqzzv89w1KW1tUhmetkj/Tg8=,tag:7msyGzPeP4iYkEz8zY33Ew==,type:str] -BACKEND_POSTGRES_HOST: ENC[AES256_GCM,data:pq3H1LU0ppeQzgJz19azJnSv+EsOxaZrnzpcTygWmoim3TqpNonxHn8aPhfTekCriXS3sxaiI7su,iv:iigAfAk4LR3zLxCyE/jg1KsuDQZWa8RrV1oXUGPZAhg=,tag:gnaLLi5yveB24j1ILkybgQ==,type:str] -BACKEND_POSTGRES_USER: ENC[AES256_GCM,data:KT0LUAm1J2xc71AFmezn,iv:AE35XWp2QKcIqFfnYyoyCj4TQwvSjDvLn6vKYTfozYA=,tag:t4B94dJcnmpUPyMWMrT4Qg==,type:str] -BACKEND_POSTGRES_PASSWORD: ENC[AES256_GCM,data:KrH0iB3+7OBTBrU3Y3v81jZY19iryF8DMrYltsjqjbM=,iv:304nRB+tFez5XFRylf7yb0vh5nWvaBMCOGzQIr4j+SE=,tag:ePtQAvGgiG2alCqUnYCRbA==,type:str] -REDIS_URL: ENC[AES256_GCM,data:mu9NVuJr/XJnovoccW7rPCRv4w==,iv:lfZOVSKIY7XzD694Wkd485CgiX7fEO8ztk21bO0Kc4M=,tag:hNdfvSUHBwyDoDzMMK1cwQ==,type:str] -ALLOWED_ORIGINS: ENC[AES256_GCM,data:986Q77EpT/K66Sr8kyRj2AOAwBF4k1Or/VdU0ann/ua5UcO6g0EVFTZ+RJayHA==,iv:fJVnUHnlniCdXO8gpr0fSoGHm1qAt1pzpPyi1V2G/NY=,tag:NyA+4O6iH38FlPk/MyjDbg==,type:str] -SENTRY_DSN: ENC[AES256_GCM,data:Ucs+4firOFy7FtYwrGWMaO3iirCrTVa1uQX/S0OGbCcYm9EvmnAI/A3psTfIKFPiom088YWTISDwMT0/Lv79YLgOTKhrh7AXxik=,iv:wtIphs+4ZahlUpcx3e051WrLlviyE4j134JyHLn5Elg=,tag:Sd1wl+lbtYrKNVSuaH/fuQ==,type:str] -SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:gxE5,iv:X/o+l4+5zE0VQYnphA6Z0RMelVIqoTPyLtmOFuLGyBo=,tag:b+pj5vURsH5gU3cLStw2XA==,type:str] -ALCHEMY_API_URL: ENC[AES256_GCM,data:1qeZ5J9g4Un4ivZHSBrFH4cdY1twv1v3YIqrV2q7Hs2eC8FZjxYWxM7WTxm+LKfmHYodzHipHy1oXfq/gnO4UeZV7/dO,iv:gWYuntyEv6x7qG6MEizGmxVFh9UuccbzVkqx9OVtojc=,tag:0H7MHcP846npVX7t1maG5g==,type:str] -IMGIX_API_KEY: ENC[AES256_GCM,data:QVVs27yZYSCYHhqdjOa41DI2ueRV+4EviUcP7rdXcdpZ/jS3dgmnBPRpfGfZGucjyxkJR4D63hDxZ73qVrBmwgdnTQ==,iv:cXlC5gD4Vx2hmetLtSRXbyU2F3YrJx3W6LVy9UxwOQ8=,tag:b8UbQ6NnvLNSui0VM26CqA==,type:str] -TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:2UOop6pNHOf5TiUcqO6LXMtOvX9XAMB+fkoxxYFmHsZOi4TiA+ebdaa2E+WubilPbMo=,iv:ghBqNYWETWi2XJoIq4pizne+CXnHxC7Rh/g/3wHyOwk=,tag:GeTXegUsnTAb7/UI4UfI9A==,type:str] -TOKEN_PROCESSING_QUEUE: ENC[AES256_GCM,data:/61f6dfD/1L/NiS8qwsVyOtWJC+Vqhxx3n9RG2IB9aTUPlAUMnHQou57TFqXYo4a0kEue/9tRf9lz5zTH41Q453cotg=,iv:NJSIP/iJzQzPy09pUB2CdUvOa8vwY40o5qrN3V2/99c=,tag:vKXvvRETWW/bZ/xELvZ1GA==,type:str] -OPENSEA_API_KEY: ENC[AES256_GCM,data:59AdNiN4NoB0RWDp8hQL0XAsmVWXGEUkxmwscDy7f/Q=,iv:uZdCdAxK8w366EvIOTyIz3l3aLCuYj35RbdtIm8bNhM=,tag:qxoCnnnhhNtx5++kYRAnmA==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-prod/cryptoKeys/sops-key - created_at: "2023-01-19T14:28:43Z" - enc: CiQA9/UBbhmDUH7DmRd7rVTgakKDuVSLNeXewdrBaCryRuqLcqgSSQDWc4my7NbR1j0QNjyqv6p0qMMi4AbvpIG9EwKp60Gztp4kiHk4MW72kK9hWZ9SYJUufoU/V083WZ3A86NIqtpYI9djs4gzmd4= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2024-03-15T18:06:57Z" - mac: ENC[AES256_GCM,data:1D6zIkn2+911btEbj5RaMZq21Pvxv/dJfRpY5dyUEfQLgF3VNmMPNHru9Jf5+Jc3SZszeCzeOnbxcEX4HTIc6cpBB/2p8vcbjNRr6Vwdqvp9Haqp4tn3PAgee4Qsnab3UnN+nzmNNR70mvSJqHOfBh5w2EXg0LjyGH+ocWUH4kk=,iv:UJuv+1JOXtJXWvIQ6X+G5Nkq64B0JdrQRbpDd24qe1k=,tag:gjpG+FKSjP75RAPED3sMwg==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/prod/local/app-prod-indexer-server.yaml b/secrets/prod/local/app-prod-indexer-server.yaml deleted file mode 100644 index e7a23a930..000000000 --- a/secrets/prod/local/app-prod-indexer-server.yaml +++ /dev/null @@ -1,26 +0,0 @@ -#ENC[AES256_GCM,data:KzuMxahIwM7mnbvIoClR50xZC4b50BG33pb7mAV25/8cpgNxNmkXEoVPeDsbyy1pQA==,iv:ES8m+Dfv0RRExC3zKzIuGiF3U9LtWWNnQrI8L1h2dZs=,tag:8NXQLSyRMJJ0kCbUnwfMxQ==,type:comment] -RPC_URL: ENC[AES256_GCM,data:uWv3fY5ZrPQqy6wAHRuM8NoYDgzz4dwpJISOG/8jAz2hVnUKTjE+Uy/i54T/+smT2UKYvgpy1ILuhGsBuO4BzG/PqgWM,iv:xcQrCE3NjXwLwlaqm4tW+ie2kEVp4EXVQ4HsGdAMgE0=,tag:RbN8+iwCC9zybLVXFrhlOQ==,type:str] -POSTGRES_HOST: ENC[AES256_GCM,data:vjhoTrLzN2ME,iv:ZqqlXVDHChWauKJGTMPQiZG9auQWBdXkFyUjBEYySO8=,tag:4g4buwX6wHUtLFyIrnMhAA==,type:str] -POSTGRES_PORT: ENC[AES256_GCM,data:LXZIOQ==,iv:DGG0hg0js7lia7IOIypPAmsSD3QTqrUvFqlp10vgL5E=,tag:d7ufHMoqV7FXK2DgO1ZTkA==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:U3rjjh7wNJXvrFkzmFY=,iv:uvXkaE0VlteL9sSd96/t1WHqQg7tRmC/pB927yGcY20=,tag:RunLH1KGOMaLOwIbFhDhaA==,type:str] -IMGIX_API_KEY: ENC[AES256_GCM,data:T/bew1f4XtSON7n+EyeDw8WRXlAzQ+b1jtDndaftcyQvUBrnjsHwlT0lEeoojeqQh9c8+qmM905vUYB2Z+inMO6QZA==,iv:bM3J/mpHSX0lh+Co2Lr82UAX9aa7tm3v8Q+uwoaV29A=,tag:f15Fbn5tixxfo6Qh6AlMDg==,type:str] -IPFS_PROJECT_ID: ENC[AES256_GCM,data:0HDkTBKwAz4B8VyeWXXaW0X9d74BRvqPrFIe,iv:5Y8P1ZjnBQs4/b8OlYdU29FDdqSyhyGYLuYM6MioHxk=,tag:U2jJFvQzxAhGHTM/SfwK/w==,type:str] -IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:ck99tg0re/8GL5iDjCK0WH3ofYmOK/450kczN114Las=,iv:lUnWZAULc0CFUMfklvfatR3ww7uZcZ4KXO/i+qkSPsc=,tag:efz7Bxwl4Azu73KuXiD9yg==,type:str] -GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:C//34woVq3I5ucviO9g6/BPm1w==,iv:yB/4/cXEueR+wYO1rBsFRI6CTD9BBu7jAV9IEq4QnP0=,tag:XmIpPwCD6nntqpDzazcpkA==,type:str] -GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:bOwusFszqEg/9IVJ5uVO5Kf2,iv:vIfQc7SQZuzt5O4N2c51bIHHmYQZY+Vmnapq69DvYlY=,tag:OJuHKUKSMRuqhyr77UBPtg==,type:str] -OPENSEA_API_KEY: ENC[AES256_GCM,data:WqUH+tQAGr9J9/JzGxCAGG5PEj2z/kXlqsuRDQkAb2g=,iv:DY+0R4BFBxP56yAiVAGJ6pMwNzQdtrvniQr9YLj0NEo=,tag:t/ugk1pEed8oA1brMuRKxg==,type:str] -ALCHEMY_API_URL: ENC[AES256_GCM,data:Fx3fb8N2pakbSnA4ZvyZ9BzEM301VNc9DNESbKG3nXRtGHRTv5O6r0g2LcEV5VK8XP0+NAtbVift3+y+w1edHlKA1KrCMKlwDA==,iv:HwnuVPLhbl5pLhreZq0ZCJbMnKr/8hZDSL3azoBTG1E=,tag:D5s/Et9Nx6ACZIDo+yCRHQ==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-prod/cryptoKeys/sops-key - created_at: "2023-01-19T14:42:38Z" - enc: CiQA9/UBbvlYV9Lg0bRG8IxGanXle/zgtOYC5HbBb3xm8SfRi2QSSQDWc4my8KIGzQOVAvCMuskRa/lBCUfiJkZF4E8wcUWXaLO6/7hTDVGgXX+CUQYSa7ck4WxzOVLh5BYxjAdNYEbBJH8TxrOnqEM= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2024-02-20T16:56:12Z" - mac: ENC[AES256_GCM,data:VjZlEjSCvQynsP7v6cscW1Yw3ZqHj6849sQex2gqGK+bzFzxDovU70IzllF2JBJBPLBS37WXlJfQdCy8qs/UUFFMcY623gvhADiN2cetSWHpfxc1ud6+J0fwHBENX7+xHWOUW9ibyjQcSRlA/pmIWiuhTGgXYOfk0m4S1Zzlslc=,iv:zclSOW+eVMzcWV1Jo0Jn2on4lHwX9VgTzMkkFkNhySE=,tag:2KUtLDpSIrxueLfLw87Q4A==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/prod/local/app-prod-indexer.yaml b/secrets/prod/local/app-prod-indexer.yaml deleted file mode 100644 index 4e53484e9..000000000 --- a/secrets/prod/local/app-prod-indexer.yaml +++ /dev/null @@ -1,32 +0,0 @@ -ENV: ENC[AES256_GCM,data:Ph64uadmzXqXaA==,iv:UYsElFAbRUSqK5F6026A5ScxbU5WnxP6WCBLUVkyZLM=,tag:tSrDSNPizBRjjx9XQAwiCw==,type:str] -IPFS_URL: ENC[AES256_GCM,data:dpTYOmwb09BUSV4t4SeOpAzvAxcmMxS5+9116xV0,iv:Rd+B4PNbUefGnUPQPZItVsCuL8mved48Mykl4smgWhY=,tag:gSqYe9HAUlP+8HOHdWgRlg==,type:str] -IPFS_API_URL: ENC[AES256_GCM,data:tmIl2GAzZ6E1jIeJ7Uwg+h2UrLlpYW/FYxJa,iv:B5pSgm8npR+m7bx5uSF+NNIynM/0GEv0HPhVJujUrQc=,tag:0vNEGUe6bxXGISdxxUNc7g==,type:str] -IPFS_PROJECT_ID: ENC[AES256_GCM,data:JC8vNsASCBV93MMm+7GOyquOPkuYsv1epOBR,iv:QZG8NUGX8DwcrcMEV6NgKgKeDKkN7SnKDFkpWCegis4=,tag:Q5Dqt/kNN/r1QBhupsY1iA==,type:str] -IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:z4IKPidkXQssNzX/K8l7kEBXBS0WV+fkH46/g6+s6Vg=,iv:f6JZYFpGt688cnuBTyd24Q92o96uN57H4Hxz0BTCYR0=,tag:0V6KxjBNN9j38QmdvWstxw==,type:str] -RPC_URL: ENC[AES256_GCM,data:QHGI5Dt9SPy0FZudKVsleLBX+E/Eh2N0mjhPH0TWsCuUxBeoHYYI5P38zvi+bvDuGZ0LwYvlgjYVbBdvLHJqfgUZIMXQ,iv:Zq/0QWlPiTnxgUhL1cDqVMM+t99tw1Gf1vJx/2zKySw=,tag:KrByUS4OCgryTBAGdIWLLw==,type:str] -GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:OFw5uRbo0NfcmN3FcFMIBvOmQA==,iv:kXu5/LKI4/PeE5Df0TKiknr7PjKEvMRng/MmYOtUEZ4=,tag:Eb88wd+N5Jj47VTDjD6e0g==,type:str] -GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:0Ux5QqrIEP4ySmAfVtb/cFjI,iv:UQq2wW4F/cV/D6Nv9Tm6MXarx6mVIEag96AB6gvnRZQ=,tag:lS1FfeHqMtZMGFl4NKAXdw==,type:str] -CHAIN: ENC[AES256_GCM,data:bsyn,iv:8QEEj5x5C/Ww39gkm3tc04tGhJP7r08pwrlcZ1+hyTc=,tag:hAulal1/nNikFo0i+JxKsw==,type:str] -POSTGRES_HOST: ENC[AES256_GCM,data:6IIeRLC9qYaz,iv:dSNCglZCwfrbnorxOXja83Q1xteFG5PNDfeIxD7r24I=,tag:asaBUR/JaPX8eUaF87OkyA==,type:str] -POSTGRES_PORT: ENC[AES256_GCM,data:hVAYlg==,iv:IJOCH79HetI0nktOT4xSy5qpY2UCrOd3QVk9QvgDmSg=,tag:8HTGKD5v9MQdvsIJR31hqA==,type:int] -POSTGRES_USER: ENC[AES256_GCM,data:9T9QbUXi/yg=,iv:ZeHrZoPRKWUHZ+GUM2TA+RTwLY0AZ0bAtE/2kpqRrnk=,tag:gF7Wu+tYh9DijS8sH+djUg==,type:str] -POSTGRES_DB: ENC[AES256_GCM,data:JXdmkjb2fCI=,iv:4FcvKYXAD4H+AtKTOw0rrJgG+mcNYvCH3Ls5a37jjew=,tag:YZwES9oidcXhKKJLg1TDKg==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:O9mxbXvvKelgsFQbUo4=,iv:w8wjo26P0Wri9236nutZ7e3hg5a/E2AgGp+DAttezoA=,tag:6pB7gIrhS3ztMZqd4ns+bA==,type:str] -ALLOWED_ORIGINS: ENC[AES256_GCM,data:qaZyXiM8C0npXdAxzHuiuXCMNDySHNwrvQ1KvJbpLGYU1FO5L5vgiGUI+wqBKQ==,iv:J9STW+F15VRMHA6Gk+43Qef/Qe26NfaecImtt/SuTXE=,tag:iwW3y569HjjxvtxdhGhnjA==,type:str] -SENTRY_DSN: ENC[AES256_GCM,data:EUbUxY9d+HSpGXyj0ilYhmllAqVRXCdWBTElADuqMy/hYAg/YvWUd55DIZ6eXrQhPAPwZ/MhuPgZC6SbhyA4N/6B3dIteXueDUc=,iv:R181PyiMz6Zzi+jUk7d9iOHFyS7O+MhzhLE6KDowgs0=,tag:YUNYPLX70ezlYJ9Nzdz+MQ==,type:str] -SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:lihG,iv:5oUZ9iaajfKCvBAz1bHjwPmbRgLe76xYzDwsLkf85Ec=,tag:aN66Fx0TP+HQWFMJhiskUw==,type:float] -ALCHEMY_API_URL: ENC[AES256_GCM,data:szvjhxb8giFgSZ7DRvnkAQx3Ej0fkKNn6DEOJXUfhSzChfYqooEPP8vqVYyn/HH1hSG2NtuzAe9W65EdBj3EX13kbC1H2QFFnQ==,iv:FspuwpQpi+SRFYp4Na2ZSLH1JPC07gPXf6+s5O+6n4U=,tag:kXcGD9dNEBC7mOTXJzutBQ==,type:str] -sops: - kms: [] - gcp_kms: - - resource_id: projects/gallery-kms-374316/locations/global/keyRings/sops-prod/cryptoKeys/sops-key - created_at: "2023-01-19T14:28:21Z" - enc: CiQA9/UBbo7hMDOTBhM1Wj9nsux1udD7u9Hazgi2RoTPSLhPMSQSSQDWc4myeoMtN1y5HMs6ZB9bGuBDX6KKFBC21N0kk4ZM5ifCxaWMd4t/QEQSHEFsZ7vnT1OoqOISrbLT2rfBLEUXfGLEAhS8PsY= - azure_kv: [] - hc_vault: [] - age: [] - lastmodified: "2023-04-06T23:22:30Z" - mac: ENC[AES256_GCM,data:P4QdE/s7k8WcKHmbpXTq/57KcZeQ4Omq35igZdGkTQDKlWibHDFGk+dFfsrvOepSzjeOtsq97QS1XrOMARH4ovJD1h1Iy8mLUXv0qKb1gUfHcq6jzAvy+lCLUXdH4wxXWvOSLbNanoPpM/A8f07YeBlXJc8xae+5mdqe8nhAPxY=,iv:g7g3k8tnjHA2Uwqkz2r5AyBtNF+X75r5jJjt4eVAqVc=,tag:PrUYjP7hwtJJMLgXX/HVjQ==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.7.3 diff --git a/secrets/prod/local/app-prod-tokenprocessing.yaml b/secrets/prod/local/app-prod-tokenprocessing.yaml index bc3c9ab13..c25e02b89 100644 --- a/secrets/prod/local/app-prod-tokenprocessing.yaml +++ b/secrets/prod/local/app-prod-tokenprocessing.yaml @@ -5,7 +5,6 @@ IPFS_PROJECT_SECRET: ENC[AES256_GCM,data:5Krd6/n3kzC+Ebl1MFAWT1gmPFmdVks0SjDMH4i GCLOUD_TOKEN_LOGS_BUCKET: ENC[AES256_GCM,data:/DbibSgBMSdFoNNKy1A5C3C/,iv:M+avpitY7btLxhX55+XiLM7MsIh8OnoHZ/lshQoYHoY=,tag:OmgsTl0QhxJXZXwqyuCIsw==,type:str] GCLOUD_TOKEN_CONTENT_BUCKET: ENC[AES256_GCM,data:Z4R+jN94rRyKtS9NZokgwXY=,iv:JAoMvcdY4/xWjGK06bA95WHmoT0oDo4CyB1eTtYjmuw=,tag:tFzFZB6MresG7BZwjhIMWQ==,type:str] IMGIX_API_KEY: ENC[AES256_GCM,data:uqTHn031IhPM8Rab+2hx6PymiGZZKpqKhe5vPnRIDDilGSezbmIy7N4b37mfUqwqKsTK9XEb4qKGdicjkgYtVW4ICA==,iv:u/qijLXqbLqup+nh/mjQjwtl2uJvhH/Hv224ghkNwdw=,tag:NBAX1/MUODZP7a58LGNyXQ==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:vG5wueWWN1Gfs4bXts5fFDtQdZtVpAzbLO7tUwRjVGW88Fherm+9lV7Fgg==,iv:JYkI/YTZwfyH5sR0iEouJgFkDnxM9wz7DeAxRSdSuLc=,tag:PI7YK4dyVmXybIZPPsXv2w==,type:str] REDIS_URL: ENC[AES256_GCM,data:BW9xNGUaTkckc0rG+Jc=,iv:WjgMZopuWGnhTz5GhOoZhOum5c3l3rbBaAAP5wrlAHs=,tag:zXhFsVL/j1+ZPTA+E6XaMw==,type:str] POSTGRES_PASSWORD: ENC[AES256_GCM,data:IEJIY11XlwQCtamgxBEfMxcGjUoy+92Ql3RcB5UKvmM=,iv:wO2pWxvsGYhBHHfz6lSpCTrBwK7OP98+o24tL2f2N3k=,tag:YYlrLvNlymAvPLBgwIaAyA==,type:str] POSTGRES_PORT: ENC[AES256_GCM,data:epEEJg==,iv:xar4V+r5C3U37M/9S58E12lsGM0x0aFRpu/Wslcqb9I=,tag:KfB9kbpA+9UtvrqqBdKC6Q==,type:str] @@ -43,8 +42,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-02-20T16:55:47Z" - mac: ENC[AES256_GCM,data:KHXlU9MqXAwWiv2f9V8D5VDvkxEGBGx17hI8VhB9UH27y4Rb8emcpgHSfNTLTHWKndXNyNbPkV/vc9JU+01ch7N0JUAK3pqpwjqYT0s3a3EPFHBKOAwvh5eIjGhWVzJlXZncxQZdVey0Pt1AICza0Du7YzS7rTPFolPXUSRsHxA=,iv:2kC7lpuxcG1q8WSEwoM29jKrWDlvclw1PeYHz62kMUo=,tag:5J8ANctLyGl4tsi2gfTPrw==,type:str] + lastmodified: "2024-04-19T19:00:41Z" + mac: ENC[AES256_GCM,data:sbENaJ9m+W4z0lXYAz80PvnQjBb+C+pnfTxRuADASFuFzCa5AWEZc9YU7kDsY6NCUAVw+sOQxDH3yoRJbS/xU7iq8noutbN1M6w4lRSMGUKrXGYfMS9Ozh66Bp83fQ/y4aWjpym48w8bMhW9Unvbvu6tci69eNtD6/kVncbIa20=,iv:0IPO7ozm3YduGwDxvx0DKTmamjJyNhWnD80BTooCpcA=,tag:YmI0bsi+Zjj590Xw08k/HA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/prod/makevars.yaml b/secrets/prod/makevars.yaml index 3fbc6e258..3287e2c94 100644 --- a/secrets/prod/makevars.yaml +++ b/secrets/prod/makevars.yaml @@ -2,14 +2,11 @@ REGION: ENC[AES256_GCM,data:y1ak99fYcNs=,iv:A2ZUDJHuaIUfo23auaqxKxDor41OHsJ9kZy4 GCP_PROJECT: ENC[AES256_GCM,data:SriCE/I1JdxO42h7fDXuUvdotg==,iv:gdHhpA7hTfL5MgbzJvnMVJykq2kbXncdY+raBY8tt20=,tag:58n2KVLzRRocUUqUs6I0xg==,type:str] GCP_PROJECT_NUMBER: ENC[AES256_GCM,data:TCucFGL0ij7H0gs=,iv:R/KzP2Zm7gmkxc2//MJ0cVe46ig2nJNxZD8nxuCbVeM=,tag:Un3Q+/1tRUNqDHmI42cHGQ==,type:str] VPC_CONNECTOR: ENC[AES256_GCM,data:Qbao5kR9J1+kD3LCVSFbtmvJyfvcLer8Fvn5g8aTwzMDQizzFkSPfPddlSydDVnJZMQQ3O9fLkW/I8dyd1XxG+Nqj9Wx,iv:iid7GoMOEgHb3L1BOMkK1NAhwvIOdI/zjRun3Nq8IgE=,tag:QiOI6TJQ/k469ply8vzojw==,type:str] -SQL_INSTANCES: ENC[AES256_GCM,data:Vq902sHKUxMFsYJcWbtMskkUQfz/tCdFSGn4wKY7U63CRL2q/xUxk5yELc09IzF1oTFlCSMKDqJjzM1kxvuMOIExLMUWOg==,iv:jmGMTkb3WMJemlWH0FEugF+/uaePX6UpMrOnRhAj56Q=,tag:aDG576MkqF31Asu9gEJwOg==,type:str] SQL_CONNECTION_NAME_PROD_DB: ENC[AES256_GCM,data:aIhKp9eJQHlVMMuoHkhqNXCyYZwrBiP+qCZCKw2IibgZIckeRgwkwAiVHufEy6M=,iv:lrRgB8SWZOXxoCZQ3wuXNB5/BO+9IDKg+SQpcpbZU5s=,tag:iU6sH5F9slZS5uN1OCBQxA==,type:str] SQL_CONNECTION_NAME_PROD_DB_REPLICA: ENC[AES256_GCM,data:v3vQ2mHinxsq0CRfaPVvp127gStUslY+vYt0lhpAx3y0jgxeBBwX8+WYZzdfSXJ48+/Hmg==,iv:EAItge0937G2Ou/2vrmX9HD3rtB+9TqnI4n+MqKF3jU=,tag:H2T1r6SoIMSjaeH7fN4wEw==,type:str] -SQL_CONNECTION_NAME_PROD_ETH_INDEXER: ENC[AES256_GCM,data:Tj15Q11EpMM6n30XHriHm0DG7IxQTkJFScj1DZE0/C/WbXaciCcbK7ayJwUH/YOrwaScOWs=,iv:vfU4iL3DclNh0C37t9wHlbveTFEg3fMcT0gN35udQd0=,tag:DfrQLoWyHXB5zL7XHWLpxQ==,type:str] POSTGRES_DB: ENC[AES256_GCM,data:tkzAtxvOCM8=,iv:Mw7Tpb6fHG8JP7If9lHwi45ET/KIdMmIL/xQCOzdZaY=,tag:KJ6eova8U2HK0CghzmPtVQ==,type:str] POSTGRES_MIGRATION_USER: ENC[AES256_GCM,data:nAOjC0hHs/fQ7ccKxN/zEg==,iv:u2WJwEmqWr+ID+ZbKzGvzDNfFHWoKWlNyg8zAhUH0M8=,tag:+h5ftCU+BqmG9Z1phS+pWw==,type:str] POSTGRES_MIGRATION_PASSWORD: ENC[AES256_GCM,data:NWN0jcb/sZv1CLG7lxWv13gabbe8N+4b9Hab3hnuDis=,iv:wWNx7JfE23CP+MoM5/etJ1Zc0rlofmX0707EplwqSe8=,tag:mMeCYvOtDW4YaSjlkAWPhw==,type:str] -POSTGRES_INDEXER_PASSWORD: ENC[AES256_GCM,data:fR74+OxzPzM+oHlgddA=,iv:ljn7QcOMcluwtBjuC/av9yWjX441eYElS2Dc7giaalM=,tag:kW5hVx3fJBJfBTxPUz48mg==,type:str] SENTRY_ORG: ENC[AES256_GCM,data:mdCkl/gd+xMyWQ==,iv:COPTw3bJlQ1ygeG8miJpJsuZdd4jDOv/W+q9ZqCHkDg=,tag:Cfi2VN+PU8Itorxuw0zl2Q==,type:str] TOKENPROCESSING_TIMEOUT: ENC[AES256_GCM,data:ktYp+8k=,iv:wHzMl74UpuJWROq0HwVEOX3RgUyojginuhQy9sDvzfU=,tag:9COwfpQWRiVcHu78aRjWog==,type:str] TOKENPROCESSING_CPU: ENC[AES256_GCM,data:wQZHhc0=,iv:t9mLDGAKHJJaadwiLOGWklRq2YK0udAhjGhdJ6CcSzQ=,tag:tJTc1jPrtodBvF+Lt3zzMg==,type:str] @@ -27,14 +24,6 @@ ACTIVITYSTATS_TIMEOUT: ENC[AES256_GCM,data:PGxNPvg=,iv:7gRrlrlaI8KlSd3K26ei32drP ACTIVITYSTATS_CPU: ENC[AES256_GCM,data:aQf/Hpo=,iv:YfyIIC4cSxUMw9+q4qB3iz8G1iHJp51opKcWdTx45DA=,tag:ZAoomBuxsmNLEPF18J12/Q==,type:str] ACTIVITYSTATS_MEMORY: ENC[AES256_GCM,data:e4d5,iv:YE992Y5jxFNINKJ3JeqoA08IHayWIxvvkEY5MWrP3Uw=,tag:h40U03OxsaUAcWJTLNf5Eg==,type:str] ACTIVITYSTATS_CONCURRENCY: ENC[AES256_GCM,data:Wg==,iv:xZly5EN/fpmhZ1ciNF4cQb088FYzLo2XazlWYwyNk90=,tag:T+wTd0iFfTwfCVSrTULtGg==,type:str] -INDEXER_SERVER_TIMEOUT: ENC[AES256_GCM,data:MibLAhw=,iv:/FyjSNjdOQMfV2zNT8FCXKGlWnBHDxodSeAJaeAdeUE=,tag:3ay5gezyxZMc1wHTWqmhig==,type:str] -INDEXER_SERVER_CPU: ENC[AES256_GCM,data:cU8mvv4=,iv:xjSdwdWciCgI8m5A055DBavrItIm70IWJejzAoJIc60=,tag:qM4Kc2JdbXUzjkGo1sOFuw==,type:str] -INDEXER_SERVER_MEMORY: ENC[AES256_GCM,data:fdkZ9A==,iv:wxnuT4iyHONGfvytqDTCSAHWDSQnu03nggSUXrYpvO0=,tag:lpWDodUp+astOnwvjpq3lA==,type:str] -INDEXER_SERVER_CONCURRENCY: ENC[AES256_GCM,data:2v/+8VX4+A==,iv:TExx3ztyvhBKIn6SKQpC+mZj4wTc8e+eMYuJTeZd2z4=,tag:m8JyA/Yu5+XvWKSLpPvzUQ==,type:str] -INDEXER_TIMEOUT: ENC[AES256_GCM,data:2PxYJ6s=,iv:Yl+c5VpvkPUa/bfhi1WqaZm512zV8CfKrc4O7FdZHeQ=,tag:QNzsuYv8/+lfUzIuYRi4dg==,type:str] -INDEXER_CPU: ENC[AES256_GCM,data:bWzN6uA=,iv:hpqb9t2MHcdJkhgQgfVNcuMI/3uiaIJTIZSZcVktvRA=,tag:PuEsiQsdKRgp0skcN3t2LQ==,type:str] -INDEXER_MEMORY: ENC[AES256_GCM,data:BCmA,iv:ucESccxQYYNrBb8MDri0vI9CkjaVnYnbiOVXxAuVUgE=,tag:fj50GtbMymt3+/xC0FI06g==,type:str] -INDEXER_CONCURRENCY: ENC[AES256_GCM,data:9I5b,iv:pAlM5hQxu8B4UKOWwgfU3PparV/+k/YPG7YITz3cE5s=,tag:NlAofsDh2R1tPjePgKgdfg==,type:str] EMAILS_TIMEOUT: ENC[AES256_GCM,data:mJIq7UA=,iv:2kTP1IyV6e3kGFLQaL1zG4+hkMraSAQmqjaKNpEVfbo=,tag:cYDa0j1K2Nu3FKVldZIQzg==,type:str] EMAILS_CPU: ENC[AES256_GCM,data:pIJ+/VE=,iv:QYyUnXR6vpMb3ePF+AKygkXdzmfEKx1p1cVUHNKqsDU=,tag:irFx69Jey5beR0d/RsEATg==,type:str] EMAILS_MEMORY: ENC[AES256_GCM,data:ekK3,iv:HYORvKR/5G1AyI5nhidMtIUdtLUlKe2XY68pbaDnfUo=,tag:TolgYgtRcXzlbB+diHs7aQ==,type:str] @@ -77,8 +66,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-03-15T18:11:54Z" - mac: ENC[AES256_GCM,data:ZfPsD0RKmPxGwShKKK6ZN8EevnPakCyfin8t5ILeTGG6ux5IqVrLbJM6DWt8pz1XthoDMFxJwNPEt/CnrX7vY92U56pVjBH+LwxVzS5lx1YG08AwKgL5UN9XTBR9LlNxh6rHrbz35rgT2pTDvtCcsEnaSBcYhEMh5j2J7ngR+7M=,iv:AzxVu78w/cHUW/dgXlcNcaAVvtNVAeoParww/H6pnCI=,tag:O+y7PwmobPJogFy7jSbwQQ==,type:str] + lastmodified: "2024-04-19T18:56:06Z" + mac: ENC[AES256_GCM,data:nUGo/xaXswHhFyeuCjUKwrAt4IzceZ+OnLnuEAQmqfLHLI5eBRc5TfpICius993P+i8Qzbt1bGFdzYU/xbPZMp49g4TSz9fauBsNLh9jG+wxmGX3tyICmTfZyyH47327tV/TAqtTlpXzi2vyacYV4UhDjC9PeakboPeab+IXYJU=,iv:wXqYbkxr/Qp+vvbUk093GppfzUoodEO2THvhRsy7dNw=,tag:EUckSaRTT4OHI6VUSeQvwg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/secrets/prod/tokenprocessing-env.yaml b/secrets/prod/tokenprocessing-env.yaml index 529391acc..13b87208e 100644 --- a/secrets/prod/tokenprocessing-env.yaml +++ b/secrets/prod/tokenprocessing-env.yaml @@ -15,7 +15,6 @@ REDIS_URL: ENC[AES256_GCM,data:vUEv7q/aAZnlEaHYyAy2plj7zw==,iv:LkaSC38OcvM13BvSg SENTRY_DSN: ENC[AES256_GCM,data:BxYk67DTnbPpLjMJxOvSu1IEtAUlLHeAcbvkqUq820cKPHf47XR+ACSqe2JxYAAqfEgCoXQYrCLiOV7z4ETd/o0ZFo/D/QsQ33Y=,iv:82T2sYKB76M8NPsj/2SgisvEfV8sbrTkHZrrtXcJk0g=,tag:QV17XRYHsBjJTH/nGYNKZg==,type:str] SENTRY_TRACES_SAMPLE_RATE: ENC[AES256_GCM,data:LToK,iv:xBIAcvDedIrpUMy3y+/XZGu0HNjZsoSzL+ekxiq1H+o=,tag:MglsDDZfCsloSf3RZwZSUQ==,type:str] IMGIX_API_KEY: ENC[AES256_GCM,data:uqTHn031IhPM8Rab+2hx6PymiGZZKpqKhe5vPnRIDDilGSezbmIy7N4b37mfUqwqKsTK9XEb4qKGdicjkgYtVW4ICA==,iv:u/qijLXqbLqup+nh/mjQjwtl2uJvhH/Hv224ghkNwdw=,tag:NBAX1/MUODZP7a58LGNyXQ==,type:str] -INDEXER_HOST: ENC[AES256_GCM,data:mkpc7Yizyrw7FzTdmoGzCwx50oHxz5T6iG0RieZ24nAwW6zJ0EwTCcpWzQ==,iv:LMUXGEhwnMJ2w56ITJ/vCnN6lHtLlp3LIiQB1Hr6Tu8=,tag:wGIDxoOtqNGPf6NjWmRNYA==,type:str] RPC_URL: ENC[AES256_GCM,data:SDexkJAqQvXGlqKibnf+CefTVr4Hvs2tDoJgdemsDrFPuoBtXzuMVnhYnR5sm3P8uwLUHWfsz6QIwrybjO4eNd33R1cB,iv:iwWl0yXDZvghE+rph+WxK6QKlZTH8bwb6v+7dR0WSqU=,tag:hujev87t3BGeZ68Mty7prQ==,type:str] TOKEN_PROCESSING_URL: ENC[AES256_GCM,data:eGaBJe29FCoiDRTMDkKqsL2FFdBpSPt7JrjDnzjw34QwUS6XP0spEevna9sPF/62+s0=,iv:B97uUpJdqZN/ersOyyFX6igiNb7r/K/20LFSIx5C+DE=,tag:Zka16f2+I9dvBrisGx0J5A==,type:str] ALCHEMY_API_URL: ENC[AES256_GCM,data:G35/zicSsetlvXMaoWQeNsQKOWSi0mjXFosAhbXhuWRYqW+bj9NfTy/5Ydgm6bg8oS88YyaBlwZVqdGXNQxG2gsE/9sB,iv:eS4vl5MCrdLDdDCFOS1iHcMll9aC3nkm6ju+nWpQDQo=,tag:0W80rg+TVhp7VRao7li9MA==,type:str] @@ -61,8 +60,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2024-04-15T23:38:09Z" - mac: ENC[AES256_GCM,data:V0+aBYVUe5HxolohQKSNtX4l4KfSg89qkKFdafmD0vu9SBAfDDTeUKdTTPDlKlhJJXnPdCJHNU+ujRemvbauy2VZnsWWb9xOndpB1YQxfnvQK7twGRi1uM1Tx6uah3StorTAiDHRO2JlMkcn6zIQ5c3vFwIGlkUk1NWt1xYPkqA=,iv:riLPYiDCXT/aiQ2YUpy9ud46Bj40tVhHVxtVIMITlOs=,tag:63AiM4dKHnAb7joeo6bApQ==,type:str] + lastmodified: "2024-04-19T19:00:06Z" + mac: ENC[AES256_GCM,data:jT7ekz7y7vKxCrMbJfafANFWrBSnDif058nFo0I76HA8OLt+DzpMriFu66qJJcVcIDN3KrV7NjaEkd/dxJ85ptRqBEta+7fs4E0qQh9Z1j7NdjRlmM4uTBRObeXAxd0OMJMLGWN3+sChyJ/m6wc3gNVvV38U2dw8nv7JM8G9fGE=,iv:KhhmCpbVUTqm2SKNXEhmC8hqWDvVsr0iAs9Ycb3SS0A=,tag:hdZjexuIDHytgtBKfut3WA==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/server/inject.go b/server/inject.go index f56c3f95c..63b56c3a7 100644 --- a/server/inject.go +++ b/server/inject.go @@ -8,12 +8,13 @@ import ( "database/sql" "net/http" + "github.com/ethereum/go-ethereum/ethclient" "github.com/google/wire" "github.com/jackc/pgx/v4/pgxpool" db "github.com/mikeydub/go-gallery/db/gen/coredb" + "github.com/mikeydub/go-gallery/service/eth" "github.com/mikeydub/go-gallery/service/multichain" - "github.com/mikeydub/go-gallery/service/multichain/indexer" "github.com/mikeydub/go-gallery/service/multichain/poap" "github.com/mikeydub/go-gallery/service/multichain/simplehash" "github.com/mikeydub/go-gallery/service/multichain/tezos" @@ -120,15 +121,19 @@ func ethInjector(envInit, context.Context, *http.Client) (*multichain.EthereumPr wire.Value(persist.ChainETH), ethProviderInjector, ethSyncPipelineInjector, - indexer.NewProvider, + ethVerifierInjector, simplehash.NewProvider, )) } +func ethVerifierInjector(ethClient *ethclient.Client) *eth.Verifier { + panic(wire.Build(wire.Struct(new(eth.Verifier), "*"))) +} + func ethProviderInjector( ctx context.Context, syncPipeline *wrapper.SyncPipelineWrapper, - indexerProvider *indexer.Provider, + verifier *eth.Verifier, simplehashProvider *simplehash.Provider, ) *multichain.EthereumProvider { panic(wire.Build( @@ -143,7 +148,7 @@ func ethProviderInjector( wire.Bind(new(multichain.TokensByTokenIdentifiersFetcher), util.ToPointer(syncPipeline)), wire.Bind(new(multichain.TokensIncrementalContractFetcher), util.ToPointer(syncPipeline)), wire.Bind(new(multichain.TokensIncrementalOwnerFetcher), util.ToPointer(syncPipeline)), - wire.Bind(new(multichain.Verifier), util.ToPointer(indexerProvider)), + wire.Bind(new(multichain.Verifier), util.ToPointer(verifier)), )) } @@ -427,7 +432,7 @@ func polygonSyncPipelineInjector( wire.Bind(new(multichain.TokenIdentifierOwnerFetcher), util.ToPointer(simplehashProvider)), wire.Bind(new(multichain.TokensIncrementalOwnerFetcher), util.ToPointer(simplehashProvider)), wire.Bind(new(multichain.TokensIncrementalContractFetcher), util.ToPointer(simplehashProvider)), - wire.Bind(new(multichain.TokenMetadataBatcher), util.ToPointer(simplehashProvider)) + wire.Bind(new(multichain.TokenMetadataBatcher), util.ToPointer(simplehashProvider)), wire.Bind(new(multichain.TokensByContractWalletFetcher), util.ToPointer(simplehashProvider)), wire.Bind(new(multichain.TokensByTokenIdentifiersFetcher), util.ToPointer(simplehashProvider)), customMetadataHandlersInjector, diff --git a/server/server.go b/server/server.go index 74d8c7afc..8eae5081f 100644 --- a/server/server.go +++ b/server/server.go @@ -48,7 +48,6 @@ import ( func init() { env.RegisterValidation("TOKEN_PROCESSING_URL", "required") - env.RegisterValidation("INDEXER_HOST", "required") } // Init initializes the server @@ -194,7 +193,6 @@ func SetDefaults() { viper.SetDefault("ADD_ADDRESS_TOPIC", "user-add-address") viper.SetDefault("OPENSEA_API_KEY", "") viper.SetDefault("GCLOUD_SERVICE_KEY", "") - viper.SetDefault("INDEXER_HOST", "http://localhost:6000") viper.SetDefault("SNAPSHOT_BUCKET", "gallery-dev-322005.appspot.com") viper.SetDefault("TASK_QUEUE_HOST", "") viper.SetDefault("SENTRY_DSN", "") diff --git a/server/wire_gen.go b/server/wire_gen.go index 6d6dbfb1b..ec38370d4 100644 --- a/server/wire_gen.go +++ b/server/wire_gen.go @@ -9,11 +9,12 @@ package server import ( "context" "database/sql" + "github.com/ethereum/go-ethereum/ethclient" "github.com/google/wire" "github.com/jackc/pgx/v4/pgxpool" "github.com/mikeydub/go-gallery/db/gen/coredb" + "github.com/mikeydub/go-gallery/service/eth" "github.com/mikeydub/go-gallery/service/multichain" - "github.com/mikeydub/go-gallery/service/multichain/indexer" "github.com/mikeydub/go-gallery/service/multichain/poap" "github.com/mikeydub/go-gallery/service/multichain/simplehash" "github.com/mikeydub/go-gallery/service/multichain/tezos" @@ -101,8 +102,8 @@ func ethInjector(serverEnvInit envInit, contextContext context.Context, client * provider := simplehash.NewProvider(chain, client) syncPipelineWrapper, cleanup := ethSyncPipelineInjector(contextContext, client, chain, provider) ethclientClient := rpc.NewEthClient() - indexerProvider := indexer.NewProvider(client, ethclientClient) - ethereumProvider := ethProviderInjector(contextContext, syncPipelineWrapper, indexerProvider, provider) + verifier := ethVerifierInjector(ethclientClient) + ethereumProvider := ethProviderInjector(contextContext, syncPipelineWrapper, verifier, provider) return ethereumProvider, func() { cleanup() } @@ -112,7 +113,14 @@ var ( _wireChainValue = persist.ChainETH ) -func ethProviderInjector(ctx context.Context, syncPipeline *wrapper.SyncPipelineWrapper, indexerProvider *indexer.Provider, simplehashProvider *simplehash.Provider) *multichain.EthereumProvider { +func ethVerifierInjector(ethClient *ethclient.Client) *eth.Verifier { + verifier := ð.Verifier{ + Client: ethClient, + } + return verifier +} + +func ethProviderInjector(ctx context.Context, syncPipeline *wrapper.SyncPipelineWrapper, verifier *eth.Verifier, simplehashProvider *simplehash.Provider) *multichain.EthereumProvider { ethereumProvider := &multichain.EthereumProvider{ ContractFetcher: simplehashProvider, ContractsCreatorFetcher: simplehashProvider, @@ -124,7 +132,7 @@ func ethProviderInjector(ctx context.Context, syncPipeline *wrapper.SyncPipeline TokensByTokenIdentifiersFetcher: syncPipeline, TokensIncrementalContractFetcher: syncPipeline, TokensIncrementalOwnerFetcher: syncPipeline, - Verifier: indexerProvider, + Verifier: verifier, } return ethereumProvider } diff --git a/service/auth/auth.go b/service/auth/auth.go index 3f2b6568b..86e12ea24 100644 --- a/service/auth/auth.go +++ b/service/auth/auth.go @@ -60,9 +60,6 @@ const AuthCookieKey = "GLRY_JWT" // RefreshCookieKey is the key used to store the refresh token in the cookie const RefreshCookieKey = "GLRY_REFRESH_JWT" -// ErrAddressSignatureMismatch is returned when the address signature does not match the address cryptographically -var ErrAddressSignatureMismatch = errors.New("address does not match signature") - // ErrNonceMismatch is returned when the nonce does not match the expected nonce var ErrNonceMismatch = errors.New("incorrect nonce input") diff --git a/service/eth/eth.go b/service/eth/eth.go index 1d68d013b..4d1c2cde1 100644 --- a/service/eth/eth.go +++ b/service/eth/eth.go @@ -7,13 +7,19 @@ import ( "math/big" "regexp" "strings" + "time" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" ens "github.com/wealdtech/go-ens/v3" "github.com/wealdtech/go-ens/v3/contracts/resolver" "github.com/wealdtech/go-ens/v3/contracts/reverseresolver" + "github.com/mikeydub/go-gallery/contracts" + "github.com/mikeydub/go-gallery/service/logger" "github.com/mikeydub/go-gallery/service/persist" "github.com/mikeydub/go-gallery/service/rpc" ) @@ -24,6 +30,11 @@ var ErrChainNotSupported = errors.New("chain not supported") var ErrUnknownTokenType = errors.New("unknown token type") var ErrNoAvatarRecord = errors.New("no avatar record set") +// ErrAddressSignatureMismatch is returned when the address signature does not match the address cryptographically +var ErrAddressSignatureMismatch = errors.New("address does not match signature") + +var eip1271MagicValue = [4]byte{0x16, 0x26, 0xBA, 0x7E} + // Regex for CAIP-19 asset type with required asset ID // https://github.com/ChainAgnostic/CAIPs/blob/master/CAIPs/caip-19.md var caip19AssetTypeWithAssetID = regexp.MustCompile( @@ -304,3 +315,102 @@ func (e EnsTokenRecord) Address() (persist.Address, error) { } return persist.Address(e.AssetReference), nil } + +type Verifier struct { + Client *ethclient.Client +} + +// VerifySignature will verify a signature using all available methods (eth_sign and personal_sign) +func (p *Verifier) VerifySignature(pCtx context.Context, pAddressStr persist.PubKey, pWalletType persist.WalletType, pMessage string, pSignatureStr string) (bool, error) { + + // personal_sign + validBool, err := verifySignature(pSignatureStr, pMessage, pAddressStr, pWalletType, true, p.Client) + + if !validBool || err != nil { + // eth_sign + validBool, err = verifySignature(pSignatureStr, pMessage, pAddressStr, pWalletType, false, p.Client) + } + + if err != nil { + return false, err + } + + return validBool, nil +} + +func verifySignature(pSignatureStr string, + pData string, + pAddress persist.PubKey, pWalletType persist.WalletType, + pUseDataHeaderBool bool, ec *ethclient.Client) (bool, error) { + + // eth_sign: + // - https://goethereumbook.org/signature-verify/ + // - http://man.hubwiz.com/docset/Ethereum.docset/Contents/Resources/Documents/eth_sign.html + // - sign(keccak256("\x19Ethereum Signed Message:\n" + len(message) + message))) + + var data string + if pUseDataHeaderBool { + data = fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(pData), pData) + } else { + data = pData + } + + switch pWalletType { + case persist.WalletTypeEOA: + dataHash := crypto.Keccak256Hash([]byte(data)) + + sig, err := hexutil.Decode(pSignatureStr) + if err != nil { + return false, err + } + // Ledger-produced signatures have v = 0 or 1 + if sig[64] == 0 || sig[64] == 1 { + sig[64] += 27 + } + v := sig[64] + if v != 27 && v != 28 { + return false, errors.New("invalid signature (V is not 27 or 28)") + } + sig[64] -= 27 + + sigPublicKeyECDSA, err := crypto.SigToPub(dataHash.Bytes(), sig) + if err != nil { + return false, err + } + + pubkeyAddressHexStr := crypto.PubkeyToAddress(*sigPublicKeyECDSA).Hex() + logger.For(nil).Infof("pubkeyAddressHexStr: %s", pubkeyAddressHexStr) + logger.For(nil).Infof("pAddress: %s", pAddress) + if !strings.EqualFold(pubkeyAddressHexStr, pAddress.String()) { + return false, ErrAddressSignatureMismatch + } + + publicKeyBytes := crypto.CompressPubkey(sigPublicKeyECDSA) + + signatureNoRecoverID := sig[:len(sig)-1] + + return crypto.VerifySignature(publicKeyBytes, dataHash.Bytes(), signatureNoRecoverID), nil + case persist.WalletTypeGnosis: + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + sigValidator, err := contracts.NewISignatureValidator(common.HexToAddress(pAddress.String()), ec) + if err != nil { + return false, err + } + + hashedData := crypto.Keccak256([]byte(data)) + var input [32]byte + copy(input[:], hashedData) + + result, err := sigValidator.IsValidSignature(&bind.CallOpts{Context: ctx}, input, []byte{}) + if err != nil { + logger.For(nil).WithError(err).Error("IsValidSignature") + return false, nil + } + + return result == eip1271MagicValue, nil + default: + return false, errors.New("wallet type not supported") + } +} diff --git a/service/multichain/indexer/indexer.go b/service/multichain/indexer/indexer.go deleted file mode 100644 index c2c7d9c83..000000000 --- a/service/multichain/indexer/indexer.go +++ /dev/null @@ -1,237 +0,0 @@ -package indexer - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "strings" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" - - "github.com/mikeydub/go-gallery/contracts" - "github.com/mikeydub/go-gallery/env" - "github.com/mikeydub/go-gallery/indexer" - "github.com/mikeydub/go-gallery/service/auth" - "github.com/mikeydub/go-gallery/service/logger" - "github.com/mikeydub/go-gallery/service/multichain" - "github.com/mikeydub/go-gallery/service/persist" - "github.com/mikeydub/go-gallery/util" -) - -var eip1271MagicValue = [4]byte{0x16, 0x26, 0xBA, 0x7E} - -// Provider is an the struct for retrieving data from the Ethereum blockchain -type Provider struct { - indexerBaseURL string - httpClient *http.Client - ethClient *ethclient.Client -} - -// NewProvider creates a new ethereum Provider -func NewProvider(httpClient *http.Client, ec *ethclient.Client) *Provider { - return &Provider{ - indexerBaseURL: env.GetString("INDEXER_HOST"), - httpClient: httpClient, - ethClient: ec, - } -} - -// GetContractByAddress retrieves an ethereum contract by address -func (d *Provider) GetContractByAddress(ctx context.Context, addr persist.Address) (multichain.ChainAgnosticContract, error) { - logger.For(ctx).Warn("ETH") - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/contracts/get?address=%s", d.indexerBaseURL, addr), nil) - if err != nil { - return multichain.ChainAgnosticContract{}, err - } - res, err := d.httpClient.Do(req) - if err != nil { - return multichain.ChainAgnosticContract{}, err - } - defer res.Body.Close() - - if res.StatusCode != 200 { - return multichain.ChainAgnosticContract{}, util.GetErrFromResp(res) - } - var contract indexer.GetContractOutput - err = json.NewDecoder(res.Body).Decode(&contract) - if err != nil { - return multichain.ChainAgnosticContract{}, err - } - - return contractToChainAgnostic(contract.Contract), nil - -} - -func (d *Provider) GetContractsByCreatorAddress(ctx context.Context, addr persist.Address) ([]multichain.ChainAgnosticContract, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/contracts/get?owner=%s", d.indexerBaseURL, addr), nil) - if err != nil { - return nil, err - } - res, err := d.httpClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - if res.StatusCode != 200 { - return nil, util.GetErrFromResp(res) - } - var contract indexer.GetContractsOutput - err = json.NewDecoder(res.Body).Decode(&contract) - if err != nil { - return nil, err - } - - out := make([]multichain.ChainAgnosticContract, len(contract.Contracts)) - for i, c := range contract.Contracts { - out[i] = contractToChainAgnostic(c) - } - - return out, nil -} - -// RefreshContract refreshes the metadata for a contract -func (d *Provider) RefreshContract(ctx context.Context, addr persist.Address) error { - input := indexer.UpdateContractMetadataInput{ - Address: persist.EthereumAddress(persist.ChainETH.NormalizeAddress(addr)), - } - - asJSON, err := json.Marshal(input) - if err != nil { - return err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s/contracts/refresh", d.indexerBaseURL), bytes.NewReader(asJSON)) - if err != nil { - return err - } - - res, err := d.httpClient.Do(req) - if err != nil { - return err - } - - defer res.Body.Close() - - if res.StatusCode != 200 { - return util.GetErrFromResp(res) - } - - return nil -} - -// VerifySignature will verify a signature using all available methods (eth_sign and personal_sign) -func (d *Provider) VerifySignature(pCtx context.Context, - pAddressStr persist.PubKey, pWalletType persist.WalletType, pMessage string, pSignatureStr string) (bool, error) { - - // personal_sign - validBool, err := verifySignature(pSignatureStr, pMessage, pAddressStr, pWalletType, true, d.ethClient) - - if !validBool || err != nil { - // eth_sign - validBool, err = verifySignature(pSignatureStr, pMessage, pAddressStr, pWalletType, false, d.ethClient) - } - - if err != nil { - return false, err - } - - return validBool, nil -} - -func verifySignature(pSignatureStr string, - pData string, - pAddress persist.PubKey, pWalletType persist.WalletType, - pUseDataHeaderBool bool, ec *ethclient.Client) (bool, error) { - - // eth_sign: - // - https://goethereumbook.org/signature-verify/ - // - http://man.hubwiz.com/docset/Ethereum.docset/Contents/Resources/Documents/eth_sign.html - // - sign(keccak256("\x19Ethereum Signed Message:\n" + len(message) + message))) - - var data string - if pUseDataHeaderBool { - data = fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(pData), pData) - } else { - data = pData - } - - switch pWalletType { - case persist.WalletTypeEOA: - dataHash := crypto.Keccak256Hash([]byte(data)) - - sig, err := hexutil.Decode(pSignatureStr) - if err != nil { - return false, err - } - // Ledger-produced signatures have v = 0 or 1 - if sig[64] == 0 || sig[64] == 1 { - sig[64] += 27 - } - v := sig[64] - if v != 27 && v != 28 { - return false, errors.New("invalid signature (V is not 27 or 28)") - } - sig[64] -= 27 - - sigPublicKeyECDSA, err := crypto.SigToPub(dataHash.Bytes(), sig) - if err != nil { - return false, err - } - - pubkeyAddressHexStr := crypto.PubkeyToAddress(*sigPublicKeyECDSA).Hex() - logger.For(nil).Infof("pubkeyAddressHexStr: %s", pubkeyAddressHexStr) - logger.For(nil).Infof("pAddress: %s", pAddress) - if !strings.EqualFold(pubkeyAddressHexStr, pAddress.String()) { - return false, auth.ErrAddressSignatureMismatch - } - - publicKeyBytes := crypto.CompressPubkey(sigPublicKeyECDSA) - - signatureNoRecoverID := sig[:len(sig)-1] - - return crypto.VerifySignature(publicKeyBytes, dataHash.Bytes(), signatureNoRecoverID), nil - case persist.WalletTypeGnosis: - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - sigValidator, err := contracts.NewISignatureValidator(common.HexToAddress(pAddress.String()), ec) - if err != nil { - return false, err - } - - hashedData := crypto.Keccak256([]byte(data)) - var input [32]byte - copy(input[:], hashedData) - - result, err := sigValidator.IsValidSignature(&bind.CallOpts{Context: ctx}, input, []byte{}) - if err != nil { - logger.For(nil).WithError(err).Error("IsValidSignature") - return false, nil - } - - return result == eip1271MagicValue, nil - default: - return false, errors.New("wallet type not supported") - } - -} - -func contractToChainAgnostic(contract persist.Contract) multichain.ChainAgnosticContract { - return multichain.ChainAgnosticContract{ - Address: persist.Address(contract.Address.String()), - Descriptors: multichain.ChainAgnosticContractDescriptors{ - Name: contract.Name.String(), - Symbol: contract.Symbol.String(), - OwnerAddress: persist.Address(util.FirstNonEmptyString(contract.OwnerAddress.String(), contract.CreatorAddress.String())), - }, - } -} diff --git a/service/multichain/simplehash/simplehash.go b/service/multichain/simplehash/simplehash.go index df9e6dbb0..205bf4041 100644 --- a/service/multichain/simplehash/simplehash.go +++ b/service/multichain/simplehash/simplehash.go @@ -120,10 +120,6 @@ func setContractAddress(u url.URL, chain persist.Chain, contract persist.Address return u } -func setContractID(u url.URL, contractID string) url.URL { - return setContractIDs(u, []string{contractID}) -} - func setContractIDs(u url.URL, contractIDs []string) url.URL { query := u.Query() query.Set("contract_ids", strings.Join(contractIDs, ",")) @@ -297,12 +293,6 @@ type getContractsByWalletResponse struct { Contracts []simplehashContractOwnership `json:"contracts"` } -type getCollectionByContractResponse struct { - NextCursor string `json:"next_cursor"` - Next string `json:"next"` - Collections []simplehashCollection `json:"collections"` -} - type getContractsByOwnerResponse struct { NextCursor string `json:"next_cursor"` Next string `json:"next"` @@ -315,18 +305,6 @@ type getContractsByDeployerResponse struct { Contracts []simplehashContractDetailed `json:"contracts"` } -type simpleHashCollector struct { - OwnerAddress string `json:"owner_address"` - DistinctNftsOwned int `json:"distinct_nfts_owned"` - TotalCopiesOwned int `json:"total_copies_owned"` -} - -type getCollectorsByContractResponse struct { - NextCursor string `json:"next_cursor"` - Next string `json:"next"` - TopCollectors []simpleHashCollector `json:"top_collectors"` -} - type simplehashTokenOwner struct { OwnerAddress string `json:"owner_address"` Quantity int `json:"quantity"` diff --git a/service/rpc/rpc.go b/service/rpc/rpc.go index 58c137762..b82591821 100644 --- a/service/rpc/rpc.go +++ b/service/rpc/rpc.go @@ -24,12 +24,9 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/everFinance/goar" "github.com/getsentry/sentry-go" @@ -46,7 +43,6 @@ import ( "github.com/mikeydub/go-gallery/service/rpc/arweave" "github.com/mikeydub/go-gallery/service/rpc/ipfs" "github.com/mikeydub/go-gallery/service/rpc/onchfs" - sentryutil "github.com/mikeydub/go-gallery/service/sentry" "github.com/mikeydub/go-gallery/service/tracing" "github.com/mikeydub/go-gallery/util" "github.com/mikeydub/go-gallery/util/retry" @@ -61,13 +57,9 @@ const ( defaultHTTPKeepAlive = 600 defaultHTTPMaxIdleConns = 250 defaultHTTPMaxIdleConnsPerHost = 250 - GethSocketOpName = "geth.wss" ) -var ( - defaultHTTPClient = newHTTPClientForRPC(true) - defaultMetricsHandler = metricsHandler{} -) +var defaultHTTPClient = newHTTPClientForRPC(true) // rateLimited is the content returned from an RPC call when rate limited. var rateLimited = "429 Too Many Requests" @@ -88,27 +80,6 @@ func (e ErrTokenURINotFound) Error() string { return e.Err.Error() } -// Transfer represents a Transfer from the RPC response -type Transfer struct { - BlockNumber persist.BlockNumber - From persist.EthereumAddress - To persist.EthereumAddress - TokenID persist.HexTokenID - TokenType persist.TokenType - Amount uint64 - ContractAddress persist.EthereumAddress - // These are geth types which are useful for getting more details about a transaction. - TxHash common.Hash - BlockHash common.Hash - TxIndex uint -} - -// TokenContractMetadata represents a token contract's metadata -type TokenContractMetadata struct { - Name string - Symbol string -} - // NewEthClient returns an ethclient.Client func NewEthClient() *ethclient.Client { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -132,19 +103,6 @@ func NewEthClient() *ethclient.Client { return ethclient.NewClient(client) } -// NewEthSocketClient returns a websocket client with request tracing enabled -func NewEthSocketClient() *ethclient.Client { - if strings.HasPrefix(env.GetString("RPC_URL"), "wss") { - log.Root().SetHandler(log.FilterHandler(func(r *log.Record) bool { - if reqID := valFromSlice(r.Ctx, "reqid"); reqID == nil || r.Msg != "Handled RPC response" { - return false - } - return true - }, defaultMetricsHandler)) - } - return NewEthClient() -} - func NewStorageClient(ctx context.Context) *storage.Client { opts := append([]option.ClientOption{}, option.WithScopes([]string{storage.ScopeFullControl}...)) @@ -178,28 +136,6 @@ func NewStorageClient(ctx context.Context) *storage.Client { return storageClient } -// metricsHandler traces RPC records that get logged by the RPC client -type metricsHandler struct{} - -// Log sends trace information to Sentry. -// Geth logs each response it receives in the handleImmediate method of handler: https://github.com/ethereum/go-ethereum/blob/master/rpc/handler.go -// We take advantage of this by configuring the client's root logger with a custom handler that sends a span to Sentry each time we get the log message. -func (h metricsHandler) Log(r *log.Record) error { - reqID := valFromSlice(r.Ctx, "reqid") - - // A useful context isn't passed to the log record, so we use the background context here. - ctx := context.Background() - span, _ := tracing.StartSpan(ctx, GethSocketOpName, "rpcCall", sentryutil.TransactionNameSafe("gethRPC")) - tracing.AddEventDataToSpan(span, map[string]interface{}{"reqID": reqID}) - defer tracing.FinishSpan(span) - - // Fix the duration to 100ms because there isn't a useful duration to use - span.EndTime = r.Time - span.StartTime = r.Time.Add(time.Millisecond * -100) - - return nil -} - // newHTTPClientForRPC returns an http.Client configured with default settings intended for RPC calls. func newHTTPClientForRPC(continueTrace bool, spanOptions ...sentry.SpanOption) *http.Client { // get x509 cert pool @@ -242,102 +178,6 @@ func newHTTPClientForRPC(continueTrace bool, spanOptions ...sentry.SpanOption) * } } -// GetBlockNumber returns the current block height. -func GetBlockNumber(ctx context.Context, ethClient *ethclient.Client) (uint64, error) { - return ethClient.BlockNumber(ctx) -} - -// RetryGetBlockNumber calls GetBlockNumber with backoff. -func RetryGetBlockNumber(ctx context.Context, ethClient *ethclient.Client) (uint64, error) { - var height uint64 - var err error - for i := 0; i < retry.DefaultRetry.MaxRetries; i++ { - height, err = GetBlockNumber(ctx, ethClient) - if !isRateLimitedError(err) { - break - } - <-time.After(retry.WaitTime(retry.DefaultRetry.MinWait, retry.DefaultRetry.MaxRetries, i)) - } - return height, err -} - -// GetLogs returns log events for the given block range and query. -func GetLogs(ctx context.Context, ethClient *ethclient.Client, query ethereum.FilterQuery) ([]types.Log, error) { - return ethClient.FilterLogs(ctx, query) -} - -// RetryGetLogs calls GetLogs with backoff. -func RetryGetLogs(ctx context.Context, ethClient *ethclient.Client, query ethereum.FilterQuery) ([]types.Log, error) { - logs := make([]types.Log, 0) - var err error - for i := 0; i < retry.DefaultRetry.MaxRetries; i++ { - logs, err = GetLogs(ctx, ethClient, query) - if !isRateLimitedError(err) { - break - } - <-time.After(retry.WaitTime(retry.DefaultRetry.MinWait, retry.DefaultRetry.MaxWait, i)) - } - return logs, err -} - -// GetTransaction returns the transaction of the given hash. -func GetTransaction(ctx context.Context, ethClient *ethclient.Client, txHash common.Hash) (*types.Transaction, bool, error) { - return ethClient.TransactionByHash(ctx, txHash) -} - -// RetryGetTransaction calls GetTransaction with backoff. -func RetryGetTransaction(ctx context.Context, ethClient *ethclient.Client, txHash common.Hash, r retry.Retry) (*types.Transaction, bool, error) { - var tx *types.Transaction - var pending bool - var err error - for i := 0; i < r.MaxRetries; i++ { - tx, pending, err = GetTransaction(ctx, ethClient, txHash) - if !isRateLimitedError(err) { - break - } - <-time.After(retry.WaitTime(r.MinWait, r.MaxWait, i)) - } - return tx, pending, err -} - -// GetTokenContractMetadata returns the metadata for a given contract (without URI) -func GetTokenContractMetadata(ctx context.Context, address persist.EthereumAddress, ethClient *ethclient.Client) (*TokenContractMetadata, error) { - contract := address.Address() - instance, err := contracts.NewIERC721MetadataCaller(contract, ethClient) - if err != nil { - return nil, err - } - - name, err := instance.Name(&bind.CallOpts{ - Context: ctx, - }) - if err != nil { - return nil, err - } - symbol, err := instance.Symbol(&bind.CallOpts{ - Context: ctx, - }) - if err != nil { - return nil, err - } - - return &TokenContractMetadata{Name: name, Symbol: symbol}, nil -} - -// RetryGetTokenContractMetaData calls GetTokenContractMetadata with backoff. -func RetryGetTokenContractMetadata(ctx context.Context, contractAddress persist.EthereumAddress, ethClient *ethclient.Client) (*TokenContractMetadata, error) { - var metadata *TokenContractMetadata - var err error - for i := 0; i < retry.DefaultRetry.MaxRetries; i++ { - metadata, err = GetTokenContractMetadata(ctx, contractAddress, ethClient) - if !isRateLimitedError(err) { - break - } - <-time.After(retry.WaitTime(retry.DefaultRetry.MinWait, retry.DefaultRetry.MaxWait, i)) - } - return metadata, err -} - // GetMetadataFromURI parses and returns the NFT metadata for a given token URI func GetMetadataFromURI(ctx context.Context, turi persist.TokenURI, ipfsClient *shell.Shell, arweaveClient *goar.Client) (persist.TokenMetadata, error) { @@ -355,7 +195,6 @@ func GetMetadataFromURI(ctx context.Context, turi persist.TokenURI, ipfsClient * } return meta, nil - } // GetDataFromURIAsReader calls URI and returns the data as an unread reader with the headers pre-read. @@ -728,72 +567,6 @@ func GetOwnerOfERC721Token(ctx context.Context, pContractAddress persist.Ethereu return persist.EthereumAddress(strings.ToLower(owner.String())), nil } -// GetContractCreator returns the address of the contract creator -func GetContractCreator(ctx context.Context, contractAddress persist.EthereumAddress, ethClient *ethclient.Client) (persist.EthereumAddress, error) { - highestBlock, err := ethClient.BlockNumber(ctx) - if err != nil { - return "", fmt.Errorf("error getting highest block: %s", err.Error()) - } - _, err = ethClient.CodeAt(ctx, contractAddress.Address(), big.NewInt(int64(highestBlock))) - if err != nil { - return "", fmt.Errorf("error getting code at: %s", err.Error()) - } - lowestBlock := uint64(0) - - for lowestBlock <= highestBlock { - midBlock := uint64((highestBlock + lowestBlock) / 2) - codeAt, err := ethClient.CodeAt(ctx, contractAddress.Address(), big.NewInt(int64(midBlock))) - if err != nil { - return "", fmt.Errorf("error getting code at: %s", err.Error()) - } - if len(codeAt) > 0 { - highestBlock = midBlock - } else { - lowestBlock = midBlock - } - - if lowestBlock+1 == highestBlock { - break - } - } - block, err := ethClient.BlockByNumber(ctx, big.NewInt(int64(highestBlock))) - if err != nil { - return "", fmt.Errorf("error getting block: %s", err.Error()) - } - txs := block.Transactions() - for _, tx := range txs { - receipt, err := ethClient.TransactionReceipt(ctx, tx.Hash()) - if err != nil { - return "", fmt.Errorf("error getting transaction receipt: %s", err.Error()) - } - if receipt.ContractAddress == contractAddress.Address() { - msg, err := tx.AsMessage(types.HomesteadSigner{}, nil) - if err != nil { - return "", fmt.Errorf("error getting message: %s", err.Error()) - } - return persist.EthereumAddress(fmt.Sprintf("0x%s", strings.ToLower(msg.From().String()))), nil - } - } - return "", fmt.Errorf("could not find contract creator") -} - -// GetContractOwner returns the address of the contract owner -func GetContractOwner(ctx context.Context, contractAddress persist.EthereumAddress, ethClient *ethclient.Client) (persist.EthereumAddress, error) { - instance, err := contracts.NewOwnableCaller(contractAddress.Address(), ethClient) - if err != nil { - return "", err - } - - owner, err := instance.Owner(&bind.CallOpts{ - Context: ctx, - }) - if err != nil { - return "", err - } - - return persist.EthereumAddress(strings.ToLower(owner.String())), nil -} - func GetArweaveDataHTTPReader(ctx context.Context, id string) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://arweave.net/%s", id), nil) if err != nil { @@ -827,16 +600,6 @@ func GetArweaveDataHTTP(ctx context.Context, id string) ([]byte, error) { return data, nil } -// valFromSlice returns the value from a slice formatted as [key val key val ...] -func valFromSlice(s []interface{}, keyName string) interface{} { - for i, key := range s { - if key == keyName { - return s[i+1] - } - } - return nil -} - func isRateLimitedError(err error) bool { if err != nil && strings.Contains(err.Error(), rateLimited) { return true diff --git a/sqlc.yaml b/sqlc.yaml index 93f68ffb4..a774113c8 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -4,38 +4,6 @@ plugins: process: cmd: db/bin/sqlc-gen-json sql: - # Indexer model gen - - schema: 'db/migrations/indexer' - queries: 'db/queries/indexer' - engine: 'postgresql' - gen: - go: - package: 'indexerdb' - out: 'db/gen/indexerdb' - sql_package: 'pgx/v4' - output_db_file_name: 'db_gen.go' - output_models_file_name: 'models_gen.go' - overrides: - # Overrides are prioritized from top to bottom, so if we need to override one of the * entries (like *.id), - # the override should come _before_ the * entry - - - column: 'reprocess_jobs.id' - go_type: 'int' - - column: 'contracts.owner_method' - go_type: 'github.com/mikeydub/go-gallery/service/persist.ContractOwnerMethod' - - column: 'contracts.*address' - go_type: 'github.com/mikeydub/go-gallery/service/persist.EthereumAddress' - # Wildcards - # Note: to override one of these wildcard entries, add a more specific entry (like some_table.id) above - - column: '*.id' - go_type: 'github.com/mikeydub/go-gallery/service/persist.DBID' - - column: '*.block_start' - go_type: 'github.com/mikeydub/go-gallery/service/persist.BlockNumber' - - column: '*.block_end' - go_type: 'github.com/mikeydub/go-gallery/service/persist.BlockNumber' - - column: '*.chain' - go_type: 'github.com/mikeydub/go-gallery/service/persist.Chain' - # Backend model gen - schema: - 'db/migrations/core' diff --git a/tokenprocessing/process.go b/tokenprocessing/process.go index ef83bd198..a96840403 100644 --- a/tokenprocessing/process.go +++ b/tokenprocessing/process.go @@ -732,10 +732,6 @@ func (m *highlightTracker) setStatusMediaProcessing(ctx context.Context, claimID }) } -func (m *highlightTracker) setStatusTxPending(ctx context.Context, claimID persist.DBID) (db.HighlightMintClaim, error) { - return m.setStatus(ctx, claimID, highlight.ClaimStatusTxPending, "") -} - func (m *highlightTracker) setStatusFailed(ctx context.Context, claimID persist.DBID, err error) (db.HighlightMintClaim, error) { return m.setStatus(ctx, claimID, highlight.ClaimStatusFailedInternal, err.Error()) }