diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 30a38fa..33df29e 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -1,31 +1,44 @@ name: Gateway Conformance on: + workflow_dispatch: push: branches: - main pull_request: + paths-ignore: + - '**/*.md' + +env: + GATEWAY_CONFORMANCE_TEST: true # rainbow preset for conformance testing + KUBO_VER: 'v0.28.0' # kubo daemon used as no-libp2p-remote-* backend + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true jobs: - gateway-conformance: + test: runs-on: ubuntu-latest + strategy: + matrix: + backend: ["libp2p-bitswap", "remote-block-gw", "remote-car-gw"] + steps: # 1. Start the Kubo gateway - - name: Setup Go - uses: actions/setup-go@v5 + - name: Install Kubo + uses: ipfs/download-ipfs-distribution-action@v1 with: - go-version: 1.21.x + name: kubo + version: "${{ env.KUBO_VER }}" - - name: Install Kubo gateway from source - #uses: ipfs/download-ipfs-distribution-action@v1 - run: | - go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0-rc1 - name: Setup kubo config run: | ipfs init --profile=test ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" ipfs config --json Gateway.ExposeRoutingAPI true + ipfs config Routing.Type "autoclient" # 2. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures @@ -33,10 +46,7 @@ jobs: with: output: fixtures - - name: Start Kubo gateway - uses: ipfs/start-ipfs-daemon-action@v1 - - # 3. Populate the Kubo gateway with the gateway-conformance fixtures + # 3. Populate the Kubo node with the gateway-conformance fixtures - name: Import fixtures run: | # Import car files @@ -56,7 +66,15 @@ jobs: export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + - name: Start Kubo gateway + uses: ipfs/start-ipfs-daemon-action@v1 + # 4. Build rainbow + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + - name: Checkout rainbow uses: actions/checkout@v4 with: @@ -65,16 +83,47 @@ jobs: run: go build working-directory: rainbow - # 5. Start rainbow - - name: Start rainbow + # 5. Start rainbow variant + - name: Start rainbow (libp2p and bitswap) + if: ${{ matrix.backend == 'libp2p-bitswap' }} env: - GATEWAY_CONFORMANCE_TEST: true + RAINBOW_DHT_ROUTING: off + RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080 run: | - # get kubo peerID + # set up peering with kubo to ensure fixtures can be found fast kuboNodeMultiaddr=$(ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) - # run gw - ./rainbow --http-routers=http://127.0.0.1:8080 --dht-routing=off --peering=$kuboNodeMultiaddr & + ./rainbow --peering=$kuboNodeMultiaddr & + working-directory: rainbow + + # 5. Start rainbow variant + - name: Start rainbow (no libp2p, remote block gateway) + if: ${{ matrix.backend == 'remote-block-gw' }} + env: + RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080 + RAINBOW_REMOTE_BACKENDS_MODE: block + RAINBOW_REMOTE_BACKENDS_IPNS: true + RAINBOW_LIBP2P: false + RAINBOW_BITSWAP: false + RAINBOW_DHT_ROUTING: off + RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080 + run: | + ./rainbow & + working-directory: rainbow + # + # 5. Start rainbow variant + - name: Start rainbow (no libp2p, remote car gateway) + if: ${{ matrix.backend == 'remote-car-gw' }} + env: + RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080 + RAINBOW_REMOTE_BACKENDS_MODE: car + RAINBOW_REMOTE_BACKENDS_IPNS: true + RAINBOW_LIBP2P: false + RAINBOW_BITSWAP: false + RAINBOW_DHT_ROUTING: off + RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080 + run: | + ./rainbow & working-directory: rainbow # 6. Run the gateway-conformance tests @@ -92,7 +141,7 @@ jobs: # # only-if-cached: rainbow does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip 'TestGatewayCache/.*_for_/ipfs/_with_only-if-cached_succeeds_when_in_local_datastore' + args: -skip 'TestGatewayCache/.*_with_only-if-cached_succeeds_when_in_local_datastore' # 7. Upload the results - name: Upload MD summary @@ -102,11 +151,11 @@ jobs: if: failure() || success() uses: actions/upload-artifact@v4 with: - name: gateway-conformance.html + name: ${{ matrix.backend }}_gateway-conformance.html path: output.html - name: Upload JSON report if: failure() || success() uses: actions/upload-artifact@v4 with: - name: gateway-conformance.json + name: ${{ matrix.backend }}_gateway-conformance.json path: output.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4b7fa..4a23911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ The following emojis are used to highlight certain changes: ### Added +- Now supports remote backends (using RAW block or CAR requests) via `--remote-backends` (`RAINBOW_REMOTE_BACKENDS`). + ### Changed ### Removed diff --git a/docs/environment-variables.md b/docs/environment-variables.md index a943ec7..977bad5 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -6,15 +6,22 @@ - [`RAINBOW_GATEWAY_DOMAINS`](#rainbow_gateway_domains) - [`RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`](#rainbow_subdomain_gateway_domains) - [`RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`](#rainbow_trustless_gateway_domains) + - [`RAINBOW_DATADIR`](#rainbow_datadir) - [`RAINBOW_GC_INTERVAL`](#rainbow_gc_interval) - [`RAINBOW_GC_THRESHOLD`](#rainbow_gc_threshold) - [`RAINBOW_IPNS_MAX_CACHE_TTL`](#rainbow_ipns_max_cache_ttl) - [`RAINBOW_PEERING`](#rainbow_peering) - [`RAINBOW_SEED`](#rainbow_seed) - [`RAINBOW_SEED_INDEX`](#rainbow_seed_index) + - [`RAINBOW_DHT_ROUTING`](#rainbow_dht_routing) + - [`RAINBOW_HTTP_ROUTERS`](#rainbow_http_routers) +- [Experiments](#experiments) - [`RAINBOW_SEED_PEERING`](#rainbow_seed_peering) - [`RAINBOW_SEED_PEERING_MAX_INDEX`](#rainbow_seed_peering_max_index) - [`RAINBOW_PEERING_SHARED_CACHE`](#rainbow_peering_shared_cache) + - [`RAINBOW_REMOTE_BACKENDS`](#rainbow_remote_backends) + - [`RAINBOW_REMOTE_BACKENDS_MODE`](#rainbow_remote_backends_mode) + - [`RAINBOW_REMOTE_BACKENDS_IPNS`](#rainbow_remote_backends_ipns) - [Logging](#logging) - [`GOLOG_LOG_LEVEL`](#golog_log_level) - [`GOLOG_LOG_FMT`](#golog_log_fmt) @@ -69,6 +76,12 @@ when request comes with the `Host` header set to `trustless-gateway.link`. Default: none (`Host` is ignored and gateway at `127.0.0.1` supports both deserialized and verifiable response types) +### `RAINBOW_DATADIR` + +Directory for persistent data (keys, blocks, denylists) + +Default: not set (uses the current directory) + ### `RAINBOW_GC_INTERVAL` The interval at which the garbage collector will be called. This is given as a string that corresponds to the duration of the interval. Set 0 to disable. @@ -121,6 +134,20 @@ Index to derivate the PeerID identity from `RAINBOW_SEED`. Default: not set +### `RAINBOW_DHT_ROUTING` + +Control the type of Amino DHT client used for for routing. Options are `accelerated`, `standard` and `off`. + +Default: `accelerated` + +### `RAINBOW_HTTP_ROUTERS` + +HTTP servers with /routing/v1 endpoints to use for delegated routing (comma-separated). + +Default: `https://cid.contact` + +## Experiments + ### `RAINBOW_SEED_PEERING` > [!WARNING] @@ -162,6 +189,34 @@ queries from these safelisted peers, serving locally cached blocks if requested. Default: `false` (no cache sharing) +### `RAINBOW_REMOTE_BACKENDS` + +> [!WARNING] +> Experimental feature, forces setting `RAINBOW_LIBP2P=false`. + +URL(s) of of remote [trustless gateways](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) +to use as backend instead of libp2p node with Bitswap. + +Default: not set + +### `RAINBOW_REMOTE_BACKENDS_MODE` + +Requires `RAINBOW_REMOTE_BACKENDS` to be set. + +Controls how requests to remote backend are made. + +- `block` will use [application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw) to fetch raw blocks one by one +- `car` will use [application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) and [IPIP-402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) for fetching multiple blocks per request + +Default: `block` + +### `RAINBOW_REMOTE_BACKENDS_IPNS` + +Controls whether to fetch IPNS Records ([`application/vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record)) from trustless gateway defined in `RAINBOW_REMOTE_BACKENDS`. +This is done in addition to other routing systems, such as `RAINBOW_DHT_ROUTING` or `RAINBOW_HTTP_ROUTERS` (if also enabled). + +Default: `true` + ## Logging ### `GOLOG_LOG_LEVEL` diff --git a/gc.go b/gc.go index 4e875ec..487dbd2 100644 --- a/gc.go +++ b/gc.go @@ -8,8 +8,13 @@ import ( ) // GC is a really stupid simple algorithm where we just delete things until -// weve deleted enough things +// we've deleted enough things. It is no-op if the current setup does not have +// a (local) blockstore. func (nd *Node) GC(ctx context.Context, todelete int64) error { + if nd.blockstore == nil { + return nil + } + keys, err := nd.blockstore.AllKeysChan(ctx) if err != nil { return err diff --git a/gc_test.go b/gc_test.go index 1a7cb8d..8e44c78 100644 --- a/gc_test.go +++ b/gc_test.go @@ -12,7 +12,9 @@ import ( func TestPeriodicGC(t *testing.T) { t.Parallel() - gnd := mustTestNode(t, Config{}) + gnd := mustTestNode(t, Config{ + Bitswap: true, + }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/handler_test.go b/handler_test.go index 120d775..c246166 100644 --- a/handler_test.go +++ b/handler_test.go @@ -12,6 +12,7 @@ func TestTrustless(t *testing.T) { t.Parallel() ts, gnd := mustTestServer(t, Config{ + Bitswap: true, TrustlessGatewayDomains: []string{"trustless.com"}, }) diff --git a/handlers.go b/handlers.go index c7d938c..9040649 100644 --- a/handlers.go +++ b/handlers.go @@ -85,12 +85,27 @@ func withRequestLogger(next http.Handler) http.Handler { } func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { - backend, err := gateway.NewBlocksBackend( - nd.bsrv, + var ( + backend gateway.IPFSBackend + err error + ) + + options := []gateway.BackendOption{ gateway.WithValueStore(nd.vs), gateway.WithNameSystem(nd.ns), - gateway.WithResolver(nd.resolver), - ) + gateway.WithResolver(nd.resolver), // May be nil, but that is fine. + } + + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendCAR { + var fetcher gateway.CarFetcher + fetcher, err = gateway.NewRemoteCarFetcher(cfg.RemoteBackends, nil) + if err != nil { + return nil, err + } + backend, err = gateway.NewCarBackend(fetcher, options...) + } else { + backend, err = gateway.NewBlocksBackend(nd.bsrv, options...) + } if err != nil { return nil, err } diff --git a/main.go b/main.go index f832e80..7f09cef 100644 --- a/main.go +++ b/main.go @@ -156,23 +156,29 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_GC_THRESHOLD"}, Usage: "Percentage of how much of the disk free space must be available", }, + &cli.BoolFlag{ + Name: "libp2p", + Value: true, + EnvVars: []string{"RAINBOW_LIBP2P"}, + Usage: "Controls if a local libp2p node is used (useful for testing or when remote backend is used instead)", + }, &cli.IntFlag{ - Name: "connmgr-low", + Name: "libp2p-connmgr-low", Value: 100, - EnvVars: []string{"RAINBOW_CONNMGR_LOW"}, - Usage: "Minimum number of connections to keep", + EnvVars: []string{"RAINBOW_LIBP2P_CONNMGR_LOW"}, + Usage: "Number of connections that the connection manager will trim down to during GC", }, &cli.IntFlag{ - Name: "connmgr-high", + Name: "libp2p-connmgr-high", Value: 3000, - EnvVars: []string{"RAINBOW_CONNMGR_HIGH"}, - Usage: "Maximum number of connections to keep", + EnvVars: []string{"RAINBOW_LIBP2P_CONNMGR_HIGH"}, + Usage: "Number of libp2p connections that, when exceeded, will trigger a connection GC operation", }, &cli.DurationFlag{ - Name: "connmgr-grace", + Name: "libp2p-connmgr-grace", Value: time.Minute, - EnvVars: []string{"RAINBOW_CONNMGR_GRACE_PERIOD"}, - Usage: "Minimum connection TTL", + EnvVars: []string{"RAINBOW_LIBP2P_CONNMGR_GRACE_PERIOD"}, + Usage: "How long new libp2p connections are immune from being closed by the connection manager", }, &cli.IntFlag{ Name: "inmem-block-cache", @@ -181,16 +187,16 @@ Generate an identity seed and launch a gateway: Usage: "Size of the in-memory block cache (currently only used for badger). 0 to disable (disables compression on disk too)", }, &cli.Uint64Flag{ - Name: "max-memory", + Name: "libp2p-max-memory", Value: 0, - EnvVars: []string{"RAINBOW_MAX_MEMORY"}, - Usage: "Max memory to use. Defaults to 85% of the system's available RAM", + EnvVars: []string{"RAINBOW_LIBP2P_MAX_MEMORY"}, + Usage: "Max memory to use for libp2p node. Defaults to 85% of the system's available RAM", }, &cli.Uint64Flag{ - Name: "max-fd", + Name: "libp2p-max-fd", Value: 0, - EnvVars: []string{"RAINBOW_MAX_FD"}, - Usage: "Maximum number of file descriptors. Defaults to 50% of the process' limit", + EnvVars: []string{"RAINBOW_LIBP2P_MAX_FD"}, + Usage: "Maximum number of file descriptors used by libp2p node. Defaults to 50% of the process' limit", }, &cli.StringSliceFlag{ Name: "http-routers", @@ -228,13 +234,13 @@ Generate an identity seed and launch a gateway: Name: "peering", Value: cli.NewStringSlice(), EnvVars: []string{"RAINBOW_PEERING"}, - Usage: "Multiaddresses of peers to stay connected to (comma-separated)", + Usage: "(EXPERIMENTAL) Multiaddresses of peers to stay connected to and ask for missing blocks over Bitswap (comma-separated)", }, &cli.BoolFlag{ Name: "peering-shared-cache", Value: false, EnvVars: []string{"RAINBOW_PEERING_SHARED_CACHE"}, - Usage: "(EXPERIMENTAL: increased network I/O) Enable sharing of local cache to peers safe-listed with --peering. Rainbow will respond to Bitswap queries from these peers, serving locally cached data as needed.", + Usage: "(EXPERIMENTAL: increased network I/O) Enable sharing of local cache to peers safe-listed with --peering. Rainbow will respond to Bitswap queries from these peers, serving locally cached data as needed (requires --bitswap=true).", }, &cli.StringFlag{ Name: "blockstore", @@ -248,6 +254,38 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_IPNS_MAX_CACHE_TTL"}, Usage: "Optional cap on caching duration for IPNS/DNSLink lookups. Set to 0 to respect original TTLs", }, + &cli.BoolFlag{ + Name: "bitswap", + Value: true, + EnvVars: []string{"RAINBOW_BITSWAP"}, + Usage: "Controls if Bitswap is enabled (useful for testing or when remote backend is used instead)", + }, + &cli.StringSliceFlag{ + Name: "remote-backends", + Value: cli.NewStringSlice(), + EnvVars: []string{"RAINBOW_REMOTE_BACKENDS"}, + Usage: "(EXPERIMENTAL) Trustless gateways to use as backend instead of Bitswap (comma-separated urls)", + }, + &cli.BoolFlag{ + Name: "remote-backends-ipns", + Value: true, + EnvVars: []string{"RAINBOW_REMOTE_BACKENDS_IPNS"}, + Usage: "(EXPERIMENTAL) Whether to fetch IPNS Records (application/vnd.ipfs.ipns-record) from the remote backends", + }, + &cli.StringFlag{ + Name: "remote-backends-mode", + Value: "block", + EnvVars: []string{"RAINBOW_REMOTE_BACKENDS_MODE"}, + Usage: "(EXPERIMENTAL) Whether to fetch raw blocks or CARs from the remote backends. Options are 'block' or 'car'", + Action: func(ctx *cli.Context, s string) error { + switch RemoteBackendMode(s) { + case RemoteBackendBlock, RemoteBackendCAR: + return nil + default: + return errors.New("invalid value for --remote-backend-mode: use 'block' or 'car'") + } + }, + }, } app.Commands = []*cli.Command{ @@ -289,57 +327,77 @@ share the same seed as long as the indexes are different. var seed string var priv crypto.PrivKey + var peeringAddrs []peer.AddrInfo + var index int var err error - credDir := os.Getenv("CREDENTIALS_DIRECTORY") - secretsDir := ddir + bitswap := cctx.Bool("bitswap") + dhtRouting := DHTRouting(cctx.String("dht-routing")) + seedPeering := cctx.Bool("seed-peering") - if len(credDir) > 0 { - secretsDir = credDir - } + libp2p := cctx.Bool("libp2p") - // attempt to read seed from disk - seedBytes, err := os.ReadFile(filepath.Join(secretsDir, "seed")) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - // set seed from command line or env-var - seed = cctx.String("seed") - } else { - return fmt.Errorf("error reading seed credentials: %w", err) - } - } else { - seed = strings.TrimSpace(string(seedBytes)) + // as a convenience to the end user, and to reduce confusion + // libp2p is disabled when remote backends are defined + remoteBackends := cctx.StringSlice("remote-backends") + if len(remoteBackends) > 0 { + fmt.Printf("RAINBOW_REMOTE_BACKENDS set, forcing RAINBOW_LIBP2P=false\n") + libp2p = false + bitswap = false + dhtRouting = DHTOff } - index := cctx.Int("seed-index") - if len(seed) > 0 && index >= 0 { - fmt.Printf("Deriving identity from seed[%d]\n", index) - priv, err = deriveKey(seed, deriveKeyInfo(index)) - } else { - fmt.Println("Setting identity from libp2p.key") - keyFile := filepath.Join(secretsDir, "libp2p.key") - priv, err = loadOrInitPeerKey(keyFile) - } - if err != nil { - return err - } + // Only load secrets if we need Libp2p. + if libp2p { + credDir := os.Getenv("CREDENTIALS_DIRECTORY") + secretsDir := ddir - var peeringAddrs []peer.AddrInfo - for _, maStr := range cctx.StringSlice("peering") { - if len(seed) > 0 && index >= 0 { - maStr, err = replaceRainbowSeedWithPeer(maStr, seed) - if err != nil { - return err + if len(credDir) > 0 { + secretsDir = credDir + } + + // attempt to read seed from disk + seedBytes, err := os.ReadFile(filepath.Join(secretsDir, "seed")) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // set seed from command line or env-var + seed = cctx.String("seed") + } else { + return fmt.Errorf("error reading seed credentials: %w", err) } - } else if rainbowSeedRegex.MatchString(maStr) { - return fmt.Errorf("unable to peer with %q without defining --seed-index of this instance first", maStr) + } else { + seed = strings.TrimSpace(string(seedBytes)) } - ai, err := peer.AddrInfoFromString(maStr) + index = cctx.Int("seed-index") + if len(seed) > 0 && index >= 0 { + fmt.Println("Deriving identity from seed") + priv, err = deriveKey(seed, deriveKeyInfo(index)) + } else { + fmt.Println("Setting identity from libp2p.key") + keyFile := filepath.Join(secretsDir, "libp2p.key") + priv, err = loadOrInitPeerKey(keyFile) + } if err != nil { return err } - peeringAddrs = append(peeringAddrs, *ai) + + for _, maStr := range cctx.StringSlice("peering") { + if len(seed) > 0 && index >= 0 { + maStr, err = replaceRainbowSeedWithPeer(maStr, seed) + if err != nil { + return err + } + } else if rainbowSeedRegex.MatchString(maStr) { + return fmt.Errorf("unable to peer with %q without defining --seed-index of this instance first", maStr) + } + + ai, err := peer.AddrInfoFromString(maStr) + if err != nil { + return err + } + peeringAddrs = append(peeringAddrs, *ai) + } } cfg := Config{ @@ -348,30 +406,40 @@ share the same seed as long as the indexes are different. GatewayDomains: cctx.StringSlice("gateway-domains"), SubdomainGatewayDomains: cctx.StringSlice("subdomain-gateway-domains"), TrustlessGatewayDomains: cctx.StringSlice("trustless-gateway-domains"), - ConnMgrLow: cctx.Int("connmgr-low"), - ConnMgrHi: cctx.Int("connmgr-high"), - ConnMgrGrace: cctx.Duration("connmgr-grace"), - MaxMemory: cctx.Uint64("max-memory"), - MaxFD: cctx.Int("max-fd"), + ConnMgrLow: cctx.Int("libp2p-connmgr-low"), + ConnMgrHi: cctx.Int("libp2p-connmgr-high"), + ConnMgrGrace: cctx.Duration("libp2p-connmgr-grace"), + MaxMemory: cctx.Uint64("libp2p-max-memory"), + MaxFD: cctx.Int("libp2p-max-fd"), InMemBlockCache: cctx.Int64("inmem-block-cache"), RoutingV1Endpoints: cctx.StringSlice("http-routers"), - DHTRouting: DHTRouting(cctx.String("dht-routing")), + DHTRouting: dhtRouting, DHTSharedHost: cctx.Bool("dht-shared-host"), + Bitswap: bitswap, IpnsMaxCacheTTL: cctx.Duration("ipns-max-cache-ttl"), DenylistSubs: cctx.StringSlice("denylists"), Peering: peeringAddrs, PeeringSharedCache: cctx.Bool("peering-shared-cache"), Seed: seed, SeedIndex: index, - SeedPeering: cctx.Bool("seed-peering"), + SeedPeering: seedPeering, SeedPeeringMaxIndex: cctx.Int("seed-peering-max-index"), + RemoteBackends: remoteBackends, + RemoteBackendsIPNS: cctx.Bool("remote-backends-ipns"), + RemoteBackendMode: RemoteBackendMode(cctx.String("remote-backends-mode")), GCInterval: cctx.Duration("gc-interval"), GCThreshold: cctx.Float64("gc-threshold"), } - goLog.Debugf("Rainbow config: %+v", cfg) + var gnd *Node - gnd, err := Setup(cctx.Context, cfg, priv, cdns) + goLog.Infof("Rainbow config: %+v", cfg) + + if libp2p { + gnd, err = SetupWithLibp2p(cctx.Context, cfg, priv, cdns) + } else { + gnd, err = SetupNoLibp2p(cctx.Context, cfg, cdns) + } if err != nil { return err } @@ -389,11 +457,14 @@ share the same seed as long as the indexes are different. Handler: handler, } - pid, err := peer.IDFromPublicKey(priv.GetPublic()) - if err != nil { - return err + fmt.Printf("Starting %s %s\n", name, version) + if priv != nil { + pid, err := peer.IDFromPublicKey(priv.GetPublic()) + if err != nil { + return err + } + fmt.Printf("PeerID: %s\n\n", pid) } - fmt.Printf("PeerID: %s\n\n", pid) registerVersionMetric(version) registerIpfsNodeCollector(gnd) @@ -424,6 +495,8 @@ share the same seed as long as the indexes are different. printIfListConfigured(" RAINBOW_GATEWAY_DOMAINS = ", cfg.GatewayDomains) printIfListConfigured(" RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS = ", cfg.SubdomainGatewayDomains) printIfListConfigured(" RAINBOW_TRUSTLESS_GATEWAY_DOMAINS = ", cfg.TrustlessGatewayDomains) + printIfListConfigured(" RAINBOW_HTTP_ROUTERS = ", cfg.RoutingV1Endpoints) + printIfListConfigured(" RAINBOW_REMOTE_BACKENDS = ", cfg.RemoteBackends) fmt.Printf("\n") fmt.Printf("CTL endpoint listening at http://%s\n", ctlListen) diff --git a/main_test.go b/main_test.go index 5b0d9b3..33c0354 100644 --- a/main_test.go +++ b/main_test.go @@ -63,7 +63,7 @@ func mustTestNodeWithKey(t *testing.T, cfg Config, sk ic.PrivKey) *Node { _ = cdns.Close() }) - nd, err := Setup(ctx, cfg, sk, cdns) + nd, err := SetupWithLibp2p(ctx, cfg, sk, cdns) require.NoError(t, err) return nd } diff --git a/rcmgr.go b/rcmgr.go index e5aff9b..e9cc9b7 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -1,8 +1,6 @@ package main import ( - "log" - "github.com/dustin/go-humanize" "github.com/pbnjay/memory" @@ -166,7 +164,7 @@ func makeResourceManagerConfig(maxMemory uint64, maxFD int, connMgrHighWater int partialLimits.System.ConnsInbound = rcmgr.LimitVal(maxInboundConns) } - log.Printf(` + goLog.Infof(` go-libp2p Resource Manager limits based on: - --max-memory: %s @@ -248,7 +246,7 @@ func makeSeparateDHTClientResourceManagerConfig(maxMemory uint64, maxFD int) (li // Anything in scalingLimitConfig that wasn't defined in partialLimits above will be added (e.g., libp2p's default service limits). partialLimits = partialLimits.Build(scalingLimitConfig.Scale(int64(maxMemory), maxFD)).ToPartialLimitConfig() - log.Printf(` + goLog.Infof(` go-libp2p Separate DHT Resource Manager limits based on: - --max-memory: %s diff --git a/setup.go b/setup.go index c91af8e..f86c679 100644 --- a/setup.go +++ b/setup.go @@ -16,7 +16,7 @@ import ( nopfsipfs "github.com/ipfs-shipyard/nopfs/ipfs" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange" + "github.com/ipfs/boxo/exchange/offline" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/namesys" @@ -55,23 +55,30 @@ const ( DHTOff DHTRouting = "off" ) +type RemoteBackendMode string + +const ( + RemoteBackendBlock RemoteBackendMode = "block" + RemoteBackendCAR RemoteBackendMode = "car" +) + func init() { // Lets us discover our own public address with a single observation identify.ActivationThresh = 1 } type Node struct { - vs routing.ValueStore - host host.Host - + ns namesys.NameSystem + vs routing.ValueStore dataDir string - datastore datastore.Batching - blockstore blockstore.Blockstore - exchange exchange.Interface bsrv blockservice.BlockService - resolver resolver.Resolver - ns namesys.NameSystem denylistSubs []*nopfs.HTTPSubscriber + + // Maybe not be set depending on the configuration: + host host.Host + datastore datastore.Batching + blockstore blockstore.Blockstore + resolver resolver.Resolver } type Config struct { @@ -96,6 +103,7 @@ type Config struct { DHTRouting DHTRouting DHTSharedHost bool IpnsMaxCacheTTL time.Duration + Bitswap bool DenylistSubs []string @@ -107,11 +115,15 @@ type Config struct { SeedPeering bool SeedPeeringMaxIndex int + RemoteBackends []string + RemoteBackendsIPNS bool + RemoteBackendMode RemoteBackendMode + GCInterval time.Duration GCThreshold float64 } -func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { +func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, error) { var err error cfg.DataDir, err = filepath.Abs(cfg.DataDir) @@ -119,11 +131,71 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached return nil, err } - ds, err := setupDatastore(cfg) + denylists, blocker, err := setupDenylists(cfg) + if err != nil { + return nil, err + } + + // The stars aligned and Libp2p does not need to be turned on at all. + if len(cfg.RemoteBackends) == 0 { + return nil, errors.New("RAINBOW_REMOTE_BACKENDS must be set if RAINBOW_LIBP2P is disabled") + } + + // Setup a Value Store composed of both the remote backends and the delegated + // routers, if they exist. This vs is only used for resolving IPNS Records. + vs, err := setupRoutingNoLibp2p(cfg, dnsCache) + if err != nil { + return nil, err + } + + // Setup the remote blockstore if that's the mode we're using. + var bsrv blockservice.BlockService + if cfg.RemoteBackendMode == RemoteBackendBlock { + blkst, err := gateway.NewRemoteBlockstore(cfg.RemoteBackends, nil) + if err != nil { + return nil, err + } + + bsrv = blockservice.New(blkst, offline.Exchange(blkst)) + bsrv = nopfsipfs.WrapBlockService(bsrv, blocker) + } + + ns, err := setupNamesys(cfg, vs, blocker) + if err != nil { + return nil, err + } + + return &Node{ + vs: vs, + ns: ns, + dataDir: cfg.DataDir, + denylistSubs: denylists, + bsrv: bsrv, + }, nil +} + +func SetupWithLibp2p(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { + if !cfg.Bitswap && cfg.DHTRouting == DHTOff { + return nil, errors.New("libp2p is enabled, but not used: bitswap and dht are disabled") + } + + var err error + + cfg.DataDir, err = filepath.Abs(cfg.DataDir) if err != nil { return nil, err } + denylists, blocker, err := setupDenylists(cfg) + if err != nil { + return nil, err + } + + n := &Node{ + dataDir: cfg.DataDir, + denylistSubs: denylists, + } + bwc := metrics.NewBandwidthCounter() cmgr, err := connmgr.NewConnManager(cfg.ConnMgrLow, cfg.ConnMgrHi, connmgr.WithGracePeriod(cfg.ConnMgrGrace)) @@ -170,20 +242,15 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached })) } - blkst := blockstore.NewBlockstore(ds, - blockstore.NoPrefix(), - // Every Has() for every written block is a transaction with a - // seek onto LSM. If not in memory it will be a pain. - // We opt to write every block Put into the blockstore. - // See also comment in blockservice. - blockstore.WriteThrough(), - ) - blkst = blockstore.NewIdStore(blkst) + ds, err := setupDatastore(cfg) + if err != nil { + return nil, err + } var ( + vs routing.ValueStore cr routing.ContentRouting pr routing.PeerRouting - vs routing.ValueStore ) opts = append(opts, libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { @@ -200,74 +267,67 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached return nil, err } - bswap := setupBitswapExchange(ctx, cfg, h, cr, blkst) + var bsrv blockservice.BlockService + if cfg.Bitswap { + blkst := blockstore.NewBlockstore(ds, + blockstore.NoPrefix(), + // Every Has() for every written block is a transaction with a + // seek onto LSM. If not in memory it will be a pain. + // We opt to write every block Put into the blockstore. + // See also comment in blockservice. + blockstore.WriteThrough(), + ) + blkst = blockstore.NewIdStore(blkst) + n.blockstore = blkst + + bsrv = blockservice.New(blkst, setupBitswapExchange(ctx, cfg, h, cr, blkst), + // if we are doing things right, our bitswap wantlists should + // not have blocks that we already have (see + // https://github.com/ipfs/boxo/blob/e0d4b3e9b91e9904066a10278e366c9a6d9645c7/blockservice/blockservice.go#L272). Thus + // we should not be writing many blocks that we already + // have. Thus, no point in checking whether we have a block + // before writing new blocks. + blockservice.WriteThrough(), + ) + } else { + if len(cfg.RemoteBackends) == 0 || cfg.RemoteBackendMode != RemoteBackendBlock { + return nil, errors.New("remote backends in block mode must be set when disabling bitswap") + } - err = os.Mkdir(filepath.Join(cfg.DataDir, "denylists"), 0755) - if err != nil && !errors.Is(err, fs.ErrExist) { - return nil, err - } + if cfg.PeeringSharedCache { + return nil, errors.New("disabling bitswap is incompatible with peering cache") + } - var denylists []*nopfs.HTTPSubscriber - for _, dl := range cfg.DenylistSubs { - s, err := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + blkst, err := gateway.NewRemoteBlockstore(cfg.RemoteBackends, nil) if err != nil { return nil, err } - denylists = append(denylists, s) - } - files, err := nopfs.GetDenylistFilesInDir(filepath.Join(cfg.DataDir, "denylists")) - if err != nil { - return nil, err - } - blocker, err := nopfs.NewBlocker(files) - if err != nil { - return nil, err + bsrv = blockservice.New(blkst, offline.Exchange(blkst)) } - bsrv := blockservice.New(blkst, bswap, - // if we are doing things right, our bitswap wantlists should - // not have blocks that we already have (see - // https://github.com/ipfs/boxo/blob/e0d4b3e9b91e9904066a10278e366c9a6d9645c7/blockservice/blockservice.go#L272). Thus - // we should not be writing many blocks that we already - // have. Thus, no point in checking whether we have a block - // before writing new blocks. - blockservice.WriteThrough(), - ) bsrv = nopfsipfs.WrapBlockService(bsrv, blocker) - dns, err := gateway.NewDNSResolver(nil) - if err != nil { - return nil, err - } - nsOptions := []namesys.Option{namesys.WithDNSResolver(dns)} - if cfg.IpnsMaxCacheTTL > 0 { - nsOptions = append(nsOptions, namesys.WithMaxCacheTTL(cfg.IpnsMaxCacheTTL)) - } - ns, err := namesys.NewNameSystem(vs, nsOptions...) - if err != nil { - return nil, err - } - ns = nopfsipfs.WrapNameSystem(ns, blocker) - fetcherCfg := bsfetcher.NewFetcherConfig(bsrv) fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) fetcher := fetcherCfg.WithReifier(unixfsnode.Reify) r := resolver.NewBasicResolver(fetcher) r = nopfsipfs.WrapResolver(r, blocker) - return &Node{ - host: h, - blockstore: blkst, - dataDir: cfg.DataDir, - datastore: ds, - exchange: bswap, - ns: ns, - vs: vs, - bsrv: bsrv, - resolver: r, - denylistSubs: denylists, - }, nil + n.host = h + n.datastore = ds + n.bsrv = bsrv + n.resolver = r + + ns, err := setupNamesys(cfg, vs, blocker) + if err != nil { + return nil, err + } + + n.vs = vs + n.ns = ns + + return n, nil } func setupDatastore(cfg Config) (datastore.Batching, error) { @@ -390,3 +450,47 @@ func setupPeering(cfg Config, h host.Host) error { return nil } + +func setupDenylists(cfg Config) ([]*nopfs.HTTPSubscriber, *nopfs.Blocker, error) { + err := os.Mkdir(filepath.Join(cfg.DataDir, "denylists"), 0755) + if err != nil && !errors.Is(err, fs.ErrExist) { + return nil, nil, err + } + + var denylists []*nopfs.HTTPSubscriber + for _, dl := range cfg.DenylistSubs { + s, err := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + if err != nil { + return nil, nil, err + } + denylists = append(denylists, s) + } + + files, err := nopfs.GetDenylistFilesInDir(filepath.Join(cfg.DataDir, "denylists")) + if err != nil { + return nil, nil, err + } + blocker, err := nopfs.NewBlocker(files) + if err != nil { + return nil, nil, err + } + + return denylists, blocker, nil +} + +func setupNamesys(cfg Config, vs routing.ValueStore, blocker *nopfs.Blocker) (namesys.NameSystem, error) { + dns, err := gateway.NewDNSResolver(nil) + if err != nil { + return nil, err + } + nsOptions := []namesys.Option{namesys.WithDNSResolver(dns)} + if cfg.IpnsMaxCacheTTL > 0 { + nsOptions = append(nsOptions, namesys.WithMaxCacheTTL(cfg.IpnsMaxCacheTTL)) + } + ns, err := namesys.NewNameSystem(vs, nsOptions...) + if err != nil { + return nil, err + } + ns = nopfsipfs.WrapNameSystem(ns, blocker) + return ns, nil +} diff --git a/setup_routing.go b/setup_routing.go index dd8d24e..02a4fde 100644 --- a/setup_routing.go +++ b/setup_routing.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/ipns" routingv1client "github.com/ipfs/boxo/routing/http/client" httpcontentrouter "github.com/ipfs/boxo/routing/http/contentrouter" @@ -180,6 +181,18 @@ func setupRouting(ctx context.Context, cfg Config, h host.Host, ds datastore.Bat vs routing.ValueStore = router ) + // If we're using a remote backend, but we also have libp2p enabled (e.g. for + // seed peering), we can still leverage the remote backend here. + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendsIPNS { + remoteValueStore, err := gateway.NewRemoteValueStore(cfg.RemoteBackends, nil) + if err != nil { + return nil, nil, nil, err + } + vs = setupCompositeRouting(append(delegatedRouters, &routinghelpers.Compose{ + ValueStore: remoteValueStore, + }), dhtRouter) + } + // If we're using seed peering, we need to run a lighter Amino DHT for the // peering routing. We need to run a separate DHT with the main host if // the shared host is disabled, or if we're not running any DHT at all. @@ -197,6 +210,25 @@ func setupRouting(ctx context.Context, cfg Config, h host.Host, ds datastore.Bat return cr, pr, vs, nil } +func setupRoutingNoLibp2p(cfg Config, dnsCache *cachedDNS) (routing.ValueStore, error) { + delegatedRouters, err := setupDelegatedRouting(cfg, dnsCache) + if err != nil { + return nil, err + } + + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendsIPNS { + remoteValueStore, err := gateway.NewRemoteValueStore(cfg.RemoteBackends, nil) + if err != nil { + return nil, err + } + delegatedRouters = append(delegatedRouters, &routinghelpers.Compose{ + ValueStore: remoteValueStore, + }) + } + + return setupCompositeRouting(delegatedRouters, nil), nil +} + type bundledDHT struct { standard *dht.IpfsDHT fullRT *fullrt.FullRT diff --git a/setup_test.go b/setup_test.go index fb234d3..ca32ede 100644 --- a/setup_test.go +++ b/setup_test.go @@ -84,6 +84,7 @@ func mustPeeredNodes(t *testing.T, configuration [][]int, peeringShareCache bool ListenAddrs: []string{mas[i].String()}, Peering: []peer.AddrInfo{}, PeeringSharedCache: peeringShareCache, + Bitswap: true, } for _, j := range configuration[i] { @@ -177,13 +178,14 @@ func testSeedPeering(t *testing.T, n int, dhtRouting DHTRouting, dhtSharedHost b BlockstoreType: "flatfs", DHTRouting: dhtRouting, DHTSharedHost: dhtSharedHost, + Bitswap: true, Seed: seed, SeedIndex: i, SeedPeering: true, SeedPeeringMaxIndex: n, } - nodes[i], err = Setup(ctx, cfgs[i], keys[i], cdns) + nodes[i], err = SetupWithLibp2p(ctx, cfgs[i], keys[i], cdns) require.NoError(t, err) }