From 134612e96ac744d77a3bea1eafb5550df457e44c Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 22 Apr 2024 14:30:39 +0200 Subject: [PATCH 1/9] feat: remote backend --- CHANGELOG.md | 2 + gc.go | 7 +- gc_test.go | 4 +- handler_test.go | 1 + handlers.go | 23 ++++- main.go | 139 ++++++++++++++++++--------- setup.go | 243 +++++++++++++++++++++++++++++++++-------------- setup_routing.go | 32 +++++++ setup_test.go | 2 + 9 files changed, 329 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4b7fa..4a23911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ The following emojis are used to highlight certain changes: ### Added +- Now supports remote backends (using RAW block or CAR requests) via `--remote-backends` (`RAINBOW_REMOTE_BACKENDS`). + ### Changed ### Removed diff --git a/gc.go b/gc.go index 4e875ec..671d83a 100644 --- a/gc.go +++ b/gc.go @@ -8,8 +8,13 @@ import ( ) // GC is a really stupid simple algorithm where we just delete things until -// weve deleted enough things +// we've deleted enough things. It is no-op if the current setup does not have +// a blockstore. func (nd *Node) GC(ctx context.Context, todelete int64) error { + if nd.blockstore == nil { + return nil + } + keys, err := nd.blockstore.AllKeysChan(ctx) if err != nil { return err diff --git a/gc_test.go b/gc_test.go index 1a7cb8d..8e44c78 100644 --- a/gc_test.go +++ b/gc_test.go @@ -12,7 +12,9 @@ import ( func TestPeriodicGC(t *testing.T) { t.Parallel() - gnd := mustTestNode(t, Config{}) + gnd := mustTestNode(t, Config{ + Bitswap: true, + }) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/handler_test.go b/handler_test.go index 120d775..c246166 100644 --- a/handler_test.go +++ b/handler_test.go @@ -12,6 +12,7 @@ func TestTrustless(t *testing.T) { t.Parallel() ts, gnd := mustTestServer(t, Config{ + Bitswap: true, TrustlessGatewayDomains: []string{"trustless.com"}, }) diff --git a/handlers.go b/handlers.go index c7d938c..9040649 100644 --- a/handlers.go +++ b/handlers.go @@ -85,12 +85,27 @@ func withRequestLogger(next http.Handler) http.Handler { } func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { - backend, err := gateway.NewBlocksBackend( - nd.bsrv, + var ( + backend gateway.IPFSBackend + err error + ) + + options := []gateway.BackendOption{ gateway.WithValueStore(nd.vs), gateway.WithNameSystem(nd.ns), - gateway.WithResolver(nd.resolver), - ) + gateway.WithResolver(nd.resolver), // May be nil, but that is fine. + } + + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendCAR { + var fetcher gateway.CarFetcher + fetcher, err = gateway.NewRemoteCarFetcher(cfg.RemoteBackends, nil) + if err != nil { + return nil, err + } + backend, err = gateway.NewCarBackend(fetcher, options...) + } else { + backend, err = gateway.NewBlocksBackend(nd.bsrv, options...) + } if err != nil { return nil, err } diff --git a/main.go b/main.go index f832e80..e9b56ce 100644 --- a/main.go +++ b/main.go @@ -248,6 +248,32 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_IPNS_MAX_CACHE_TTL"}, Usage: "Optional cap on caching duration for IPNS/DNSLink lookups. Set to 0 to respect original TTLs", }, + &cli.BoolFlag{ + Name: "bitswap", + Value: true, + EnvVars: []string{"RAINBOW_BITSWAP"}, + Usage: "Enable or disable Bitswap. Disabling Bitswap is incompatible with --peering-shared-cache", + }, + &cli.StringSliceFlag{ + Name: "remote-backends", + Value: cli.NewStringSlice(), + EnvVars: []string{"RAINBOW_REMOTE_BACKENDS"}, + Usage: "Trustless remote gateways to use as backend (comma-separated). You must set --bitswap=false to use this option. You can configure the mode with --remote-backends-mode", + }, + &cli.StringFlag{ + Name: "remote-backends-mode", + Value: "block", + EnvVars: []string{"RAINBOW_REMOTE_BACKENDS_MODE"}, + Usage: "Whether to fetch raw blocks or CARs from the remote backends. Options are 'block' or 'car'", + Action: func(ctx *cli.Context, s string) error { + switch RemoteBackendMode(s) { + case RemoteBackendBlock, RemoteBackendCAR: + return nil + default: + return errors.New("invalid value for --remote-backend-mode: use 'block' or 'car'") + } + }, + }, } app.Commands = []*cli.Command{ @@ -289,57 +315,66 @@ share the same seed as long as the indexes are different. var seed string var priv crypto.PrivKey + var peeringAddrs []peer.AddrInfo + var index int var err error - credDir := os.Getenv("CREDENTIALS_DIRECTORY") - secretsDir := ddir + bitswap := cctx.Bool("bitswap") + dhtRouting := DHTRouting(cctx.String("dht-routing")) + seedPeering := cctx.Bool("seed-peering") + noLibp2p := !bitswap && dhtRouting == DHTOff && !seedPeering - if len(credDir) > 0 { - secretsDir = credDir - } + // Only load secrets if we need Libp2p. + if !noLibp2p { + credDir := os.Getenv("CREDENTIALS_DIRECTORY") + secretsDir := ddir - // attempt to read seed from disk - seedBytes, err := os.ReadFile(filepath.Join(secretsDir, "seed")) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - // set seed from command line or env-var - seed = cctx.String("seed") - } else { - return fmt.Errorf("error reading seed credentials: %w", err) + if len(credDir) > 0 { + secretsDir = credDir } - } else { - seed = strings.TrimSpace(string(seedBytes)) - } - - index := cctx.Int("seed-index") - if len(seed) > 0 && index >= 0 { - fmt.Printf("Deriving identity from seed[%d]\n", index) - priv, err = deriveKey(seed, deriveKeyInfo(index)) - } else { - fmt.Println("Setting identity from libp2p.key") - keyFile := filepath.Join(secretsDir, "libp2p.key") - priv, err = loadOrInitPeerKey(keyFile) - } - if err != nil { - return err - } - var peeringAddrs []peer.AddrInfo - for _, maStr := range cctx.StringSlice("peering") { - if len(seed) > 0 && index >= 0 { - maStr, err = replaceRainbowSeedWithPeer(maStr, seed) - if err != nil { - return err + // attempt to read seed from disk + seedBytes, err := os.ReadFile(filepath.Join(secretsDir, "seed")) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // set seed from command line or env-var + seed = cctx.String("seed") + } else { + return fmt.Errorf("error reading seed credentials: %w", err) } - } else if rainbowSeedRegex.MatchString(maStr) { - return fmt.Errorf("unable to peer with %q without defining --seed-index of this instance first", maStr) + } else { + seed = strings.TrimSpace(string(seedBytes)) } - ai, err := peer.AddrInfoFromString(maStr) + index = cctx.Int("seed-index") + if len(seed) > 0 && index >= 0 { + fmt.Println("Deriving identity from seed") + priv, err = deriveKey(seed, deriveKeyInfo(index)) + } else { + fmt.Println("Setting identity from libp2p.key") + keyFile := filepath.Join(secretsDir, "libp2p.key") + priv, err = loadOrInitPeerKey(keyFile) + } if err != nil { return err } - peeringAddrs = append(peeringAddrs, *ai) + + for _, maStr := range cctx.StringSlice("peering") { + if len(seed) > 0 && index >= 0 { + maStr, err = replaceRainbowSeedWithPeer(maStr, seed) + if err != nil { + return err + } + } else if rainbowSeedRegex.MatchString(maStr) { + return fmt.Errorf("unable to peer with %q without defining --seed-index of this instance first", maStr) + } + + ai, err := peer.AddrInfoFromString(maStr) + if err != nil { + return err + } + peeringAddrs = append(peeringAddrs, *ai) + } } cfg := Config{ @@ -355,23 +390,32 @@ share the same seed as long as the indexes are different. MaxFD: cctx.Int("max-fd"), InMemBlockCache: cctx.Int64("inmem-block-cache"), RoutingV1Endpoints: cctx.StringSlice("http-routers"), - DHTRouting: DHTRouting(cctx.String("dht-routing")), + DHTRouting: dhtRouting, DHTSharedHost: cctx.Bool("dht-shared-host"), + Bitswap: bitswap, IpnsMaxCacheTTL: cctx.Duration("ipns-max-cache-ttl"), DenylistSubs: cctx.StringSlice("denylists"), Peering: peeringAddrs, PeeringSharedCache: cctx.Bool("peering-shared-cache"), Seed: seed, SeedIndex: index, - SeedPeering: cctx.Bool("seed-peering"), + SeedPeering: seedPeering, SeedPeeringMaxIndex: cctx.Int("seed-peering-max-index"), + RemoteBackends: cctx.StringSlice("remote-backends"), + RemoteBackendMode: RemoteBackendMode(cctx.String("remote-backends-mode")), GCInterval: cctx.Duration("gc-interval"), GCThreshold: cctx.Float64("gc-threshold"), } + var gnd *Node + goLog.Debugf("Rainbow config: %+v", cfg) - gnd, err := Setup(cctx.Context, cfg, priv, cdns) + if noLibp2p { + gnd, err = SetupNoLibp2p(cctx.Context, cfg, cdns) + } else { + gnd, err = Setup(cctx.Context, cfg, priv, cdns) + } if err != nil { return err } @@ -389,11 +433,14 @@ share the same seed as long as the indexes are different. Handler: handler, } - pid, err := peer.IDFromPublicKey(priv.GetPublic()) - if err != nil { - return err + fmt.Printf("Starting %s %s\n", name, version) + if priv != nil { + pid, err := peer.IDFromPublicKey(priv.GetPublic()) + if err != nil { + return err + } + fmt.Printf("PeerID: %s\n\n", pid) } - fmt.Printf("PeerID: %s\n\n", pid) registerVersionMetric(version) registerIpfsNodeCollector(gnd) diff --git a/setup.go b/setup.go index c91af8e..365f538 100644 --- a/setup.go +++ b/setup.go @@ -16,7 +16,7 @@ import ( nopfsipfs "github.com/ipfs-shipyard/nopfs/ipfs" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange" + "github.com/ipfs/boxo/exchange/offline" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/namesys" @@ -55,23 +55,30 @@ const ( DHTOff DHTRouting = "off" ) +type RemoteBackendMode string + +const ( + RemoteBackendBlock RemoteBackendMode = "block" + RemoteBackendCAR RemoteBackendMode = "car" +) + func init() { // Lets us discover our own public address with a single observation identify.ActivationThresh = 1 } type Node struct { - vs routing.ValueStore - host host.Host - + ns namesys.NameSystem + vs routing.ValueStore dataDir string - datastore datastore.Batching - blockstore blockstore.Blockstore - exchange exchange.Interface bsrv blockservice.BlockService - resolver resolver.Resolver - ns namesys.NameSystem denylistSubs []*nopfs.HTTPSubscriber + + // Maybe not be set depending on the configuration: + host host.Host + datastore datastore.Batching + blockstore blockstore.Blockstore + resolver resolver.Resolver } type Config struct { @@ -96,6 +103,7 @@ type Config struct { DHTRouting DHTRouting DHTSharedHost bool IpnsMaxCacheTTL time.Duration + Bitswap bool DenylistSubs []string @@ -107,10 +115,64 @@ type Config struct { SeedPeering bool SeedPeeringMaxIndex int + RemoteBackends []string + RemoteBackendMode RemoteBackendMode + GCInterval time.Duration GCThreshold float64 } +func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, error) { + var err error + + cfg.DataDir, err = filepath.Abs(cfg.DataDir) + if err != nil { + return nil, err + } + + denylists, blocker, err := setupDenylists(cfg) + if err != nil { + return nil, err + } + + // The stars aligned and Libp2p does not need to be turned on at all. + if len(cfg.RemoteBackends) == 0 { + return nil, errors.New("remote backends must be set when bitswap and dht are disabled") + } + + // Setup a Value Store composed of both the remote backends and the delegated + // routers, if they exist. This vs is only used for the namesystem. + vs, err := setupRoutingNoLibp2p(cfg, dnsCache) + if err != nil { + return nil, err + } + + // Setup the remote blockstore if that's the mode we're using. + var bsrv blockservice.BlockService + if cfg.RemoteBackendMode == RemoteBackendBlock { + blkst, err := gateway.NewRemoteBlockstore(cfg.RemoteBackends, nil) + if err != nil { + return nil, err + } + + bsrv = blockservice.New(blkst, offline.Exchange(blkst)) + bsrv = nopfsipfs.WrapBlockService(bsrv, blocker) + } + + ns, err := setupNamesys(cfg, vs, blocker) + if err != nil { + return nil, err + } + + return &Node{ + vs: vs, + ns: ns, + dataDir: cfg.DataDir, + denylistSubs: denylists, + bsrv: bsrv, + }, nil +} + func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { var err error @@ -119,11 +181,16 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached return nil, err } - ds, err := setupDatastore(cfg) + denylists, blocker, err := setupDenylists(cfg) if err != nil { return nil, err } + n := &Node{ + dataDir: cfg.DataDir, + denylistSubs: denylists, + } + bwc := metrics.NewBandwidthCounter() cmgr, err := connmgr.NewConnManager(cfg.ConnMgrLow, cfg.ConnMgrHi, connmgr.WithGracePeriod(cfg.ConnMgrGrace)) @@ -170,20 +237,15 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached })) } - blkst := blockstore.NewBlockstore(ds, - blockstore.NoPrefix(), - // Every Has() for every written block is a transaction with a - // seek onto LSM. If not in memory it will be a pain. - // We opt to write every block Put into the blockstore. - // See also comment in blockservice. - blockstore.WriteThrough(), - ) - blkst = blockstore.NewIdStore(blkst) + ds, err := setupDatastore(cfg) + if err != nil { + return nil, err + } var ( + vs routing.ValueStore cr routing.ContentRouting pr routing.PeerRouting - vs routing.ValueStore ) opts = append(opts, libp2p.Routing(func(h host.Host) (routing.PeerRouting, error) { @@ -200,74 +262,67 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached return nil, err } - bswap := setupBitswapExchange(ctx, cfg, h, cr, blkst) + var bsrv blockservice.BlockService + if cfg.Bitswap { + blkst := blockstore.NewBlockstore(ds, + blockstore.NoPrefix(), + // Every Has() for every written block is a transaction with a + // seek onto LSM. If not in memory it will be a pain. + // We opt to write every block Put into the blockstore. + // See also comment in blockservice. + blockstore.WriteThrough(), + ) + blkst = blockstore.NewIdStore(blkst) + n.blockstore = blkst + + bsrv = blockservice.New(blkst, setupBitswapExchange(ctx, cfg, h, cr, blkst), + // if we are doing things right, our bitswap wantlists should + // not have blocks that we already have (see + // https://github.com/ipfs/boxo/blob/e0d4b3e9b91e9904066a10278e366c9a6d9645c7/blockservice/blockservice.go#L272). Thus + // we should not be writing many blocks that we already + // have. Thus, no point in checking whether we have a block + // before writing new blocks. + blockservice.WriteThrough(), + ) + } else { + if len(cfg.RemoteBackends) == 0 || cfg.RemoteBackendMode != RemoteBackendBlock { + return nil, errors.New("remote backends in block mode must be set when disabling bitswap") + } - err = os.Mkdir(filepath.Join(cfg.DataDir, "denylists"), 0755) - if err != nil && !errors.Is(err, fs.ErrExist) { - return nil, err - } + if cfg.PeeringSharedCache { + return nil, errors.New("disabling bitswap is incompatible with peering cache") + } - var denylists []*nopfs.HTTPSubscriber - for _, dl := range cfg.DenylistSubs { - s, err := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + blkst, err := gateway.NewRemoteBlockstore(cfg.RemoteBackends, nil) if err != nil { return nil, err } - denylists = append(denylists, s) - } - files, err := nopfs.GetDenylistFilesInDir(filepath.Join(cfg.DataDir, "denylists")) - if err != nil { - return nil, err - } - blocker, err := nopfs.NewBlocker(files) - if err != nil { - return nil, err + bsrv = blockservice.New(blkst, offline.Exchange(blkst)) } - bsrv := blockservice.New(blkst, bswap, - // if we are doing things right, our bitswap wantlists should - // not have blocks that we already have (see - // https://github.com/ipfs/boxo/blob/e0d4b3e9b91e9904066a10278e366c9a6d9645c7/blockservice/blockservice.go#L272). Thus - // we should not be writing many blocks that we already - // have. Thus, no point in checking whether we have a block - // before writing new blocks. - blockservice.WriteThrough(), - ) bsrv = nopfsipfs.WrapBlockService(bsrv, blocker) - dns, err := gateway.NewDNSResolver(nil) - if err != nil { - return nil, err - } - nsOptions := []namesys.Option{namesys.WithDNSResolver(dns)} - if cfg.IpnsMaxCacheTTL > 0 { - nsOptions = append(nsOptions, namesys.WithMaxCacheTTL(cfg.IpnsMaxCacheTTL)) - } - ns, err := namesys.NewNameSystem(vs, nsOptions...) - if err != nil { - return nil, err - } - ns = nopfsipfs.WrapNameSystem(ns, blocker) - fetcherCfg := bsfetcher.NewFetcherConfig(bsrv) fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) fetcher := fetcherCfg.WithReifier(unixfsnode.Reify) r := resolver.NewBasicResolver(fetcher) r = nopfsipfs.WrapResolver(r, blocker) - return &Node{ - host: h, - blockstore: blkst, - dataDir: cfg.DataDir, - datastore: ds, - exchange: bswap, - ns: ns, - vs: vs, - bsrv: bsrv, - resolver: r, - denylistSubs: denylists, - }, nil + n.host = h + n.datastore = ds + n.bsrv = bsrv + n.resolver = r + + ns, err := setupNamesys(cfg, vs, blocker) + if err != nil { + return nil, err + } + + n.vs = vs + n.ns = ns + + return n, nil } func setupDatastore(cfg Config) (datastore.Batching, error) { @@ -390,3 +445,47 @@ func setupPeering(cfg Config, h host.Host) error { return nil } + +func setupDenylists(cfg Config) ([]*nopfs.HTTPSubscriber, *nopfs.Blocker, error) { + err := os.Mkdir(filepath.Join(cfg.DataDir, "denylists"), 0755) + if err != nil && !errors.Is(err, fs.ErrExist) { + return nil, nil, err + } + + var denylists []*nopfs.HTTPSubscriber + for _, dl := range cfg.DenylistSubs { + s, err := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + if err != nil { + return nil, nil, err + } + denylists = append(denylists, s) + } + + files, err := nopfs.GetDenylistFilesInDir(filepath.Join(cfg.DataDir, "denylists")) + if err != nil { + return nil, nil, err + } + blocker, err := nopfs.NewBlocker(files) + if err != nil { + return nil, nil, err + } + + return denylists, blocker, nil +} + +func setupNamesys(cfg Config, vs routing.ValueStore, blocker *nopfs.Blocker) (namesys.NameSystem, error) { + dns, err := gateway.NewDNSResolver(nil) + if err != nil { + return nil, err + } + nsOptions := []namesys.Option{namesys.WithDNSResolver(dns)} + if cfg.IpnsMaxCacheTTL > 0 { + nsOptions = append(nsOptions, namesys.WithMaxCacheTTL(cfg.IpnsMaxCacheTTL)) + } + ns, err := namesys.NewNameSystem(vs, nsOptions...) + if err != nil { + return nil, err + } + ns = nopfsipfs.WrapNameSystem(ns, blocker) + return ns, nil +} diff --git a/setup_routing.go b/setup_routing.go index dd8d24e..2cabf3a 100644 --- a/setup_routing.go +++ b/setup_routing.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/ipns" routingv1client "github.com/ipfs/boxo/routing/http/client" httpcontentrouter "github.com/ipfs/boxo/routing/http/contentrouter" @@ -180,6 +181,18 @@ func setupRouting(ctx context.Context, cfg Config, h host.Host, ds datastore.Bat vs routing.ValueStore = router ) + // If we're using a remote backend, but we also have libp2p enabled (e.g. for + // seed peering), we can still leverage the remote backend here. + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendBlock { + remoteValueStore, err := gateway.NewRemoteValueStore(cfg.RemoteBackends, nil) + if err != nil { + return nil, nil, nil, err + } + vs = setupCompositeRouting(append(delegatedRouters, &routinghelpers.Compose{ + ValueStore: remoteValueStore, + }), dhtRouter) + } + // If we're using seed peering, we need to run a lighter Amino DHT for the // peering routing. We need to run a separate DHT with the main host if // the shared host is disabled, or if we're not running any DHT at all. @@ -197,6 +210,25 @@ func setupRouting(ctx context.Context, cfg Config, h host.Host, ds datastore.Bat return cr, pr, vs, nil } +func setupRoutingNoLibp2p(cfg Config, dnsCache *cachedDNS) (routing.ValueStore, error) { + delegatedRouters, err := setupDelegatedRouting(cfg, dnsCache) + if err != nil { + return nil, err + } + + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendBlock { + remoteValueStore, err := gateway.NewRemoteValueStore(cfg.RemoteBackends, nil) + if err != nil { + return nil, err + } + delegatedRouters = append(delegatedRouters, &routinghelpers.Compose{ + ValueStore: remoteValueStore, + }) + } + + return setupCompositeRouting(delegatedRouters, nil), nil +} + type bundledDHT struct { standard *dht.IpfsDHT fullRT *fullrt.FullRT diff --git a/setup_test.go b/setup_test.go index fb234d3..0d824d7 100644 --- a/setup_test.go +++ b/setup_test.go @@ -84,6 +84,7 @@ func mustPeeredNodes(t *testing.T, configuration [][]int, peeringShareCache bool ListenAddrs: []string{mas[i].String()}, Peering: []peer.AddrInfo{}, PeeringSharedCache: peeringShareCache, + Bitswap: true, } for _, j := range configuration[i] { @@ -177,6 +178,7 @@ func testSeedPeering(t *testing.T, n int, dhtRouting DHTRouting, dhtSharedHost b BlockstoreType: "flatfs", DHTRouting: dhtRouting, DHTSharedHost: dhtSharedHost, + Bitswap: true, Seed: seed, SeedIndex: i, SeedPeering: true, From 6a5f4b025ccfb0c1fa6d8127d298142d5f209d45 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 17 May 2024 23:20:59 +0200 Subject: [PATCH 2/9] docs: clarify libp2p-specific config options These are only applied to libp2p nodes, not the entire rainbow. --- gc.go | 2 +- main.go | 46 +++++++++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/gc.go b/gc.go index 671d83a..487dbd2 100644 --- a/gc.go +++ b/gc.go @@ -9,7 +9,7 @@ import ( // GC is a really stupid simple algorithm where we just delete things until // we've deleted enough things. It is no-op if the current setup does not have -// a blockstore. +// a (local) blockstore. func (nd *Node) GC(ctx context.Context, todelete int64) error { if nd.blockstore == nil { return nil diff --git a/main.go b/main.go index e9b56ce..d5b8999 100644 --- a/main.go +++ b/main.go @@ -157,22 +157,22 @@ Generate an identity seed and launch a gateway: Usage: "Percentage of how much of the disk free space must be available", }, &cli.IntFlag{ - Name: "connmgr-low", + Name: "libp2p-connmgr-low", Value: 100, - EnvVars: []string{"RAINBOW_CONNMGR_LOW"}, - Usage: "Minimum number of connections to keep", + EnvVars: []string{"RAINBOW_LIBP2P_CONNMGR_LOW"}, + Usage: "Number of connections that the connection manager will trim down to during GC", }, &cli.IntFlag{ - Name: "connmgr-high", + Name: "libp2p-connmgr-high", Value: 3000, - EnvVars: []string{"RAINBOW_CONNMGR_HIGH"}, - Usage: "Maximum number of connections to keep", + EnvVars: []string{"RAINBOW_LIBP2P_CONNMGR_HIGH"}, + Usage: "Number of libp2p connections that, when exceeded, will trigger a connection GC operation", }, &cli.DurationFlag{ - Name: "connmgr-grace", + Name: "libp2p-connmgr-grace", Value: time.Minute, - EnvVars: []string{"RAINBOW_CONNMGR_GRACE_PERIOD"}, - Usage: "Minimum connection TTL", + EnvVars: []string{"RAINBOW_LIBP2P_CONNMGR_GRACE_PERIOD"}, + Usage: "How long new libp2p connections are immune from being closed by the connection manager", }, &cli.IntFlag{ Name: "inmem-block-cache", @@ -181,16 +181,16 @@ Generate an identity seed and launch a gateway: Usage: "Size of the in-memory block cache (currently only used for badger). 0 to disable (disables compression on disk too)", }, &cli.Uint64Flag{ - Name: "max-memory", + Name: "libp2p-max-memory", Value: 0, - EnvVars: []string{"RAINBOW_MAX_MEMORY"}, - Usage: "Max memory to use. Defaults to 85% of the system's available RAM", + EnvVars: []string{"RAINBOW_LIBP2P_MAX_MEMORY"}, + Usage: "Max memory to use for libp2p node. Defaults to 85% of the system's available RAM", }, &cli.Uint64Flag{ - Name: "max-fd", + Name: "libp2p-max-fd", Value: 0, - EnvVars: []string{"RAINBOW_MAX_FD"}, - Usage: "Maximum number of file descriptors. Defaults to 50% of the process' limit", + EnvVars: []string{"RAINBOW_LIBP2P_MAX_FD"}, + Usage: "Maximum number of file descriptors used by libp2p node. Defaults to 50% of the process' limit", }, &cli.StringSliceFlag{ Name: "http-routers", @@ -234,7 +234,7 @@ Generate an identity seed and launch a gateway: Name: "peering-shared-cache", Value: false, EnvVars: []string{"RAINBOW_PEERING_SHARED_CACHE"}, - Usage: "(EXPERIMENTAL: increased network I/O) Enable sharing of local cache to peers safe-listed with --peering. Rainbow will respond to Bitswap queries from these peers, serving locally cached data as needed.", + Usage: "(EXPERIMENTAL: increased network I/O) Enable sharing of local cache to peers safe-listed with --peering. Rainbow will respond to Bitswap queries from these peers, serving locally cached data as needed (requires --bitswap=true).", }, &cli.StringFlag{ Name: "blockstore", @@ -252,13 +252,13 @@ Generate an identity seed and launch a gateway: Name: "bitswap", Value: true, EnvVars: []string{"RAINBOW_BITSWAP"}, - Usage: "Enable or disable Bitswap. Disabling Bitswap is incompatible with --peering-shared-cache", + Usage: "Controls if Bitswap is enabled (useful for testing or when remote backend is used instead)", }, &cli.StringSliceFlag{ Name: "remote-backends", Value: cli.NewStringSlice(), EnvVars: []string{"RAINBOW_REMOTE_BACKENDS"}, - Usage: "Trustless remote gateways to use as backend (comma-separated). You must set --bitswap=false to use this option. You can configure the mode with --remote-backends-mode", + Usage: "Trustless gateways to use as backend instead of Bitswap (comma-separated urls)", }, &cli.StringFlag{ Name: "remote-backends-mode", @@ -383,11 +383,11 @@ share the same seed as long as the indexes are different. GatewayDomains: cctx.StringSlice("gateway-domains"), SubdomainGatewayDomains: cctx.StringSlice("subdomain-gateway-domains"), TrustlessGatewayDomains: cctx.StringSlice("trustless-gateway-domains"), - ConnMgrLow: cctx.Int("connmgr-low"), - ConnMgrHi: cctx.Int("connmgr-high"), - ConnMgrGrace: cctx.Duration("connmgr-grace"), - MaxMemory: cctx.Uint64("max-memory"), - MaxFD: cctx.Int("max-fd"), + ConnMgrLow: cctx.Int("libp2p-connmgr-low"), + ConnMgrHi: cctx.Int("libp2p-connmgr-high"), + ConnMgrGrace: cctx.Duration("libp2p-connmgr-grace"), + MaxMemory: cctx.Uint64("libp2p-max-memory"), + MaxFD: cctx.Int("libp2p-max-fd"), InMemBlockCache: cctx.Int64("inmem-block-cache"), RoutingV1Endpoints: cctx.StringSlice("http-routers"), DHTRouting: dhtRouting, From 921d0ab25c80fa8eae2ffdcb79ebc608ebf08f71 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 17 May 2024 23:28:32 +0200 Subject: [PATCH 3/9] refactor: move rcmgr log to rainbow=info logger --- main.go | 2 +- rcmgr.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index d5b8999..f5514f5 100644 --- a/main.go +++ b/main.go @@ -409,7 +409,7 @@ share the same seed as long as the indexes are different. var gnd *Node - goLog.Debugf("Rainbow config: %+v", cfg) + goLog.Infof("Rainbow config: %+v", cfg) if noLibp2p { gnd, err = SetupNoLibp2p(cctx.Context, cfg, cdns) diff --git a/rcmgr.go b/rcmgr.go index e5aff9b..e9cc9b7 100644 --- a/rcmgr.go +++ b/rcmgr.go @@ -1,8 +1,6 @@ package main import ( - "log" - "github.com/dustin/go-humanize" "github.com/pbnjay/memory" @@ -166,7 +164,7 @@ func makeResourceManagerConfig(maxMemory uint64, maxFD int, connMgrHighWater int partialLimits.System.ConnsInbound = rcmgr.LimitVal(maxInboundConns) } - log.Printf(` + goLog.Infof(` go-libp2p Resource Manager limits based on: - --max-memory: %s @@ -248,7 +246,7 @@ func makeSeparateDHTClientResourceManagerConfig(maxMemory uint64, maxFD int) (li // Anything in scalingLimitConfig that wasn't defined in partialLimits above will be added (e.g., libp2p's default service limits). partialLimits = partialLimits.Build(scalingLimitConfig.Scale(int64(maxMemory), maxFD)).ToPartialLimitConfig() - log.Printf(` + goLog.Infof(` go-libp2p Separate DHT Resource Manager limits based on: - --max-memory: %s From 0dee5f4af555f126f97b4842a748ed830d2fed33 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 21 May 2024 01:55:53 +0200 Subject: [PATCH 4/9] docs: remote backend configuration --- docs/environment-variables.md | 27 +++++++++++++++++++++++++++ main.go | 6 +++--- setup.go | 4 ++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index a943ec7..ec8a9b8 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -162,6 +162,33 @@ queries from these safelisted peers, serving locally cached blocks if requested. Default: `false` (no cache sharing) +### `RAINBOW_DHT_ROUTING` + +Control the type of Amino DHT client used for for routing. Options are `accelerated`, `standard` and `off`. + +Default: `accelerated` + +### `RAINBOW_REMOTE_BACKENDS` + +> [!WARNING] +> Experimental feature. Requires setting `RAINBOW_DHT_ROUTING=off` and `RAINBOW_BITSWAP=false`. + +URL(s) of of remote [trustless gateways](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) +to use as backend instead of libp2p node with Bitswap. + +Default: not set + +### `RAINBOW_REMOTE_BACKENDS_MODE` + +Requires `RAINBOW_REMOTE_BACKENDS` to be set. + +Controls how requests to remote backend are made. + +- `block` will use [application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw) to fetch raw blocks one by one +- `car` will use [application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) and [IPIP-402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) for fetching multiple blocks per request + +Default: `block` + ## Logging ### `GOLOG_LOG_LEVEL` diff --git a/main.go b/main.go index f5514f5..db46aee 100644 --- a/main.go +++ b/main.go @@ -228,7 +228,7 @@ Generate an identity seed and launch a gateway: Name: "peering", Value: cli.NewStringSlice(), EnvVars: []string{"RAINBOW_PEERING"}, - Usage: "Multiaddresses of peers to stay connected to (comma-separated)", + Usage: "(EXPERIMENTAL) Multiaddresses of peers to stay connected to and ask for missing blocks over Bitswap (comma-separated)", }, &cli.BoolFlag{ Name: "peering-shared-cache", @@ -258,13 +258,13 @@ Generate an identity seed and launch a gateway: Name: "remote-backends", Value: cli.NewStringSlice(), EnvVars: []string{"RAINBOW_REMOTE_BACKENDS"}, - Usage: "Trustless gateways to use as backend instead of Bitswap (comma-separated urls)", + Usage: "(EXPERIMENTAL) Trustless gateways to use as backend instead of Bitswap (comma-separated urls)", }, &cli.StringFlag{ Name: "remote-backends-mode", Value: "block", EnvVars: []string{"RAINBOW_REMOTE_BACKENDS_MODE"}, - Usage: "Whether to fetch raw blocks or CARs from the remote backends. Options are 'block' or 'car'", + Usage: "(EXPERIMENTAL) Whether to fetch raw blocks or CARs from the remote backends. Options are 'block' or 'car'", Action: func(ctx *cli.Context, s string) error { switch RemoteBackendMode(s) { case RemoteBackendBlock, RemoteBackendCAR: diff --git a/setup.go b/setup.go index 365f538..0b2eb98 100644 --- a/setup.go +++ b/setup.go @@ -137,11 +137,11 @@ func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, // The stars aligned and Libp2p does not need to be turned on at all. if len(cfg.RemoteBackends) == 0 { - return nil, errors.New("remote backends must be set when bitswap and dht are disabled") + return nil, errors.New("URL of RAINBOW_REMOTE_BACKENDS must be set when RAINBOW_BITSWAP and RAINBOW_DHT_ROUTING are disabled") } // Setup a Value Store composed of both the remote backends and the delegated - // routers, if they exist. This vs is only used for the namesystem. + // routers, if they exist. This vs is only used for resolving IPNS Records. vs, err := setupRoutingNoLibp2p(cfg, dnsCache) if err != nil { return nil, err From 76b80647380ea1b0619aa592d1a2e3621ef03603 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 21 May 2024 11:02:42 +0200 Subject: [PATCH 5/9] feat: add global --libp2p (RAINBOW_LIBP2P) flag --- main.go | 16 +++++++++++----- main_test.go | 2 +- setup.go | 6 +++++- setup_test.go | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index db46aee..569ef68 100644 --- a/main.go +++ b/main.go @@ -156,6 +156,12 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_GC_THRESHOLD"}, Usage: "Percentage of how much of the disk free space must be available", }, + &cli.BoolFlag{ + Name: "libp2p", + Value: true, + EnvVars: []string{"RAINBOW_LIBP2P"}, + Usage: "Enable or disable the usage of Libp2p", + }, &cli.IntFlag{ Name: "libp2p-connmgr-low", Value: 100, @@ -322,10 +328,10 @@ share the same seed as long as the indexes are different. bitswap := cctx.Bool("bitswap") dhtRouting := DHTRouting(cctx.String("dht-routing")) seedPeering := cctx.Bool("seed-peering") - noLibp2p := !bitswap && dhtRouting == DHTOff && !seedPeering + libp2p := cctx.Bool("libp2p") // Only load secrets if we need Libp2p. - if !noLibp2p { + if libp2p { credDir := os.Getenv("CREDENTIALS_DIRECTORY") secretsDir := ddir @@ -411,10 +417,10 @@ share the same seed as long as the indexes are different. goLog.Infof("Rainbow config: %+v", cfg) - if noLibp2p { - gnd, err = SetupNoLibp2p(cctx.Context, cfg, cdns) + if libp2p { + gnd, err = SetupWithLibp2p(cctx.Context, cfg, priv, cdns) } else { - gnd, err = Setup(cctx.Context, cfg, priv, cdns) + gnd, err = SetupNoLibp2p(cctx.Context, cfg, cdns) } if err != nil { return err diff --git a/main_test.go b/main_test.go index 5b0d9b3..33c0354 100644 --- a/main_test.go +++ b/main_test.go @@ -63,7 +63,7 @@ func mustTestNodeWithKey(t *testing.T, cfg Config, sk ic.PrivKey) *Node { _ = cdns.Close() }) - nd, err := Setup(ctx, cfg, sk, cdns) + nd, err := SetupWithLibp2p(ctx, cfg, sk, cdns) require.NoError(t, err) return nd } diff --git a/setup.go b/setup.go index 0b2eb98..f746a72 100644 --- a/setup.go +++ b/setup.go @@ -173,7 +173,11 @@ func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, }, nil } -func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { +func SetupWithLibp2p(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { + if !cfg.Bitswap && cfg.DHTRouting == DHTOff && !cfg.SeedPeering { + return nil, errors.New("libp2p is enabled, but not used: bitswap, dht and seed peering are disabled") + } + var err error cfg.DataDir, err = filepath.Abs(cfg.DataDir) diff --git a/setup_test.go b/setup_test.go index 0d824d7..ca32ede 100644 --- a/setup_test.go +++ b/setup_test.go @@ -185,7 +185,7 @@ func testSeedPeering(t *testing.T, n int, dhtRouting DHTRouting, dhtSharedHost b SeedPeeringMaxIndex: n, } - nodes[i], err = Setup(ctx, cfgs[i], keys[i], cdns) + nodes[i], err = SetupWithLibp2p(ctx, cfgs[i], keys[i], cdns) require.NoError(t, err) } From 038e75f8da77f4c394c83793b04f16a2984d6473 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 21 May 2024 11:09:07 +0200 Subject: [PATCH 6/9] feat: --remote-backends-ipns (and RAINBOW_REMOTE_IPNS) --- main.go | 7 +++++++ setup.go | 5 +++-- setup_routing.go | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 569ef68..66f0f6a 100644 --- a/main.go +++ b/main.go @@ -266,6 +266,12 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_REMOTE_BACKENDS"}, Usage: "(EXPERIMENTAL) Trustless gateways to use as backend instead of Bitswap (comma-separated urls)", }, + &cli.BoolFlag{ + Name: "remote-backends-ipns", + Value: true, + EnvVars: []string{"RAINBOW_REMOTE_BACKENDS_IPNS"}, + Usage: "(EXPERIMENTAL) Whether to fetch IPNS Records (application/vnd.ipfs.ipns-record) from the remote backends", + }, &cli.StringFlag{ Name: "remote-backends-mode", Value: "block", @@ -408,6 +414,7 @@ share the same seed as long as the indexes are different. SeedPeering: seedPeering, SeedPeeringMaxIndex: cctx.Int("seed-peering-max-index"), RemoteBackends: cctx.StringSlice("remote-backends"), + RemoteBackendsIPNS: cctx.Bool("remote-backends-ipns"), RemoteBackendMode: RemoteBackendMode(cctx.String("remote-backends-mode")), GCInterval: cctx.Duration("gc-interval"), GCThreshold: cctx.Float64("gc-threshold"), diff --git a/setup.go b/setup.go index f746a72..7631423 100644 --- a/setup.go +++ b/setup.go @@ -115,8 +115,9 @@ type Config struct { SeedPeering bool SeedPeeringMaxIndex int - RemoteBackends []string - RemoteBackendMode RemoteBackendMode + RemoteBackends []string + RemoteBackendsIPNS bool + RemoteBackendMode RemoteBackendMode GCInterval time.Duration GCThreshold float64 diff --git a/setup_routing.go b/setup_routing.go index 2cabf3a..02a4fde 100644 --- a/setup_routing.go +++ b/setup_routing.go @@ -183,7 +183,7 @@ func setupRouting(ctx context.Context, cfg Config, h host.Host, ds datastore.Bat // If we're using a remote backend, but we also have libp2p enabled (e.g. for // seed peering), we can still leverage the remote backend here. - if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendBlock { + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendsIPNS { remoteValueStore, err := gateway.NewRemoteValueStore(cfg.RemoteBackends, nil) if err != nil { return nil, nil, nil, err @@ -216,7 +216,7 @@ func setupRoutingNoLibp2p(cfg Config, dnsCache *cachedDNS) (routing.ValueStore, return nil, err } - if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendBlock { + if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendsIPNS { remoteValueStore, err := gateway.NewRemoteValueStore(cfg.RemoteBackends, nil) if err != nil { return nil, err From e2e80706b090ce9149463524d64f43e9a9d95f44 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 21 May 2024 11:38:54 +0200 Subject: [PATCH 7/9] ci: conformance with libp2p and without --- .github/workflows/gateway-conformance.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 30a38fa..a4a1806 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -9,6 +9,9 @@ on: jobs: gateway-conformance: runs-on: ubuntu-latest + strategy: + matrix: + libp2p: ["libp2p", "no_libp2p"] steps: # 1. Start the Kubo gateway - name: Setup Go @@ -65,18 +68,27 @@ jobs: run: go build working-directory: rainbow - # 5. Start rainbow - - name: Start rainbow + # 5.1. Start rainbow (Libp2p) + - name: Start rainbow (Libp2p) + if: ${{ matrix.libp2p == 'libp2p' }} env: GATEWAY_CONFORMANCE_TEST: true run: | # get kubo peerID kuboNodeMultiaddr=$(ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) - # run gw ./rainbow --http-routers=http://127.0.0.1:8080 --dht-routing=off --peering=$kuboNodeMultiaddr & working-directory: rainbow + # 5.2. Start rainbow (No Libp2p) + - name: Start rainbow (No Libp2p) + if: ${{ matrix.libp2p == 'no_libp2p' }} + env: + GATEWAY_CONFORMANCE_TEST: true + run: | + ./rainbow --http-routers=http://127.0.0.1:8080 --remote-backends=http://127.0.0.1:8080 --libp2p=false & + working-directory: rainbow + # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests uses: ipfs/gateway-conformance/.github/actions/test@v0.5.1 @@ -102,11 +114,11 @@ jobs: if: failure() || success() uses: actions/upload-artifact@v4 with: - name: gateway-conformance.html + name: ${{ matrix.libp2p }}_gateway-conformance.html path: output.html - name: Upload JSON report if: failure() || success() uses: actions/upload-artifact@v4 with: - name: gateway-conformance.json + name: ${{ matrix.libp2p }}_gateway-conformance.json path: output.json From 3433adb2f20bd12e9d2dd8ac3ca0949938dfa489 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 22 May 2024 21:49:43 +0200 Subject: [PATCH 8/9] test(ci): both block and car gws Block and CAR backends have significate difference in code paths and abstractions. We need to run conformance against them separately to ensure both code paths behave the same way. --- .github/workflows/gateway-conformance.yml | 89 ++++++++++++++++------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index a4a1806..fae26d4 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -1,34 +1,44 @@ name: Gateway Conformance on: + workflow_dispatch: push: branches: - main pull_request: + paths-ignore: + - '**/*.md' + +env: + GATEWAY_CONFORMANCE_TEST: true # rainbow preset for conformance testing + KUBO_VER: 'v0.28.0' # kubo daemon used as no-libp2p-remote-* backend + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true jobs: - gateway-conformance: + tests: runs-on: ubuntu-latest strategy: matrix: - libp2p: ["libp2p", "no_libp2p"] + backend: ["libp2p-bitswap", "remote-block-gw", "remote-car-gw"] + steps: # 1. Start the Kubo gateway - - name: Setup Go - uses: actions/setup-go@v5 + - name: Install Kubo + uses: ipfs/download-ipfs-distribution-action@v1 with: - go-version: 1.21.x + name: kubo + version: "${{ env.KUBO_VER }}" - - name: Install Kubo gateway from source - #uses: ipfs/download-ipfs-distribution-action@v1 - run: | - go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0-rc1 - name: Setup kubo config run: | ipfs init --profile=test ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" ipfs config --json Gateway.ExposeRoutingAPI true + ipfs config Routing.Type "autoclient" # 2. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures @@ -36,10 +46,7 @@ jobs: with: output: fixtures - - name: Start Kubo gateway - uses: ipfs/start-ipfs-daemon-action@v1 - - # 3. Populate the Kubo gateway with the gateway-conformance fixtures + # 3. Populate the Kubo node with the gateway-conformance fixtures - name: Import fixtures run: | # Import car files @@ -59,7 +66,15 @@ jobs: export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + - name: Start Kubo gateway + uses: ipfs/start-ipfs-daemon-action@v1 + # 4. Build rainbow + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + - name: Checkout rainbow uses: actions/checkout@v4 with: @@ -68,25 +83,45 @@ jobs: run: go build working-directory: rainbow - # 5.1. Start rainbow (Libp2p) - - name: Start rainbow (Libp2p) - if: ${{ matrix.libp2p == 'libp2p' }} + # 5. Start rainbow variant + - name: Start rainbow (libp2p and bitswap) + if: ${{ matrix.backend == 'libp2p-bitswap' }} env: - GATEWAY_CONFORMANCE_TEST: true + RAINBOW_DHT_ROUTING: off + RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080 run: | - # get kubo peerID + # set up peering with kubo to ensure fixtures can be found fast kuboNodeMultiaddr=$(ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) - ./rainbow --http-routers=http://127.0.0.1:8080 --dht-routing=off --peering=$kuboNodeMultiaddr & + ./rainbow --peering=$kuboNodeMultiaddr & working-directory: rainbow - # 5.2. Start rainbow (No Libp2p) - - name: Start rainbow (No Libp2p) - if: ${{ matrix.libp2p == 'no_libp2p' }} + # 5. Start rainbow variant + - name: Start rainbow (no libp2p, remote block gateway) + if: ${{ matrix.backend == 'remote-block-gw' }} + env: + RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080 + RAINBOW_REMOTE_BACKENDS_MODE: block + RAINBOW_LIBP2P: false + RAINBOW_BITSWAP: false + RAINBOW_DHT_ROUTING: off + RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080 + run: | + ./rainbow & + working-directory: rainbow + # + # 5. Start rainbow variant + - name: Start rainbow (no libp2p, remote car gateway) + if: ${{ matrix.backend == 'remote-car-gw' }} env: - GATEWAY_CONFORMANCE_TEST: true + RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080 + RAINBOW_REMOTE_BACKENDS_MODE: car + RAINBOW_LIBP2P: false + RAINBOW_BITSWAP: false + RAINBOW_DHT_ROUTING: off + RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080 run: | - ./rainbow --http-routers=http://127.0.0.1:8080 --remote-backends=http://127.0.0.1:8080 --libp2p=false & + ./rainbow & working-directory: rainbow # 6. Run the gateway-conformance tests @@ -104,7 +139,7 @@ jobs: # # only-if-cached: rainbow does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip 'TestGatewayCache/.*_for_/ipfs/_with_only-if-cached_succeeds_when_in_local_datastore' + args: -skip 'TestGatewayCache/.*_with_only-if-cached_succeeds_when_in_local_datastore' # 7. Upload the results - name: Upload MD summary @@ -114,11 +149,11 @@ jobs: if: failure() || success() uses: actions/upload-artifact@v4 with: - name: ${{ matrix.libp2p }}_gateway-conformance.html + name: ${{ matrix.backend }}_gateway-conformance.html path: output.html - name: Upload JSON report if: failure() || success() uses: actions/upload-artifact@v4 with: - name: ${{ matrix.libp2p }}_gateway-conformance.json + name: ${{ matrix.backend }}_gateway-conformance.json path: output.json From bdd3fc3908417486f98ef5fab723a2cf0e926b73 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 23 May 2024 01:07:09 +0200 Subject: [PATCH 9/9] docs: RAINBOW_REMOTE_BACKENDS Setting RAINBOW_REMOTE_BACKENDS should be enough to turn off libp2p and delegate everything to HTTP endpoints. --- .github/workflows/gateway-conformance.yml | 4 ++- docs/environment-variables.md | 42 +++++++++++++++++++---- main.go | 17 +++++++-- setup.go | 6 ++-- 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index fae26d4..33df29e 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: true jobs: - tests: + test: runs-on: ubuntu-latest strategy: matrix: @@ -102,6 +102,7 @@ jobs: env: RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080 RAINBOW_REMOTE_BACKENDS_MODE: block + RAINBOW_REMOTE_BACKENDS_IPNS: true RAINBOW_LIBP2P: false RAINBOW_BITSWAP: false RAINBOW_DHT_ROUTING: off @@ -116,6 +117,7 @@ jobs: env: RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080 RAINBOW_REMOTE_BACKENDS_MODE: car + RAINBOW_REMOTE_BACKENDS_IPNS: true RAINBOW_LIBP2P: false RAINBOW_BITSWAP: false RAINBOW_DHT_ROUTING: off diff --git a/docs/environment-variables.md b/docs/environment-variables.md index ec8a9b8..977bad5 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -6,15 +6,22 @@ - [`RAINBOW_GATEWAY_DOMAINS`](#rainbow_gateway_domains) - [`RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`](#rainbow_subdomain_gateway_domains) - [`RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`](#rainbow_trustless_gateway_domains) + - [`RAINBOW_DATADIR`](#rainbow_datadir) - [`RAINBOW_GC_INTERVAL`](#rainbow_gc_interval) - [`RAINBOW_GC_THRESHOLD`](#rainbow_gc_threshold) - [`RAINBOW_IPNS_MAX_CACHE_TTL`](#rainbow_ipns_max_cache_ttl) - [`RAINBOW_PEERING`](#rainbow_peering) - [`RAINBOW_SEED`](#rainbow_seed) - [`RAINBOW_SEED_INDEX`](#rainbow_seed_index) + - [`RAINBOW_DHT_ROUTING`](#rainbow_dht_routing) + - [`RAINBOW_HTTP_ROUTERS`](#rainbow_http_routers) +- [Experiments](#experiments) - [`RAINBOW_SEED_PEERING`](#rainbow_seed_peering) - [`RAINBOW_SEED_PEERING_MAX_INDEX`](#rainbow_seed_peering_max_index) - [`RAINBOW_PEERING_SHARED_CACHE`](#rainbow_peering_shared_cache) + - [`RAINBOW_REMOTE_BACKENDS`](#rainbow_remote_backends) + - [`RAINBOW_REMOTE_BACKENDS_MODE`](#rainbow_remote_backends_mode) + - [`RAINBOW_REMOTE_BACKENDS_IPNS`](#rainbow_remote_backends_ipns) - [Logging](#logging) - [`GOLOG_LOG_LEVEL`](#golog_log_level) - [`GOLOG_LOG_FMT`](#golog_log_fmt) @@ -69,6 +76,12 @@ when request comes with the `Host` header set to `trustless-gateway.link`. Default: none (`Host` is ignored and gateway at `127.0.0.1` supports both deserialized and verifiable response types) +### `RAINBOW_DATADIR` + +Directory for persistent data (keys, blocks, denylists) + +Default: not set (uses the current directory) + ### `RAINBOW_GC_INTERVAL` The interval at which the garbage collector will be called. This is given as a string that corresponds to the duration of the interval. Set 0 to disable. @@ -121,6 +134,20 @@ Index to derivate the PeerID identity from `RAINBOW_SEED`. Default: not set +### `RAINBOW_DHT_ROUTING` + +Control the type of Amino DHT client used for for routing. Options are `accelerated`, `standard` and `off`. + +Default: `accelerated` + +### `RAINBOW_HTTP_ROUTERS` + +HTTP servers with /routing/v1 endpoints to use for delegated routing (comma-separated). + +Default: `https://cid.contact` + +## Experiments + ### `RAINBOW_SEED_PEERING` > [!WARNING] @@ -162,16 +189,10 @@ queries from these safelisted peers, serving locally cached blocks if requested. Default: `false` (no cache sharing) -### `RAINBOW_DHT_ROUTING` - -Control the type of Amino DHT client used for for routing. Options are `accelerated`, `standard` and `off`. - -Default: `accelerated` - ### `RAINBOW_REMOTE_BACKENDS` > [!WARNING] -> Experimental feature. Requires setting `RAINBOW_DHT_ROUTING=off` and `RAINBOW_BITSWAP=false`. +> Experimental feature, forces setting `RAINBOW_LIBP2P=false`. URL(s) of of remote [trustless gateways](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) to use as backend instead of libp2p node with Bitswap. @@ -189,6 +210,13 @@ Controls how requests to remote backend are made. Default: `block` +### `RAINBOW_REMOTE_BACKENDS_IPNS` + +Controls whether to fetch IPNS Records ([`application/vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record)) from trustless gateway defined in `RAINBOW_REMOTE_BACKENDS`. +This is done in addition to other routing systems, such as `RAINBOW_DHT_ROUTING` or `RAINBOW_HTTP_ROUTERS` (if also enabled). + +Default: `true` + ## Logging ### `GOLOG_LOG_LEVEL` diff --git a/main.go b/main.go index 66f0f6a..7f09cef 100644 --- a/main.go +++ b/main.go @@ -160,7 +160,7 @@ Generate an identity seed and launch a gateway: Name: "libp2p", Value: true, EnvVars: []string{"RAINBOW_LIBP2P"}, - Usage: "Enable or disable the usage of Libp2p", + Usage: "Controls if a local libp2p node is used (useful for testing or when remote backend is used instead)", }, &cli.IntFlag{ Name: "libp2p-connmgr-low", @@ -334,8 +334,19 @@ share the same seed as long as the indexes are different. bitswap := cctx.Bool("bitswap") dhtRouting := DHTRouting(cctx.String("dht-routing")) seedPeering := cctx.Bool("seed-peering") + libp2p := cctx.Bool("libp2p") + // as a convenience to the end user, and to reduce confusion + // libp2p is disabled when remote backends are defined + remoteBackends := cctx.StringSlice("remote-backends") + if len(remoteBackends) > 0 { + fmt.Printf("RAINBOW_REMOTE_BACKENDS set, forcing RAINBOW_LIBP2P=false\n") + libp2p = false + bitswap = false + dhtRouting = DHTOff + } + // Only load secrets if we need Libp2p. if libp2p { credDir := os.Getenv("CREDENTIALS_DIRECTORY") @@ -413,7 +424,7 @@ share the same seed as long as the indexes are different. SeedIndex: index, SeedPeering: seedPeering, SeedPeeringMaxIndex: cctx.Int("seed-peering-max-index"), - RemoteBackends: cctx.StringSlice("remote-backends"), + RemoteBackends: remoteBackends, RemoteBackendsIPNS: cctx.Bool("remote-backends-ipns"), RemoteBackendMode: RemoteBackendMode(cctx.String("remote-backends-mode")), GCInterval: cctx.Duration("gc-interval"), @@ -484,6 +495,8 @@ share the same seed as long as the indexes are different. printIfListConfigured(" RAINBOW_GATEWAY_DOMAINS = ", cfg.GatewayDomains) printIfListConfigured(" RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS = ", cfg.SubdomainGatewayDomains) printIfListConfigured(" RAINBOW_TRUSTLESS_GATEWAY_DOMAINS = ", cfg.TrustlessGatewayDomains) + printIfListConfigured(" RAINBOW_HTTP_ROUTERS = ", cfg.RoutingV1Endpoints) + printIfListConfigured(" RAINBOW_REMOTE_BACKENDS = ", cfg.RemoteBackends) fmt.Printf("\n") fmt.Printf("CTL endpoint listening at http://%s\n", ctlListen) diff --git a/setup.go b/setup.go index 7631423..f86c679 100644 --- a/setup.go +++ b/setup.go @@ -138,7 +138,7 @@ func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, // The stars aligned and Libp2p does not need to be turned on at all. if len(cfg.RemoteBackends) == 0 { - return nil, errors.New("URL of RAINBOW_REMOTE_BACKENDS must be set when RAINBOW_BITSWAP and RAINBOW_DHT_ROUTING are disabled") + return nil, errors.New("RAINBOW_REMOTE_BACKENDS must be set if RAINBOW_LIBP2P is disabled") } // Setup a Value Store composed of both the remote backends and the delegated @@ -175,8 +175,8 @@ func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, } func SetupWithLibp2p(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { - if !cfg.Bitswap && cfg.DHTRouting == DHTOff && !cfg.SeedPeering { - return nil, errors.New("libp2p is enabled, but not used: bitswap, dht and seed peering are disabled") + if !cfg.Bitswap && cfg.DHTRouting == DHTOff { + return nil, errors.New("libp2p is enabled, but not used: bitswap and dht are disabled") } var err error