diff --git a/config/routing.go b/config/routing.go index cecba7a5edd..90d91d28b1b 100644 --- a/config/routing.go +++ b/config/routing.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "fmt" + "runtime" ) // Routing defines configuration options for libp2p routing @@ -78,6 +79,8 @@ func (r *RouterParser) UnmarshalJSON(b []byte) error { var p interface{} switch out.Type { + case RouterTypeHTTP: + p = &HTTPRouterParams{} case RouterTypeReframe: p = &ReframeRouterParams{} case RouterTypeDHT: @@ -104,6 +107,7 @@ type RouterType string const ( RouterTypeReframe RouterType = "reframe" + RouterTypeHTTP RouterType = "http" RouterTypeDHT RouterType = "dht" RouterTypeSequential RouterType = "sequential" RouterTypeParallel RouterType = "parallel" @@ -135,6 +139,28 @@ type ReframeRouterParams struct { Endpoint string } +type HTTPRouterParams struct { + // Endpoint is the URL where the routing implementation will point to get the information. + Endpoint string + + // MaxProvideBatchSize determines the maximum amount of CIDs sent per batch. + // Servers might not accept more than 100 elements per batch. 100 elements by default. + MaxProvideBatchSize int + + // MaxProvideConcurrency determines the number of threads used when providing content. GOMAXPROCS by default. + MaxProvideConcurrency int +} + +func (hrp *HTTPRouterParams) FillDefaults() { + if hrp.MaxProvideBatchSize == 0 { + hrp.MaxProvideBatchSize = 100 + } + + if hrp.MaxProvideConcurrency == 0 { + hrp.MaxProvideConcurrency = runtime.GOMAXPROCS(0) + } +} + type DHTRouterParams struct { Mode DHTMode AcceleratedDHTClient bool `json:",omitempty"` diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 262501c69b1..9f3ae5c06e3 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -71,7 +71,7 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p Datastore: dstore, Context: ctx, }, - &irouting.ExtraReframeParams{ + &irouting.ExtraHTTPParams{ PeerID: peerID, Addrs: addrs, PrivKeyB64: privKey, diff --git a/docs/config.md b/docs/config.md index ba775ad3b73..79a55cbb0da 100644 --- a/docs/config.md +++ b/docs/config.md @@ -141,6 +141,10 @@ config file at runtime. - [`Swarm.ConnMgr.HighWater`](#swarmconnmgrhighwater) - [`Swarm.ConnMgr.GracePeriod`](#swarmconnmgrgraceperiod) - [`Swarm.ResourceMgr`](#swarmresourcemgr) + - [Levels of Configuration](#levels-of-configuration) + - [Default Limits](#default-limits) + - [Active Limits](#active-limits) + - [libp2p resource monitoring](#libp2p-resource-monitoring) - [`Swarm.ResourceMgr.Enabled`](#swarmresourcemgrenabled) - [`Swarm.ResourceMgr.MaxMemory`](#swarmresourcemgrmaxmemory) - [`Swarm.ResourceMgr.MaxFileDescriptors`](#swarmresourcemgrmaxfiledescriptors) @@ -1342,6 +1346,7 @@ It specifies the routing type that will be created. Currently supported types: - `reframe` **(DEPRECATED)** (delegated routing based on the [reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme)) +- `http` simple delegated routing based on HTTP protocol. - `dht` - `parallel` and `sequential`: Helpers that can be used to run several routers sequentially or in parallel. @@ -1356,6 +1361,11 @@ Parameters needed to create the specified router. Supported params per router ty Reframe **(DEPRECATED)**: - `Endpoint` (mandatory): URL that will be used to connect to a specified router. +HTTP: + - `Endpoint` (mandatory): URL that will be used to connect to a specified router. + - `MaxProvideBatchSize`: This number determines the maximum amount of CIDs sent per batch. Servers might not accept more than 100 elements per batch. 100 elements by default. + - `MaxProvideConcurrency`: It determines the number of threads used when providing content. GOMAXPROCS by default. + DHT: - `"Mode"`: Mode used by the DHT. Possible values: "server", "client", "auto" - `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 71b5db35200..ffc127d6ced 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -53,6 +53,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect @@ -95,6 +96,7 @@ require ( github.com/ipfs/go-ipld-git v0.1.1 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect github.com/ipfs/go-ipns v0.3.0 // indirect + github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ipfs/go-merkledag v0.8.1 // indirect @@ -173,9 +175,9 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/samber/lo v1.36.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/stretchr/objx v0.4.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 311c2a3b29d..5638eaec077 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -360,6 +360,7 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORR github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -571,6 +572,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fO github.com/ipfs/go-ipns v0.1.2/go.mod h1:ioQ0j02o6jdIVW+bmi18f4k2gRf0AV3kZ9KeHYHICnQ= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 h1:f4aUGWf4cqWJuikcwCPk/9GegRXcFMUx3V4soQ8ipT4= +github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068/go.mod h1:gAc/IsxQh4HwAOeSCKM1ONfzCQfNbm9E8QqEVfiPfOU= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -1338,6 +1341,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= +github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1401,8 +1406,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1414,6 +1418,7 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= diff --git a/go.mod b/go.mod index bf23b7aa14b..4400df79c08 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/ipfs/go-ipld-git v0.1.1 github.com/ipfs/go-ipld-legacy v0.1.1 github.com/ipfs/go-ipns v0.3.0 - github.com/ipfs/go-libipfs v0.0.0-20221130104825-592a45ae3796 + github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.8.1 @@ -150,6 +150,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect @@ -220,6 +221,7 @@ require ( github.com/prometheus/statsd_exporter v0.21.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rs/cors v1.7.0 // indirect + github.com/samber/lo v1.36.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect diff --git a/go.sum b/go.sum index 876aa208e76..74bde0f9b80 100644 --- a/go.sum +++ b/go.sum @@ -378,6 +378,7 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORR github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -593,8 +594,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fO github.com/ipfs/go-ipns v0.1.2/go.mod h1:ioQ0j02o6jdIVW+bmi18f4k2gRf0AV3kZ9KeHYHICnQ= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.0.0-20221130104825-592a45ae3796 h1:2aZUmUq+4C8Vk+pbZk3IU48H2GAZ5/kOTrbuCwIt9HI= -github.com/ipfs/go-libipfs v0.0.0-20221130104825-592a45ae3796/go.mod h1:gAc/IsxQh4HwAOeSCKM1ONfzCQfNbm9E8QqEVfiPfOU= +github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 h1:f4aUGWf4cqWJuikcwCPk/9GegRXcFMUx3V4soQ8ipT4= +github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068/go.mod h1:gAc/IsxQh4HwAOeSCKM1ONfzCQfNbm9E8QqEVfiPfOU= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -1399,6 +1400,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= +github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1480,6 +1483,7 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= diff --git a/routing/delegated.go b/routing/delegated.go index 61420efa7be..953cfac00e0 100644 --- a/routing/delegated.go +++ b/routing/delegated.go @@ -10,6 +10,8 @@ import ( "github.com/ipfs/go-datastore" drc "github.com/ipfs/go-delegated-routing/client" drp "github.com/ipfs/go-delegated-routing/gen/proto" + drclient "github.com/ipfs/go-libipfs/routing/http/client" + "github.com/ipfs/go-libipfs/routing/http/contentrouter" logging "github.com/ipfs/go-log" "github.com/ipfs/kubo/config" dht "github.com/libp2p/go-libp2p-kad-dht" @@ -28,7 +30,7 @@ import ( var log = logging.Logger("routing/delegated") -func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTParams, extraReframe *ExtraReframeParams) (routing.Routing, error) { +func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTParams, extraHTTP *ExtraHTTPParams) (routing.Routing, error) { if err := methods.Check(); err != nil { return nil, err } @@ -38,7 +40,7 @@ func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTPar // Create all needed routers from method names for mn, m := range methods { - router, err := parse(make(map[string]bool), createdRouters, m.RouterName, routers, extraDHT, extraReframe) + router, err := parse(make(map[string]bool), createdRouters, m.RouterName, routers, extraDHT, extraHTTP) if err != nil { return nil, err } @@ -67,7 +69,7 @@ func parse(visited map[string]bool, routerName string, routersCfg config.Routers, extraDHT *ExtraDHTParams, - extraReframe *ExtraReframeParams, + extraHTTP *ExtraHTTPParams, ) (routing.Routing, error) { // check if we already created it r, ok := createdRouters[routerName] @@ -91,15 +93,17 @@ func parse(visited map[string]bool, var router routing.Routing var err error switch cfg.Type { + case config.RouterTypeHTTP: + router, err = httpRoutingFromConfig(cfg.Router, extraHTTP) case config.RouterTypeReframe: - router, err = reframeRoutingFromConfig(cfg.Router, extraReframe) + router, err = reframeRoutingFromConfig(cfg.Router, extraHTTP) case config.RouterTypeDHT: router, err = dhtRoutingFromConfig(cfg.Router, extraDHT) case config.RouterTypeParallel: crp := cfg.Parameters.(*config.ComposableRouterParams) var pr []*routinghelpers.ParallelRouter for _, cr := range crp.Routers { - ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraReframe) + ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraHTTP) if err != nil { return nil, err } @@ -118,7 +122,7 @@ func parse(visited map[string]bool, crp := cfg.Parameters.(*config.ComposableRouterParams) var sr []*routinghelpers.SequentialRouter for _, cr := range crp.Routers { - ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraReframe) + ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraHTTP) if err != nil { return nil, err } @@ -147,13 +151,62 @@ func parse(visited map[string]bool, return router, nil } -type ExtraReframeParams struct { +type ExtraHTTPParams struct { PeerID string Addrs []string PrivKeyB64 string } -func reframeRoutingFromConfig(conf config.Router, extraReframe *ExtraReframeParams) (routing.Routing, error) { +func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (routing.Routing, error) { + params := conf.Parameters.(*config.HTTPRouterParams) + if params.Endpoint == "" { + return nil, NewParamNeededErr("Endpoint", conf.Type) + } + + params.FillDefaults() + + // Increase per-host connection pool since we are making lots of concurrent requests. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConns = 500 + transport.MaxIdleConnsPerHost = 100 + + delegateHTTPClient := &http.Client{ + Transport: transport, + } + + key, err := decodePrivKey(extraHTTP.PrivKeyB64) + if err != nil { + return nil, err + } + + addrInfo, err := createAddrInfo(extraHTTP.PeerID, extraHTTP.Addrs) + if err != nil { + return nil, err + } + + cli, err := drclient.New( + params.Endpoint, + drclient.WithHTTPClient(delegateHTTPClient), + drclient.WithIdentity(key), + drclient.WithProviderInfo(addrInfo.ID, addrInfo.Addrs), + ) + if err != nil { + return nil, err + } + + cr := contentrouter.NewContentRoutingClient( + cli, + contentrouter.WithMaxProvideBatchSize(params.MaxProvideBatchSize), + contentrouter.WithMaxProvideConcurrency(params.MaxProvideConcurrency), + ) + + return &httpRoutingWrapper{ + ContentRouting: cr, + ProvideManyRouter: cr, + }, nil +} + +func reframeRoutingFromConfig(conf config.Router, extraReframe *ExtraHTTPParams) (routing.Routing, error) { var dr drp.DelegatedRouting_Client params := conf.Parameters.(*config.ReframeRouterParams) @@ -223,27 +276,35 @@ func decodePrivKey(keyB64 string) (ic.PrivKey, error) { return ic.UnmarshalPrivateKey(pk) } -func createProvider(peerID string, addrs []string) (*drc.Provider, error) { +func createAddrInfo(peerID string, addrs []string) (peer.AddrInfo, error) { pID, err := peer.Decode(peerID) if err != nil { - return nil, err + return peer.AddrInfo{}, err } var mas []ma.Multiaddr for _, a := range addrs { m, err := ma.NewMultiaddr(a) if err != nil { - return nil, err + return peer.AddrInfo{}, err } mas = append(mas, m) } + return peer.AddrInfo{ + ID: pID, + Addrs: mas, + }, nil +} + +func createProvider(peerID string, addrs []string) (*drc.Provider, error) { + addrInfo, err := createAddrInfo(peerID, addrs) + if err != nil { + return nil, err + } return &drc.Provider{ - Peer: peer.AddrInfo{ - ID: pID, - Addrs: mas, - }, + Peer: addrInfo, ProviderProto: []drc.TransferProtocol{ {Codec: multicodec.TransportBitswap}, }, diff --git a/routing/delegated_test.go b/routing/delegated_test.go index 04b8e282fba..ee7543114d9 100644 --- a/routing/delegated_test.go +++ b/routing/delegated_test.go @@ -45,7 +45,7 @@ func TestReframeRoutingFromConfig(t *testing.T) { Parameters: &config.ReframeRouterParams{ Endpoint: "test", }, - }, &ExtraReframeParams{ + }, &ExtraHTTPParams{ PeerID: id.String(), Addrs: []string{"/ip4/0.0.0.0/tcp/4001"}, PrivKeyB64: base64.StdEncoding.EncodeToString(privM), diff --git a/routing/wrapper.go b/routing/wrapper.go index 3a64d6a6373..d4215ca9c90 100644 --- a/routing/wrapper.go +++ b/routing/wrapper.go @@ -40,3 +40,35 @@ type ProvideManyRouter interface { routinghelpers.ProvideManyRouter routing.Routing } + +var _ routing.Routing = &httpRoutingWrapper{} +var _ routinghelpers.ProvideManyRouter = &httpRoutingWrapper{} + +// httpRoutingWrapper is a wrapper needed to construct the routing.Routing interface from +// http delegated routing. +type httpRoutingWrapper struct { + routing.ContentRouting + routinghelpers.ProvideManyRouter +} + +func (c *httpRoutingWrapper) Bootstrap(ctx context.Context) error { + return nil +} + +func (c *httpRoutingWrapper) FindPeer(ctx context.Context, id peer.ID) (peer.AddrInfo, error) { + return peer.AddrInfo{}, routing.ErrNotSupported +} + +func (c *httpRoutingWrapper) PutValue(context.Context, string, []byte, ...routing.Option) error { + return routing.ErrNotSupported +} + +func (c *httpRoutingWrapper) GetValue(context.Context, string, ...routing.Option) ([]byte, error) { + return nil, routing.ErrNotSupported +} + +func (c *httpRoutingWrapper) SearchValue(context.Context, string, ...routing.Option) (<-chan []byte, error) { + out := make(chan []byte) + close(out) + return out, routing.ErrNotSupported +} diff --git a/test/sharness/t0702-delegated-routing-http.sh b/test/sharness/t0702-delegated-routing-http.sh new file mode 100755 index 00000000000..03f452fd4d7 --- /dev/null +++ b/test/sharness/t0702-delegated-routing-http.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +test_description="Test delegated routing via HTTP endpoint" + +. lib/test-lib.sh + +if ! test_have_prereq SOCAT; then + skip_all="skipping '$test_description': socat is not available" + test_done +fi + +# simple http routing server mock +# local endpoint responds with deterministic application/vnd.ipfs.rpc+dag-json; version=1 +HTTP_ROUTING_PORT=5098 +function start_http_routing_mock_endpoint() { + REMOTE_SERVER_LOG="http-routing-server.log" + rm -f $REMOTE_SERVER_LOG + + touch response + socat tcp-listen:$HTTP_ROUTING_PORT,fork,bind=127.0.0.1,reuseaddr 'SYSTEM:cat response'!!CREATE:$REMOTE_SERVER_LOG & + REMOTE_SERVER_PID=$! + + socat /dev/null tcp:127.0.0.1:$HTTP_ROUTING_PORT,retry=10 + return $? +} +function serve_http_routing_response() { + local body=$1 + local status_code=${2:-"200 OK"} + local length=$((1 + ${#body})) + echo -e "HTTP/1.1 $status_code\nContent-Length: $length\nContent-Type: application/json\n\n$body" > response +} +function stop_http_routing_mock_endpoint() { + exec 7<&- + kill $REMOTE_SERVER_PID > /dev/null 2>&1 + wait $REMOTE_SERVER_PID || true +} + +# daemon running in online mode to ensure Pin.origins/PinStatus.delegates work +test_init_ipfs + +# based on static, synthetic http routing messages: +# t0702-delegated-routing-http/FindProvidersRequest +# t0702-delegated-routing-http/FindProvidersResponse +FINDPROV_CID="baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq" +EXPECTED_PROV="12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ" + +test_expect_success "default Routing config has no Routers defined" ' + echo null > expected && + ipfs config show | jq .Routing.Routers > actual && + test_cmp expected actual +' + +# turn off all implicit routers +ipfs config Routing.Type none || exit 1 +test_launch_ipfs_daemon +test_expect_success "disabling default router (dht) works" ' + ipfs config Routing.Type > actual && + echo none > expected && + test_cmp expected actual +' +test_expect_success "no routers means findprovs returns no results" ' + ipfs routing findprovs "$FINDPROV_CID" > actual && + echo -n > expected && + test_cmp expected actual +' + +test_kill_ipfs_daemon + +ipfs config Routing.Type --json '"custom"' || exit 1 +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "TestDelegatedRouter" + }, + "find-providers": { + "RouterName": "TestDelegatedRouter" + }, + "get-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "provide": { + "RouterName": "TestDelegatedRouter" + } + }' || exit 1 + +test_expect_success "missing method params makes daemon fails" ' + echo "Error: constructing the node (see log for full detail): method name \"put-ipns\" is missing from Routing.Methods config param" > expected_error && + GOLOG_LOG_LEVEL=fatal ipfs daemon 2> actual_error || exit 0 && + test_cmp expected_error actual_error +' + +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "TestDelegatedRouter" + }, + "find-providers": { + "RouterName": "TestDelegatedRouter" + }, + "get-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "provide": { + "RouterName": "TestDelegatedRouter" + }, + "put-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "NOT_SUPPORTED": { + "RouterName": "TestDelegatedRouter" + } + }' || exit 1 + +test_expect_success "having wrong methods makes daemon fails" ' + echo "Error: constructing the node (see log for full detail): method name \"NOT_SUPPORTED\" is not a supported method on Routing.Methods config param" > expected_error && + GOLOG_LOG_LEVEL=fatal ipfs daemon 2> actual_error || exit 0 && + test_cmp expected_error actual_error +' + +# set Routing config to only use delegated routing via mocked http routing endpoint + +ipfs config Routing.Type --json '"custom"' || exit 1 +ipfs config Routing.Routers.TestDelegatedRouter --json '{ + "Type": "http", + "Parameters": { + "Endpoint": "http://127.0.0.1:5098/routing/v1" + } +}' || exit 1 +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "TestDelegatedRouter" + }, + "find-providers": { + "RouterName": "TestDelegatedRouter" + }, + "get-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "provide": { + "RouterName": "TestDelegatedRouter" + }, + "put-ipns": { + "RouterName": "TestDelegatedRouter" + } + }' || exit 1 + +test_expect_success "adding http delegated routing endpoint to Routing.Routers config works" ' + echo "http://127.0.0.1:5098/routing/v1" > expected && + ipfs config Routing.Routers.TestDelegatedRouter.Parameters.Endpoint > actual && + test_cmp expected actual +' + +test_launch_ipfs_daemon + +test_expect_success "start_http_routing_mock_endpoint" ' + start_http_routing_mock_endpoint +' + +test_expect_success "'ipfs routing findprovs' returns result from delegated http router" ' + serve_http_routing_response "$(<../t0702-delegated-routing-http/FindProvidersResponse)" && + echo "$EXPECTED_PROV" > expected && + ipfs routing findprovs "$FINDPROV_CID" > actual && + test_cmp expected actual +' + +test_expect_success "stop_http_routing_mock_endpoint" ' + stop_http_routing_mock_endpoint +' + + +test_kill_ipfs_daemon +test_done +# vim: ts=2 sw=2 sts=2 et: diff --git a/test/sharness/t0702-delegated-routing-http/FindProvidersResponse b/test/sharness/t0702-delegated-routing-http/FindProvidersResponse new file mode 100644 index 00000000000..16815329ab8 --- /dev/null +++ b/test/sharness/t0702-delegated-routing-http/FindProvidersResponse @@ -0,0 +1 @@ +{"Providers":[{"Protocol":"bitswap","ID":"12D3KooWARYacCc6eoCqvsS9RW9MA2vo51CV75deoiqssx3YgyYJ","Addrs":["/ip4/0.0.0.0/tcp/4001","/ip4/0.0.0.0/tcp/4002"]}]} \ No newline at end of file