Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature: delegated-routing: Add HTTP delegated routing. #9453

Merged
merged 5 commits into from
Dec 7, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions config/routing.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"encoding/json"
"fmt"
"runtime"
)

// Routing defines configuration options for libp2p routing
Expand Down Expand Up @@ -78,6 +79,8 @@ func (r *RouterParser) UnmarshalJSON(b []byte) error {

var p interface{}
switch out.Type {
case RouterTypeHTTP:
p = &HTTPRouterParams{}
case RouterTypeReframe:
p = &ReframeRouterParams{}
case RouterTypeDHT:
Expand All @@ -104,6 +107,7 @@ type RouterType string

const (
RouterTypeReframe RouterType = "reframe"
RouterTypeHTTP RouterType = "http"
RouterTypeDHT RouterType = "dht"
RouterTypeSequential RouterType = "sequential"
RouterTypeParallel RouterType = "parallel"
Expand Down Expand Up @@ -135,6 +139,28 @@ type ReframeRouterParams struct {
Endpoint string
}

type HTTPRouterParams struct {
// Endpoint is the URL where the routing implementation will point to get the information.
Endpoint string

// MaxProvideBatchSize determines the maximum amount of CIDs sent per batch.
// Servers might not accept more than 100 elements per batch. 100 elements by default.
MaxProvideBatchSize int

// MaxProvideConcurrency determines the number of threads used when providing content. GOMAXPROCS by default.
MaxProvideConcurrency int
}

func (hrp *HTTPRouterParams) FillDefaults() {
if hrp.MaxProvideBatchSize == 0 {
hrp.MaxProvideBatchSize = 100
}

if hrp.MaxProvideConcurrency == 0 {
hrp.MaxProvideConcurrency = runtime.GOMAXPROCS(0)
}
}

type DHTRouterParams struct {
Mode DHTMode
AcceleratedDHTClient bool `json:",omitempty"`
Expand Down
2 changes: 1 addition & 1 deletion core/node/libp2p/routingopt.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p
Datastore: dstore,
Context: ctx,
},
&irouting.ExtraReframeParams{
&irouting.ExtraHTTPParams{
PeerID: peerID,
Addrs: addrs,
PrivKeyB64: privKey,
Expand Down
10 changes: 10 additions & 0 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,10 @@ config file at runtime.
- [`Swarm.ConnMgr.HighWater`](#swarmconnmgrhighwater)
- [`Swarm.ConnMgr.GracePeriod`](#swarmconnmgrgraceperiod)
- [`Swarm.ResourceMgr`](#swarmresourcemgr)
- [Levels of Configuration](#levels-of-configuration)
- [Default Limits](#default-limits)
- [Active Limits](#active-limits)
- [libp2p resource monitoring](#libp2p-resource-monitoring)
- [`Swarm.ResourceMgr.Enabled`](#swarmresourcemgrenabled)
- [`Swarm.ResourceMgr.MaxMemory`](#swarmresourcemgrmaxmemory)
- [`Swarm.ResourceMgr.MaxFileDescriptors`](#swarmresourcemgrmaxfiledescriptors)
Expand Down Expand Up @@ -1342,6 +1346,7 @@ It specifies the routing type that will be created.
Currently supported types:

- `reframe` **(DEPRECATED)** (delegated routing based on the [reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme))
- `http` simple delegated routing based on HTTP protocol. <!-- TODO add link to specs when we have them merged. -->
- `dht`
- `parallel` and `sequential`: Helpers that can be used to run several routers sequentially or in parallel.

Expand All @@ -1356,6 +1361,11 @@ Parameters needed to create the specified router. Supported params per router ty
Reframe **(DEPRECATED)**:
- `Endpoint` (mandatory): URL that will be used to connect to a specified router.

HTTP:
- `Endpoint` (mandatory): URL that will be used to connect to a specified router.
- `MaxProvideBatchSize`: This number determines the maximum amount of CIDs sent per batch. Servers might not accept more than 100 elements per batch. 100 elements by default.
- `MaxProvideConcurrency`: It determines the number of threads used when providing content. GOMAXPROCS by default.

DHT:
- `"Mode"`: Mode used by the DHT. Possible values: "server", "client", "auto"
- `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT.
Expand Down
4 changes: 3 additions & 1 deletion docs/examples/kubo-as-a-library/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
Expand Down Expand Up @@ -95,6 +96,7 @@ require (
github.com/ipfs/go-ipld-git v0.1.1 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-ipns v0.3.0 // indirect
github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-merkledag v0.8.1 // indirect
Expand Down Expand Up @@ -173,9 +175,9 @@ require (
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/samber/lo v1.36.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stretchr/objx v0.4.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
Expand Down
9 changes: 7 additions & 2 deletions docs/examples/kubo-as-a-library/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,7 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORR
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
Expand Down Expand Up @@ -571,6 +572,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fO
github.com/ipfs/go-ipns v0.1.2/go.mod h1:ioQ0j02o6jdIVW+bmi18f4k2gRf0AV3kZ9KeHYHICnQ=
github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 h1:f4aUGWf4cqWJuikcwCPk/9GegRXcFMUx3V4soQ8ipT4=
github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068/go.mod h1:gAc/IsxQh4HwAOeSCKM1ONfzCQfNbm9E8QqEVfiPfOU=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
Expand Down Expand Up @@ -1338,6 +1341,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw=
github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down Expand Up @@ -1401,8 +1406,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
Expand All @@ -1414,6 +1418,7 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ require (
github.com/ipfs/go-ipld-git v0.1.1
github.com/ipfs/go-ipld-legacy v0.1.1
github.com/ipfs/go-ipns v0.3.0
github.com/ipfs/go-libipfs v0.0.0-20221130104825-592a45ae3796
github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068
github.com/ipfs/go-log v1.0.5
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.8.1
Expand Down Expand Up @@ -150,6 +150,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
Expand Down Expand Up @@ -220,6 +221,7 @@ require (
github.com/prometheus/statsd_exporter v0.21.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/samber/lo v1.36.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORR
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
Expand Down Expand Up @@ -593,8 +594,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fO
github.com/ipfs/go-ipns v0.1.2/go.mod h1:ioQ0j02o6jdIVW+bmi18f4k2gRf0AV3kZ9KeHYHICnQ=
github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
github.com/ipfs/go-libipfs v0.0.0-20221130104825-592a45ae3796 h1:2aZUmUq+4C8Vk+pbZk3IU48H2GAZ5/kOTrbuCwIt9HI=
github.com/ipfs/go-libipfs v0.0.0-20221130104825-592a45ae3796/go.mod h1:gAc/IsxQh4HwAOeSCKM1ONfzCQfNbm9E8QqEVfiPfOU=
github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068 h1:f4aUGWf4cqWJuikcwCPk/9GegRXcFMUx3V4soQ8ipT4=
github.com/ipfs/go-libipfs v0.0.0-20221206130039-8dda74356068/go.mod h1:gAc/IsxQh4HwAOeSCKM1ONfzCQfNbm9E8QqEVfiPfOU=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
Expand Down Expand Up @@ -1399,6 +1400,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw=
github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down Expand Up @@ -1480,6 +1483,7 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
Expand Down
91 changes: 76 additions & 15 deletions routing/delegated.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"github.com/ipfs/go-datastore"
drc "github.com/ipfs/go-delegated-routing/client"
drp "github.com/ipfs/go-delegated-routing/gen/proto"
drclient "github.com/ipfs/go-libipfs/routing/http/client"
"github.com/ipfs/go-libipfs/routing/http/contentrouter"
logging "github.com/ipfs/go-log"
"github.com/ipfs/kubo/config"
dht "github.com/libp2p/go-libp2p-kad-dht"
Expand All @@ -28,7 +30,7 @@ import (

var log = logging.Logger("routing/delegated")

func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTParams, extraReframe *ExtraReframeParams) (routing.Routing, error) {
func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTParams, extraHTTP *ExtraHTTPParams) (routing.Routing, error) {
if err := methods.Check(); err != nil {
return nil, err
}
Expand All @@ -38,7 +40,7 @@ func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTPar

// Create all needed routers from method names
for mn, m := range methods {
router, err := parse(make(map[string]bool), createdRouters, m.RouterName, routers, extraDHT, extraReframe)
router, err := parse(make(map[string]bool), createdRouters, m.RouterName, routers, extraDHT, extraHTTP)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -67,7 +69,7 @@ func parse(visited map[string]bool,
routerName string,
routersCfg config.Routers,
extraDHT *ExtraDHTParams,
extraReframe *ExtraReframeParams,
extraHTTP *ExtraHTTPParams,
) (routing.Routing, error) {
// check if we already created it
r, ok := createdRouters[routerName]
Expand All @@ -91,15 +93,17 @@ func parse(visited map[string]bool,
var router routing.Routing
var err error
switch cfg.Type {
case config.RouterTypeHTTP:
router, err = httpRoutingFromConfig(cfg.Router, extraHTTP)
case config.RouterTypeReframe:
router, err = reframeRoutingFromConfig(cfg.Router, extraReframe)
router, err = reframeRoutingFromConfig(cfg.Router, extraHTTP)
case config.RouterTypeDHT:
router, err = dhtRoutingFromConfig(cfg.Router, extraDHT)
case config.RouterTypeParallel:
crp := cfg.Parameters.(*config.ComposableRouterParams)
var pr []*routinghelpers.ParallelRouter
for _, cr := range crp.Routers {
ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraReframe)
ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraHTTP)
if err != nil {
return nil, err
}
Expand All @@ -118,7 +122,7 @@ func parse(visited map[string]bool,
crp := cfg.Parameters.(*config.ComposableRouterParams)
var sr []*routinghelpers.SequentialRouter
for _, cr := range crp.Routers {
ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraReframe)
ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraHTTP)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -147,13 +151,62 @@ func parse(visited map[string]bool,
return router, nil
}

type ExtraReframeParams struct {
type ExtraHTTPParams struct {
PeerID string
Addrs []string
PrivKeyB64 string
}

func reframeRoutingFromConfig(conf config.Router, extraReframe *ExtraReframeParams) (routing.Routing, error) {
func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (routing.Routing, error) {
params := conf.Parameters.(*config.HTTPRouterParams)
if params.Endpoint == "" {
return nil, NewParamNeededErr("Endpoint", conf.Type)
}

params.FillDefaults()

// Increase per-host connection pool since we are making lots of concurrent requests.
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.MaxIdleConns = 500
transport.MaxIdleConnsPerHost = 100

delegateHTTPClient := &http.Client{
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will need to wrap this in https://github.com/ipfs/go-libipfs/blob/main/routing/http/client/transport.go#L9 but I can take care of that in a followup PR

Transport: transport,
}

key, err := decodePrivKey(extraHTTP.PrivKeyB64)
if err != nil {
return nil, err
}

addrInfo, err := createAddrInfo(extraHTTP.PeerID, extraHTTP.Addrs)
if err != nil {
return nil, err
}

cli, err := drclient.New(
params.Endpoint,
drclient.WithHTTPClient(delegateHTTPClient),
drclient.WithIdentity(key),
drclient.WithProviderInfo(addrInfo.ID, addrInfo.Addrs),
)
if err != nil {
return nil, err
}

cr := contentrouter.NewContentRoutingClient(
cli,
contentrouter.WithMaxProvideBatchSize(params.MaxProvideBatchSize),
contentrouter.WithMaxProvideConcurrency(params.MaxProvideConcurrency),
)

return &httpRoutingWrapper{
ContentRouting: cr,
ProvideManyRouter: cr,
}, nil
}

func reframeRoutingFromConfig(conf config.Router, extraReframe *ExtraHTTPParams) (routing.Routing, error) {
var dr drp.DelegatedRouting_Client

params := conf.Parameters.(*config.ReframeRouterParams)
Expand Down Expand Up @@ -223,27 +276,35 @@ func decodePrivKey(keyB64 string) (ic.PrivKey, error) {
return ic.UnmarshalPrivateKey(pk)
}

func createProvider(peerID string, addrs []string) (*drc.Provider, error) {
func createAddrInfo(peerID string, addrs []string) (peer.AddrInfo, error) {
pID, err := peer.Decode(peerID)
if err != nil {
return nil, err
return peer.AddrInfo{}, err
}

var mas []ma.Multiaddr
for _, a := range addrs {
m, err := ma.NewMultiaddr(a)
if err != nil {
return nil, err
return peer.AddrInfo{}, err
}

mas = append(mas, m)
}

return peer.AddrInfo{
ID: pID,
Addrs: mas,
}, nil
}

func createProvider(peerID string, addrs []string) (*drc.Provider, error) {
addrInfo, err := createAddrInfo(peerID, addrs)
if err != nil {
return nil, err
}
return &drc.Provider{
Peer: peer.AddrInfo{
ID: pID,
Addrs: mas,
},
Peer: addrInfo,
ProviderProto: []drc.TransferProtocol{
{Codec: multicodec.TransportBitswap},
},
Expand Down
Loading