Skip to content

Commit

Permalink
refactor(ci): libp2p conformance is separate job
Browse files Browse the repository at this point in the history 
this ensures the libp2p experiment runs independently
and its failure does not impact the result of job
that tests stable features on http port
  • Loading branch information
@lidel
lidel committed Sep 5, 2023
1 parent df66dac commit 5497023
Showing 1 changed file with 84 additions and 73 deletions.
157 changes: 84 additions & 73 deletions .github/workflows/gateway-conformance.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,25 @@ defaults:
run:
shell: bash

env:
# hostnames expected by https://github.com/ipfs/gateway-conformance
GATEWAY_PUBLIC_GATEWAYS: |
{
"example.com": {
"UseSubdomains": true,
"InlineDNSLink": true,
"Paths": ["/ipfs", "/ipns"]
},
"localhost": {
"UseSubdomains": true,
"InlineDNSLink": true,
"Paths": ["/ipfs", "/ipns"]
}
}
jobs:
gateway-conformance:
# Testing all gateway features via TCP port specified in Addresses.Gateway
full-over-http:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
Expand All @@ -30,11 +47,14 @@ jobs:

# 2. Build the kubo-gateway
- name: Setup Go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: 1.20.x
- uses: protocol/cache-go-action@v1
with:
name: ${{ github.job }}
- name: Checkout kubo-gateway
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
path: kubo-gateway
- name: Build kubo-gateway
Expand All @@ -43,26 +63,9 @@ jobs:

# 3. Init the kubo-gateway
- name: Init kubo-gateway
env:
GATEWAY_PUBLIC_GATEWAYS: |
{
"example.com": {
"UseSubdomains": true,
"InlineDNSLink": true,
"Paths": ["/ipfs", "/ipns"]
},
"localhost": {
"UseSubdomains": true,
"InlineDNSLink": true,
"Paths": ["/ipfs", "/ipns"]
}
}
run: |
./ipfs init --profile=test
./ipfs init -e
./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
./ipfs config --json Experimental.GatewayOverLibp2p true
./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
working-directory: kubo-gateway/cmd/ipfs

# 4. Populate the Kubo gateway with the gateway-conformance fixtures
Expand All @@ -88,28 +91,7 @@ jobs:
# 5. Start the kubo-gateway
- name: Start kubo-gateway
run: |
./ipfs daemon &
endpoint="http://127.0.0.1:5001/api/v0/version"
max_retries=5
retry_interval=3
check_endpoint() {
curl -X POST --silent --fail "$endpoint" > /dev/null
return $?
}
retries=0
while ! check_endpoint; do
retries=$((retries+1))
if [ $retries -ge $max_retries ]; then
echo "daemon took too long to start"
exit 1
fi
sleep $retry_interval
done
echo "daemon started and ready to receive API calls"
./ipfs daemon --offline &
working-directory: kubo-gateway/cmd/ipfs

# 6. Run the gateway-conformance tests
Expand Down Expand Up @@ -140,56 +122,85 @@ jobs:
name: gateway-conformance.json
path: output.json

# 8. Setup a kubo http-p2p-proxy to run gateway conformance tests
# Testing trustless gateway feature subset exposed as libp2p protocol
subset-over-libp2p-experiment:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
# 1. Download the gateway-conformance fixtures
- name: Download gateway-conformance fixtures
uses: ipfs/gateway-conformance/.github/actions/[email protected]
with:
output: fixtures

# 2. Build the kubo-gateway
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: 1.20.x
- uses: protocol/cache-go-action@v1
with:
name: ${{ github.job }}
- name: Checkout kubo-gateway
uses: actions/checkout@v3
with:
path: kubo-gateway
- name: Build kubo-gateway
run: make build
working-directory: kubo-gateway

# 3. Init the kubo-gateway
- name: Init kubo-gateway
run: |
./ipfs init --profile=test
./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
./ipfs config --json Experimental.GatewayOverLibp2p true
./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
working-directory: kubo-gateway/cmd/ipfs

# 4. Populate the Kubo gateway with the gateway-conformance fixtures
- name: Import fixtures
run: |
# Import car files
find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
# 5. Start the kubo-gateway
- name: Start kubo-gateway
run: |
( ipfs daemon & ) | sed '/Daemon is ready/q'
while [[ "$(ipfs id --api=/ip4/127.0.0.1/tcp/5001 | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
working-directory: kubo-gateway/cmd/ipfs

# 6. Setup a kubo http-p2p-proxy to expose libp2p protocol as a regular HTTP port for gateway conformance tests
- name: Init p2p-proxy kubo node
env:
IPFS_PATH: "~/.kubo-p2p-proxy"
run: |
./ipfs init --profile=test
./ipfs init --profile=test -e
./ipfs config --json Experimental.Libp2pStreamMounting true
./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081"
./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002"
working-directory: kubo-gateway/cmd/ipfs

# 9. Start the kubo http-p2p-proxy
# 7. Start the kubo http-p2p-proxy
- name: Start kubo http-p2p-proxy
env:
IPFS_PATH: "~/.kubo-p2p-proxy"
run: |
./ipfs daemon &
endpoint="http://127.0.0.1:5002/api/v0/version"
max_retries=5
retry_interval=3
check_endpoint() {
curl -X POST --silent --fail "$endpoint" > /dev/null
return $?
}
retries=0
while ! check_endpoint; do
retries=$((retries+1))
if [ $retries -ge $max_retries ]; then
echo "daemon took too long to start"
exit 1
fi
sleep $retry_interval
done
echo "daemon started and ready to receive API calls"
( ipfs daemon & ) | sed '/Daemon is ready/q'
while [[ "$(ipfs id --api=/ip4/127.0.0.1/tcp/5002 | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
working-directory: kubo-gateway/cmd/ipfs

# 10. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
# 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
- name: Start http-over-libp2p forwarding proxy
run: |
gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="<id>")
./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1)
./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId
working-directory: kubo-gateway/cmd/ipfs

# 11. Run the gateway-conformance tests over libp2p
# 9. Run the gateway-conformance tests over libp2p
- name: Run gateway-conformance tests over libp2p
uses: ipfs/gateway-conformance/.github/actions/[email protected]
with:
Expand All @@ -200,7 +211,7 @@ jobs:
markdown: output.md
args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'

# 11. Upload the results
# 10. Upload the results
- name: Upload MD summary
if: failure() || success()
run: cat output.md >> $GITHUB_STEP_SUMMARY
Expand All @@ -215,4 +226,4 @@ jobs:
uses: actions/upload-artifact@v3
with:
name: gateway-conformance-libp2p.json
path: output.json
path: output.json

0 comments on commit 5497023

Please sign in to comment.