diff --git a/.github/workflows/django-test.yml b/.github/workflows/django-test.yml index 1507a26a2..d34c79a5f 100644 --- a/.github/workflows/django-test.yml +++ b/.github/workflows/django-test.yml @@ -15,68 +15,63 @@ concurrency: cancel-in-progress: true jobs: - build: + test: runs-on: ubuntu-latest strategy: - max-parallel: 4 + max-parallel: 1 matrix: - python-version: ["3.11.6", "3.12"] - lnd-version: ["v0.17.0-beta","v0.17.1-beta.rc1"] + python-tag: ['3.11.6-slim-bookworm', '3.12-slim-bookworm'] + lnd-version: ["v0.17.0-beta", "v0.17.0-beta.rc1"] + cln-version: ["v23.08.1"] + ln-vendor: ["LND", "CLN"] + # bitcoind-version: ["24.0.1"] + # include: + # - lnd-version: "v0.17.0-beta" + # ln-vendor: "LND" + # - lnd-version: "v0.17.1-beta.rc1" + # ln-vendor: "LND" + # - cln-version: "v23.08.1" + # ln-vendor: "CLN" steps: - name: 'Checkout' uses: actions/checkout@v4 - - name: 'Compose Eegtest Orchestration' + - name: Update Python version in Dockerfile + run: | + sed -i "1s/FROM python:.*/FROM python:${{ matrix.python-tag }}/" Dockerfile + cat ./Dockerfile + + - name: 'Compose Regtest Orchestration' uses: isbang/compose-action@v1.5.1 with: - compose-file: "docker-test.yml" - env: "tests/compose.env" - - # - name: 'Set up Python ${{ matrix.python-version }}' - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ matrix.python-version }} - - # - name: 'Cache pip dependencies' - # uses: actions/cache@v3 - # with: - # path: ~/.cache/pip - # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - # restore-keys: | - # ${{ runner.os }}-pip- - - # - name: 'Install Python Dependencies' - # run: | - # python -m pip install --upgrade pip - # pip install -r requirements.txt - # pip install -r requirements_dev.txt + compose-file: "./docker-tests.yml" + down-flags: "--volumes" + services: | + bitcoind + postgres + redis + coordinator-${{ matrix.ln-vendor }} + robot-LND + coordinator + env: + LND_VERSION: ${{ matrix.lnd-version }} + CLN_VERSION: ${{ matrix.cln-version }} + BITCOIND_VERSION: ${{ matrix.bitcoind-version }} + ROBOSATS_ENVS_FILE: ".env-sample" - # - name: 'Install LND/CLN gRPC Dependencies' - # run: bash ./scripts/generate_grpc.sh - - # - name: 'Create .env File' - # run: | - # mv .env-sample .env - # sed -i "s/USE_TOR=True/USE_TOR=False/" .env - - # - name: 'Wait for PostgreSQL to become ready' - # run: | - # sudo apt-get install -y postgresql-client - # until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done + - name: Wait for entrypoint to install dev deps + run: | + while ! docker exec coordinator coverage --version >/dev/null 2>&1; do + echo "Waiting for coverage to be installed..." + sleep 5 + done - name: 'Run tests with coverage' run: | docker exec coordinator coverage run manage.py test docker exec coordinator coverage report - - -# jobs: -# test: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# - name: Run Docker Compose -# run: | -# docker-compose up -d -# docker-compose run web python manage.py test \ No newline at end of file + env: + LNVENDOR: ${{ matrix.ln-vendor }} + DEVELOPMENT: True + USE_TOR: False \ No newline at end of file diff --git a/.github/workflows/py-linter.yml b/.github/workflows/py-linter.yml index 7b5f785a1..84ffb110e 100644 --- a/.github/workflows/py-linter.yml +++ b/.github/workflows/py-linter.yml @@ -26,7 +26,7 @@ jobs: with: python-version: '3.11.6' cache: pip - - run: pip install black==22.8.0 flake8==5.0.4 isort==5.10.1 + - run: pip install requirements_dev.txt - name: Run linters uses: wearerequired/lint-action@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ecf6f312..5d2159cdd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,8 +38,8 @@ jobs: fi - django-test: - uses: RoboSats/robosats/.github/workflows/django-test.yml@main + integration-tests: + uses: RoboSats/robosats/.github/workflows/integration-tests.yml@main needs: check-versions frontend-build: diff --git a/api/lightning/cln.py b/api/lightning/cln.py index 928bf23fa..adb609e5a 100755 --- a/api/lightning/cln.py +++ b/api/lightning/cln.py @@ -69,6 +69,16 @@ def get_version(cls): print(f"Cannot get CLN version: {e}") return None + @classmethod + def get_info(cls): + try: + nodestub = node_pb2_grpc.NodeStub(cls.node_channel) + request = node_pb2.GetinfoRequest() + response = nodestub.Getinfo(request) + return response + except Exception as e: + print(f"Cannot get CLN node id: {e}") + @classmethod def decode_payreq(cls, invoice): """Decodes a lightning payment request (invoice)""" diff --git a/api/serializers.py b/api/serializers.py index 078374c9c..43419c239 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -28,8 +28,8 @@ class InfoSerializer(serializers.Serializer): lifetime_volume = serializers.FloatField( help_text="Total volume in BTC since exchange's inception" ) - lnd_version = serializers.CharField(required=False) - cln_version = serializers.CharField(required=False) + lnd_version = serializers.CharField() + cln_version = serializers.CharField() robosats_running_commit_hash = serializers.CharField() alternative_site = serializers.CharField() alternative_name = serializers.CharField() diff --git a/api/tests/test_utils.py b/api/tests/test_utils.py index 5280b4bff..55ba5bcac 100644 --- a/api/tests/test_utils.py +++ b/api/tests/test_utils.py @@ -21,8 +21,6 @@ verify_signed_message, weighted_median, ) -from tests.mocks.cln import MockNodeStub -from tests.mocks.lnd import MockVersionerStub class TestUtils(TestCase): @@ -96,15 +94,13 @@ def test_get_exchange_rates(self, mock_get_session, mock_config): mock_response_blockchain.json.assert_called_once() mock_response_yadio.json.assert_called_once() - @patch("api.lightning.lnd.verrpc_pb2_grpc.VersionerStub", MockVersionerStub) def test_get_lnd_version(self): version = get_lnd_version() - self.assertEqual(version, "v0.17.0-beta") + self.assertTrue(isinstance(version, str)) - @patch("api.lightning.cln.node_pb2_grpc.NodeStub", MockNodeStub) def test_get_cln_version(self): version = get_cln_version() - self.assertEqual(version, "v23.08") + self.assertTrue(isinstance(version, str)) @patch( "builtins.open", new_callable=mock_open, read_data="00000000000000000000 dev" diff --git a/api/utils.py b/api/utils.py index 6e24f586b..e9075b709 100644 --- a/api/utils.py +++ b/api/utils.py @@ -181,7 +181,7 @@ def get_lnd_version(): return LNDNode.get_version() except Exception: - return None + return "No LND" cln_version_cache = {} @@ -194,7 +194,7 @@ def get_cln_version(): return CLNNode.get_version() except Exception: - return None + return "No CLN" robosats_commit_cache = {} diff --git a/docker-tests.yml b/docker-tests.yml index 73c7caed4..20d7650f0 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -2,20 +2,29 @@ # docker-compose -f docker-tests.yml --env-file tests/compose.env up -d # Some useful handy commands that hopefully are never needed + # docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test createwallet default # docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test -generate 101 -# docker exec -it coordinator-lnd lncli --network=regtest getinfo -# docker exec -it robot-lnd lncli --network=regtest --rpcserver localhost:10010 getinfo +# docker exec -it coordinator-LND lncli --network=regtest getinfo +# docker exec -it robot-LND lncli --network=regtest --rpcserver localhost:10010 getinfo version: '3.9' services: bitcoind: - image: ruimarinho/bitcoin-core:${BITCOIND_TAG} + image: ruimarinho/bitcoin-core:${BITCOIND_VERSION:-24.0.1}-alpine container_name: btc restart: always ports: - "8000:8000" + - "8080:8080" + - "8081:8081" + - "10009:10009" + - "10010:10010" + - "9999:9999" + - "9998:9998" + - "5432:5432" + - "6379:6379" volumes: - bitcoin:/bitcoin/.bitcoin/ command: @@ -35,9 +44,9 @@ services: --zmqpubrawtx=tcp://0.0.0.0:28333 --listenonion=0 - coordinator-lnd: - image: lightninglabs/lnd:${LND_TAG} - container_name: coordinator-lnd + coordinator-LND: + image: lightninglabs/lnd:${LND_VERSION:-v0.17.0-beta} + container_name: coordinator-LND restart: always volumes: - bitcoin:/root/.bitcoin/ @@ -47,11 +56,11 @@ services: --noseedbackup --nobootstrap --restlisten=localhost:8081 - --no-rest-tls --debuglevel=debug --maxpendingchannels=10 --rpclisten=0.0.0.0:10009 --listen=0.0.0.0:9735 + --no-rest-tls --color=#4126a7 --alias=RoboSats --bitcoin.active @@ -67,9 +76,24 @@ services: - bitcoind network_mode: service:bitcoind - robot-lnd: - image: lightninglabs/lnd:${LND_TAG} - container_name: robot-lnd + coordinator-CLN: + image: elementsproject/lightningd:${CLN_VERSION:-v23.08.1} + restart: always + container_name: coordinator-CLN + environment: + LIGHTNINGD_NETWORK: 'regtest' + volumes: + - cln:/root/.lightning + - ./docker/cln/plugins/cln-grpc-hold:/root/.lightning/plugins/cln-grpc-hold + - bitcoin:/root/.bitcoin + command: --regtest --wumbo --bitcoin-rpcuser=test --bitcoin-rpcpassword=test --rest-host=0.0.0.0 --bind-addr=127.0.0.1:9737 --grpc-port=9999 --grpc-hold-port=9998 --important-plugin=/root/.lightning/plugins/cln-grpc-hold --database-upgrade=true + depends_on: + - bitcoind + network_mode: service:bitcoind + + robot-LND: + image: lightninglabs/lnd:${LND_VERSION:-v0.17.0-beta} + container_name: robot-LND restart: always volumes: - bitcoin:/root/.bitcoin/ @@ -78,6 +102,7 @@ services: command: --noseedbackup --nobootstrap + --restlisten=localhost:8080 --no-rest-tls --debuglevel=debug --maxpendingchannels=10 @@ -99,7 +124,7 @@ services: network_mode: service:bitcoind redis: - image: redis:${REDIS_TAG} + image: redis:${REDIS_VERSION:-7.2.1}-alpine container_name: redis restart: always volumes: @@ -116,11 +141,11 @@ services: TESTING: True USE_TOR: False MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon' + CLN_DIR: '/cln/regtest/' env_file: - ${ROBOSATS_ENVS_FILE} depends_on: - redis - - coordinator-lnd - postgres network_mode: service:bitcoind volumes: @@ -130,74 +155,15 @@ services: - cln:/cln postgres: - image: postgres:${POSTGRES_TAG:-14.2-alpine} + image: postgres:${POSTGRES_VERSION:-14.2}-alpine container_name: sql restart: always environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: 'example' + POSTGRES_USER: 'postgres' + POSTGRES_DB: 'postgres' network_mode: service:bitcoind - # clean-orders: - # image: robosats-image - # restart: always - # container_name: clord - # command: python3 manage.py clean_orders - # environment: - # SKIP_COLLECT_STATIC: "true" - # POSTGRES_HOST: 'postgres' - # env_file: - # - ${ROBOSATS_ENVS_FILE} - - # follow-invoices: - # image: robosats-image - # container_name: invo - # restart: always - # env_file: - # - ${ROBOSATS_ENVS_FILE} - # environment: - # SKIP_COLLECT_STATIC: "true" - # POSTGRES_HOST: 'postgres' - # command: python3 manage.py follow_invoices - - # telegram-watcher: - # image: robosats-image - # container_name: tg - # restart: always - # environment: - # SKIP_COLLECT_STATIC: "true" - # POSTGRES_HOST: 'postgres' - # env_file: - # - ${ROBOSATS_ENVS_FILE} - # command: python3 manage.py telegram_watcher - - # celery: - # image: robosats-image - # container_name: cele - # restart: always - # env_file: - # - ${ROBOSATS_ENVS_FILE} - # environment: - # SKIP_COLLECT_STATIC: "true" - # POSTGRES_HOST: 'postgres' - # command: celery -A robosats worker --loglevel=WARNING - # depends_on: - # - redis - - # celery-beat: - # image: robosats-image - # container_name: beat - # restart: always - # env_file: - # - ${ROBOSATS_ENVS_FILE} - # environment: - # SKIP_COLLECT_STATIC: "true" - # POSTGRES_HOST: 'postgres' - # command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler - # depends_on: - # - redis - volumes: redisdata: bitcoin: diff --git a/docker/cln/plugins/cln-grpc-hold b/docker/cln/plugins/cln-grpc-hold new file mode 100755 index 000000000..7a0d46532 Binary files /dev/null and b/docker/cln/plugins/cln-grpc-hold differ diff --git a/pyproject.toml b/pyproject.toml index 5f9674fb9..a10bbc802 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,5 @@ omit = [ # omit test and mocks from coverage reports "tests/*", "*mocks*", + "manage.py", ] \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 10c8b42d9..15af5c245 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,7 @@ +coverage==7.3.2 black==23.3.0 isort==5.12.0 flake8==6.1.0 pyflakes==3.1.0 -coverage==7.3.2 drf-openapi-tester==2.3.3 pre-commit==3.5.0 \ No newline at end of file diff --git a/tests/api_specs.yaml b/tests/api_specs.yaml index 012d50511..3599b2d43 100644 --- a/tests/api_specs.yaml +++ b/tests/api_specs.yaml @@ -1161,10 +1161,12 @@ components: - alternative_site - bond_size - book_liquidity + - cln_version - current_swap_fee_rate - last_day_nonkyc_btc_premium - last_day_volume - lifetime_volume + - lnd_version - maker_fee - network - node_alias diff --git a/tests/compose.env b/tests/compose.env index ad69231d6..8fffd7565 100644 --- a/tests/compose.env +++ b/tests/compose.env @@ -1,10 +1,7 @@ ROBOSATS_ENVS_FILE=".env-sample" -BITCOIND_TAG='24.0.1-alpine' -LND_TAG='v0.17.0-beta' -REDIS_TAG='7.2.1-alpine@sha256:7f5a0dfbf379db69dc78434091dce3220e251022e71dcdf36207928cbf9010de' -POSTGRES_TAG='14.2-alpine' - -POSTGRES_DB='postgres' -POSTGRES_USER='postgres' -POSTGRES_PASSWORD='example' \ No newline at end of file +BITCOIND_VERSION='24.0.1' +LND_VERSION='v0.17.0-beta' +CLN_VERSION='v23.08.1' +REDIS_VERSION='7.2.1' +POSTGRES_VERSION='14.2' \ No newline at end of file diff --git a/tests/node_utils.py b/tests/node_utils.py index 71d43fb28..c5db258b7 100644 --- a/tests/node_utils.py +++ b/tests/node_utils.py @@ -1,24 +1,35 @@ +import codecs +import sys +import time + import requests from requests.auth import HTTPBasicAuth from requests.exceptions import ReadTimeout +wait_step = 0.2 + def get_node(name="robot"): """ We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user) """ if name == "robot": - with open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f: - macaroon = f.read() - return {"port": 8080, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}} + macaroon = codecs.encode( + open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), + "hex", + ) + port = 8080 elif name == "coordinator": - with open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f: - macaroon = f.read() - return {"port": 8081, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}} + macaroon = codecs.encode( + open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), "hex" + ) + port = 8081 + + return {"port": port, "headers": {"Grpc-Metadata-macaroon": macaroon}} -def get_node_id(node_name): +def get_lnd_node_id(node_name): node = get_node(node_name) response = requests.get( f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"] @@ -27,13 +38,99 @@ def get_node_id(node_name): return data["identity_pubkey"] +def get_cln_node_id(): + from api.lightning.cln import CLNNode + + response = CLNNode.get_info() + return response.id.hex() + + +def wait_for_lnd_node_sync(node_name): + node = get_node(node_name) + waited = 0 + while True: + response = requests.get( + f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"] + ) + if response.json()["synced_to_chain"]: + return + else: + sys.stdout.write( + f"\rWaiting for {node_name} node chain sync {round(waited,1)}s" + ) + sys.stdout.flush() + waited += wait_step + time.sleep(wait_step) + + +def wait_for_lnd_active_channels(node_name): + node = get_node(node_name) + waited = 0 + while True: + response = requests.get( + f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"] + ) + if response.json()["num_active_channels"] > 0: + return + else: + sys.stdout.write( + f"\rWaiting for {node_name} node channels to be active {round(waited,1)}s" + ) + sys.stdout.flush() + waited += wait_step + time.sleep(wait_step) + + +def wait_for_cln_node_sync(): + from api.lightning.cln import CLNNode + + waited = 0 + while True: + response = CLNNode.get_info() + if response.warning_bitcoind_sync or response.warning_lightningd_sync: + sys.stdout.write( + f"\rWaiting for coordinator CLN node sync {round(waited,1)}s" + ) + sys.stdout.flush() + waited += wait_step + time.sleep(wait_step) + else: + return + + +def wait_for_cln_active_channels(): + from api.lightning.cln import CLNNode + + waited = 0 + while True: + response = CLNNode.get_info() + if response.num_active_channels > 0: + return + else: + sys.stdout.write( + f"\rWaiting for coordinator CLN node channels to be active {round(waited,1)}s" + ) + sys.stdout.flush() + waited += wait_step + time.sleep(wait_step) + + def connect_to_node(node_name, node_id, ip_port): node = get_node(node_name) data = {"addr": {"pubkey": node_id, "host": ip_port}} - response = requests.post( - f'http://localhost:{node["port"]}/v1/peers', json=data, headers=node["headers"] - ) - return response.json() + while True: + response = requests.post( + f'http://localhost:{node["port"]}/v1/peers', + json=data, + headers=node["headers"], + ) + if response.json() == {}: + return response.json() + else: + if "already connected to peer" in response.json()["message"]: + return response.json() + print(f"Could not connect to coordinator node: {response.json()}") + time.sleep(wait_step) def open_channel(node_name, node_id, local_funding_amount, push_sat): @@ -60,6 +157,7 @@ def create_address(node_name): def generate_blocks(address, num_blocks): + print(f"Mining {num_blocks} blocks") data = { "jsonrpc": "1.0", "id": "curltest", diff --git a/tests/test_coordinator_info.py b/tests/test_coordinator_info.py index 564d69329..4b7c37866 100644 --- a/tests/test_coordinator_info.py +++ b/tests/test_coordinator_info.py @@ -44,8 +44,8 @@ def test_info(self): self.assertEqual(data["last_day_nonkyc_btc_premium"], 0) self.assertEqual(data["last_day_volume"], 0) self.assertEqual(data["lifetime_volume"], 0) - self.assertEqual(data["lnd_version"], "v0.17.0-beta") - self.assertEqual(data["cln_version"], "v23.08") + self.assertTrue(isinstance(data["lnd_version"], str)) + self.assertTrue(isinstance(data["cln_version"], str)) self.assertEqual( data["robosats_running_commit_hash"], "00000000000000000000 dev" ) diff --git a/tests/test_trade_pipeline.py b/tests/test_trade_pipeline.py index d3e44cbe6..9e87cc1cb 100644 --- a/tests/test_trade_pipeline.py +++ b/tests/test_trade_pipeline.py @@ -14,12 +14,19 @@ connect_to_node, create_address, generate_blocks, - get_node_id, + get_cln_node_id, + get_lnd_node_id, open_channel, pay_invoice, + wait_for_cln_active_channels, + wait_for_cln_node_sync, + wait_for_lnd_active_channels, + wait_for_lnd_node_sync, ) from tests.test_api import BaseAPITestCase +LNVENDOR = config("LNVENDOR", cast=str, default="LND") + def read_file(file_path): """ @@ -49,6 +56,20 @@ class TradeTest(BaseAPITestCase): "longitude": 135.503, } + def wait_nodes_sync(): + wait_for_lnd_node_sync("robot") + if LNVENDOR == "LND": + wait_for_lnd_node_sync("coordinator") + elif LNVENDOR == "CLN": + wait_for_cln_node_sync() + + def wait_active_channels(): + wait_for_lnd_active_channels("robot") + if LNVENDOR == "LND": + wait_for_lnd_active_channels("coordinator") + elif LNVENDOR == "CLN": + wait_for_cln_active_channels() + @classmethod def setUpTestData(cls): """ @@ -61,19 +82,32 @@ def setUpTestData(cls): cache_market() # Fund two LN nodes in regtest and open channels - coordinator_node_id = get_node_id("coordinator") - connect_to_node("robot", coordinator_node_id, "localhost:9735") + # Coordinator is either LND or CLN. Robot user is always LND. + if LNVENDOR == "LND": + coordinator_node_id = get_lnd_node_id("coordinator") + coordinator_port = 9735 + elif LNVENDOR == "CLN": + coordinator_node_id = get_cln_node_id() + coordinator_port = 9737 + + print("Coordinator Node ID: ", coordinator_node_id) funding_address = create_address("robot") generate_blocks(funding_address, 101) + cls.wait_nodes_sync() - time.sleep( - 2 - ) # channels cannot be created until the node is fully sync. We just created 101 blocks. + # Open channel between Robot user and coordinator + print(f"\nOpening channel from Robot user node to coordinator {LNVENDOR} node") + connect_to_node("robot", coordinator_node_id, f"localhost:{coordinator_port}") open_channel("robot", coordinator_node_id, 100_000_000, 50_000_000) - # Generate 6 blocks so the channel becomes active - generate_blocks(funding_address, 6) + # Generate 10 blocks so the channel becomes active and wait for sync + generate_blocks(funding_address, 10) + + # Wait a tiny bit so payments can be done in the new channel + cls.wait_nodes_sync() + cls.wait_active_channels() + time.sleep(1) def test_login_superuser(self): """