Add CLN regtest to integration tests

.github/workflows/django-test.yml

@@ -15,68 +15,91 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 4
+      max-parallel: 1
       matrix:
-        python-version: ["3.11.6", "3.12"]
-        lnd-version: ["v0.17.0-beta","v0.17.1-beta.rc1"]
+        python-version: ["3.11.6"] #, "3.12"]
+        ln-vendor: ["LND", "CLN"]
+        lnd-version: ["v0.17.0-beta"] #,"v0.17.1-beta.rc1"]
+        cln-version: ["v23.08.1"]
+        bitcoind-version: ["24.0.1-alpine"]
 
     steps:
     - name: 'Checkout'
       uses: actions/checkout@v4
 
-    - name: 'Compose Eegtest Orchestration'
+    - name: 'Compose Regtest Orchestration'
       uses: isbang/[email protected]
       with:
-        compose-file: "docker-test.yml"
-        env: "tests/compose.env"
+        compose-file: "./docker-tests.yml"
+        down-flags: "--volumes"
+        services: |
+          bitcoind
+          postgres
+          redis
+          coordinator-lnd
+          coordinator-cln
+          robot-lnd
+      env:
+        LND_TAG: ${{ matrix.lnd-version }}
+        CLN_TAG: ${{ matrix.cln-version }}
+        BITCOIND_TAG: ${{ matrix.bitcoind-version }}
+        ROBOSATS_ENVS_FILE: ".env-sample"
 
-    # - name: 'Set up Python ${{ matrix.python-version }}'
-    #   uses: actions/setup-python@v4
-    #   with:
-    #     python-version: ${{ matrix.python-version }}
+    - name: 'Set up Python ${{ matrix.python-version }}'
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
 
     # - name: 'Cache pip dependencies'
     #   uses: actions/cache@v3
     #   with:
     #     path: ~/.cache/pip
-    #     key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+    #     key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-pip-dev-${{ hashFiles('**/requirements_dev.txt') }}
     #     restore-keys: |
     #       ${{ runner.os }}-pip-
 
-    # - name: 'Install Python Dependencies'
-    #   run: |
-    #     python -m pip install --upgrade pip
-    #     pip install -r requirements.txt
-    #     pip install -r requirements_dev.txt
-
-    # - name: 'Install LND/CLN gRPC Dependencies'
-    #   run: bash ./scripts/generate_grpc.sh
+    - name: 'Install Python Dependencies'
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements_dev.txt
 
-    # - name: 'Create .env File'
-    #   run: |
-    #     mv .env-sample .env
-    #     sed -i "s/USE_TOR=True/USE_TOR=False/" .env
+    - name: 'Install LND/CLN gRPC Dependencies'
+      run: bash ./scripts/generate_grpc.sh
 
-    # - name: 'Wait for PostgreSQL to become ready'
-    #   run: |
-    #     sudo apt-get install -y postgresql-client
-    #     until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done
+    - name: 'Create .env File'
+      run: mv .env-sample .env
 
-    - name: 'Run tests with coverage'
+    - name: 'Print CLN status'
       run: |
-        docker exec coordinator coverage run manage.py test
-        docker exec coordinator coverage report
+        docker logs --tail 200 coordinator-cln
+        sudo ls /var/lib/docker/volumes/robosats_cln/_data/regtest/
 
+    - name: 'Copy credentials'
+      run: |
+        mkdir -p ./node/cln/regtest
+        mkdir -p ./node/lnd/data/chain/bitcoin/regtest
+        sudo cp /var/lib/docker/volumes/robosats_lnd/_data/tls.cert ./node/lnd/tls.cert
+        sudo cp /var/lib/docker/volumes/robosats_lnd/_data/data/chain/bitcoin/regtest/admin.macaroon ./node/lnd/data/chain/bitcoin/regtest/admin.macaroon
+        sudo cp /var/lib/docker/volumes/robosats_cln/_data/regtest/client.pem ./node/cln/regtest/client.pem
+        sudo cp /var/lib/docker/volumes/robosats_cln/_data/regtest/client-key.pem ./node/cln/regtest/client-key.pem
+        sudo cp /var/lib/docker/volumes/robosats_cln/_data/regtest/server.pem ./node/cln/regtest/server.pem
+        sudo chmod -R ./node/cln/regtest*.pem
+        sudo chmod -R ./node/lnd/tls.cert
+        sudo chmod -R ./node/lnd/data/chain/bitcoin/regtest/admin.macaroon
 
-# jobs:
-#   test:
-#     runs-on: ubuntu-latest
-#     steps:
-#       - uses: actions/checkout@v2
-#       - name: Run Docker Compose
-#         run: |
-#           docker-compose up -d
-#           docker-compose run web python manage.py test
+    - name: 'Run tests with coverage'
+      run: |
+        coverage run manage.py test
+        coverage report
+      env:
+        LNVENDOR: ${{ matrix.ln-vendor }}
+        DEVELOPMENT: True
+        USE_TOR: False
+        MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon'
+        CLN_DIR: './node/cln/regtest/'
+        LND_DIR: './node/lnd/'

.github/workflows/py-linter.yml

@@ -26,7 +26,7 @@ jobs:
         with:
           python-version: '3.11.6'
           cache: pip
-      - run: pip install black==22.8.0 flake8==5.0.4 isort==5.10.1
+      - run: pip install requirements_dev.txt
       - name: Run linters
         uses: wearerequired/lint-action@v2
         with:

.github/workflows/release.yml

@@ -38,8 +38,8 @@ jobs:
         fi
 
 
-  django-test:
-    uses: RoboSats/robosats/.github/workflows/django-test.yml@main
+  integration-tests:
+    uses: RoboSats/robosats/.github/workflows/integration-tests.yml@main
     needs: check-versions
 
   frontend-build:

api/lightning/cln.py

@@ -69,6 +69,16 @@ def get_version(cls):
             print(f"Cannot get CLN version: {e}")
             return None
 
+    @classmethod
+    def get_info(cls):
+        try:
+            nodestub = node_pb2_grpc.NodeStub(cls.node_channel)
+            request = node_pb2.GetinfoRequest()
+            response = nodestub.Getinfo(request)
+            return response
+        except Exception as e:
+            print(f"Cannot get CLN node id: {e}")
+
     @classmethod
     def decode_payreq(cls, invoice):
         """Decodes a lightning payment request (invoice)"""

api/serializers.py

@@ -28,8 +28,8 @@ class InfoSerializer(serializers.Serializer):
     lifetime_volume = serializers.FloatField(
         help_text="Total volume in BTC since exchange's inception"
     )
-    lnd_version = serializers.CharField(required=False)
-    cln_version = serializers.CharField(required=False)
+    lnd_version = serializers.CharField()
+    cln_version = serializers.CharField()
     robosats_running_commit_hash = serializers.CharField()
     alternative_site = serializers.CharField()
     alternative_name = serializers.CharField()

api/tests/test_utils.py

@@ -21,8 +21,6 @@
     verify_signed_message,
     weighted_median,
 )
-from tests.mocks.cln import MockNodeStub
-from tests.mocks.lnd import MockVersionerStub
 
 
 class TestUtils(TestCase):
@@ -96,15 +94,13 @@ def test_get_exchange_rates(self, mock_get_session, mock_config):
         mock_response_blockchain.json.assert_called_once()
         mock_response_yadio.json.assert_called_once()
 
-    @patch("api.lightning.lnd.verrpc_pb2_grpc.VersionerStub", MockVersionerStub)
     def test_get_lnd_version(self):
         version = get_lnd_version()
-        self.assertEqual(version, "v0.17.0-beta")
+        self.assertTrue(isinstance(version, str))
 
-    @patch("api.lightning.cln.node_pb2_grpc.NodeStub", MockNodeStub)
     def test_get_cln_version(self):
         version = get_cln_version()
-        self.assertEqual(version, "v23.08")
+        self.assertTrue(isinstance(version, str))
 
     @patch(
         "builtins.open", new_callable=mock_open, read_data="00000000000000000000 dev"

api/utils.py

@@ -181,7 +181,7 @@ def get_lnd_version():
 
         return LNDNode.get_version()
     except Exception:
-        return None
+        return "No LND"
 
 
 cln_version_cache = {}
@@ -194,7 +194,7 @@ def get_cln_version():
 
         return CLNNode.get_version()
     except Exception:
-        return None
+        return "No CLN"
 
 
 robosats_commit_cache = {}

docker-tests.yml

@@ -2,6 +2,7 @@
 # docker-compose -f docker-tests.yml --env-file tests/compose.env up -d
 
 # Some useful handy commands that hopefully are never needed
+
 # docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test createwallet default
 # docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test -generate 101
 
@@ -11,11 +12,19 @@
 version: '3.9'
 services:
   bitcoind:
-    image: ruimarinho/bitcoin-core:${BITCOIND_TAG}
+    image: ruimarinho/bitcoin-core:${BITCOIND_TAG:-24.0.1-alpine}
     container_name: btc
     restart: always
     ports:
       - "8000:8000"
+      - "8080:8080"
+      - "8081:8081"
+      - "10009:10009"
+      - "10010:10010"
+      - "9999:9999"
+      - "9998:9998"
+      - "5432:5432"
+      - "6379:6379"
     volumes:
       - bitcoin:/bitcoin/.bitcoin/
     command:
@@ -36,7 +45,7 @@ services:
       --listenonion=0
 
   coordinator-lnd:
-    image: lightninglabs/lnd:${LND_TAG}
+    image: lightninglabs/lnd:${LND_TAG:-v0.17.0-beta}
     container_name: coordinator-lnd
     restart: always
     volumes:
@@ -67,8 +76,23 @@ services:
       - bitcoind
     network_mode: service:bitcoind
 
+  coordinator-cln:
+    image: elementsproject/lightningd:${CLN_TAG:-v23.08.1}
+    restart: always
+    container_name: coordinator-cln
+    environment:
+      LIGHTNINGD_NETWORK: 'regtest'
+    volumes:
+      - cln:/root/.lightning
+      - ./docker/cln/plugins/cln-grpc-hold:/root/.lightning/plugins/cln-grpc-hold
+      - bitcoin:/root/.bitcoin
+    command: --regtest --wumbo --bitcoin-rpcuser=test --bitcoin-rpcpassword=test --bind-addr=127.0.0.1:9737 --grpc-port=9999 --grpc-hold-port=9998 --important-plugin=/root/.lightning/plugins/cln-grpc-hold --database-upgrade=true
+    depends_on:
+      - bitcoind
+    network_mode: service:bitcoind
+
   robot-lnd:
-    image: lightninglabs/lnd:${LND_TAG}
+    image: lightninglabs/lnd:${LND_TAG:-v0.17.0-beta}
     container_name: robot-lnd
     restart: always
     volumes:
@@ -99,7 +123,7 @@ services:
     network_mode: service:bitcoind
 
   redis:
-    image: redis:${REDIS_TAG}
+    image: redis:${REDIS_TAG:-7.2.1-alpine}
     container_name: redis
     restart: always
     volumes:
@@ -116,6 +140,7 @@ services:
       TESTING: True
       USE_TOR: False
       MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon'
+      CLN_DIR: '/cln/regtest/'
     env_file:
       - ${ROBOSATS_ENVS_FILE}
     depends_on:
@@ -134,70 +159,11 @@ services:
     container_name: sql
     restart: always
     environment:
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_PASSWORD: 'example'
+      POSTGRES_USER: 'postgres'
+      POSTGRES_DB: 'postgres'
     network_mode: service:bitcoind
 
-  # clean-orders:
-  #   image: robosats-image
-  #   restart: always
-  #   container_name: clord
-  #   command: python3 manage.py clean_orders
-  #   environment:
-  #     SKIP_COLLECT_STATIC: "true"
-  #     POSTGRES_HOST: 'postgres'
-  #   env_file:
-  #     - ${ROBOSATS_ENVS_FILE}
-
-  # follow-invoices:
-  #   image: robosats-image
-  #   container_name: invo
-  #   restart: always
-  #   env_file:
-  #     - ${ROBOSATS_ENVS_FILE}
-  #   environment:
-  #     SKIP_COLLECT_STATIC: "true"
-  #     POSTGRES_HOST: 'postgres'
-  #   command: python3 manage.py follow_invoices
-
-  # telegram-watcher:
-  #   image: robosats-image
-  #   container_name: tg
-  #   restart: always
-  #   environment:
-  #     SKIP_COLLECT_STATIC: "true"
-  #     POSTGRES_HOST: 'postgres'
-  #   env_file:
-  #     - ${ROBOSATS_ENVS_FILE}
-  #   command: python3 manage.py telegram_watcher
-
-  # celery:
-  #   image: robosats-image
-  #   container_name: cele
-  #   restart: always
-  #   env_file:
-  #     - ${ROBOSATS_ENVS_FILE}
-  #   environment:
-  #     SKIP_COLLECT_STATIC: "true"
-  #     POSTGRES_HOST: 'postgres'
-  #   command: celery -A robosats worker --loglevel=WARNING
-  #   depends_on:
-  #     - redis
-
-  # celery-beat:
-  #   image: robosats-image
-  #   container_name: beat
-  #   restart: always
-  #   env_file:
-  #     - ${ROBOSATS_ENVS_FILE}
-  #   environment:
-  #     SKIP_COLLECT_STATIC: "true"
-  #     POSTGRES_HOST: 'postgres'
-  #   command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
-  #   depends_on:
-  #     - redis
-
 volumes:
   redisdata:
   bitcoin:

pyproject.toml

@@ -8,4 +8,5 @@ omit = [
     # omit test and mocks from coverage reports
     "tests/*",
     "*mocks*",
+    "manage.py",
     ]

requirements_dev.txt

@@ -1,7 +1,7 @@
+coverage==7.3.2
 black==23.3.0
 isort==5.12.0
 flake8==6.1.0
 pyflakes==3.1.0
-coverage==7.3.2
 drf-openapi-tester==2.3.3
 pre-commit==3.5.0