From 7b3c62e14f65e99edd6b568018ccae01ac5b8532 Mon Sep 17 00:00:00 2001 From: jerryfletcher21 Date: Fri, 5 Jul 2024 01:26:53 +0200 Subject: [PATCH 1/2] split host and port in .env variables --- .env-sample | 27 ++++++++++++++++----------- api/lightning/cln.py | 8 ++++++-- api/lightning/lnd.py | 4 +++- api/tests/test_utils.py | 2 +- api/utils.py | 4 +++- docker-compose.yml | 8 ++++++-- docker-tests.yml | 8 +++++--- robosats/celery/__init__.py | 6 +++++- robosats/settings.py | 11 +++++++++-- 9 files changed, 54 insertions(+), 24 deletions(-) diff --git a/.env-sample b/.env-sample index 668417382..3ef472a6a 100644 --- a/.env-sample +++ b/.env-sample @@ -11,14 +11,24 @@ MACAROON_PATH='data/chain/bitcoin/testnet/admin.macaroon' LND_CERT_BASE64='LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNLVENDQWRDZ0F3SUJBZ0lRQ0VoeGpPZXY1bGQyVFNPTXhKalFvekFLQmdncWhrak9QUVFEQWpBNE1SOHcKSFFZRFZRUUtFeFpzYm1RZ1lYVjBiMmRsYm1WeVlYUmxaQ0JqWlhKME1SVXdFd1lEVlFRREV3d3dNakJtTVRnMQpZelkwTnpVd0hoY05Nakl3TWpBNE1UWXhOalV3V2hjTk1qTXdOREExTVRZeE5qVXdXakE0TVI4d0hRWURWUVFLCkV4WnNibVFnWVhWMGIyZGxibVZ5WVhSbFpDQmpaWEowTVJVd0V3WURWUVFERXd3d01qQm1NVGcxWXpZME56VXcKV1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVNJVWdkcVMrWFZKL3EzY0JZeWd6ZDc2endaanlmdQpLK3BzcWNYVkFyeGZjU2NXQ25jbXliNGRaMy9Lc3lLWlRaamlySDE3aEY0OGtIMlp5clRZSW9hZG80RzdNSUc0Ck1BNEdBMVVkRHdFQi93UUVBd0lDcERBVEJnTlZIU1VFRERBS0JnZ3JCZ0VGQlFjREFUQVBCZ05WSFJNQkFmOEUKQlRBREFRSC9NQjBHQTFVZERnUVdCQlEwWUJjZXdsd1BqYTJPRXFyTGxzZnJscEswUFRCaEJnTlZIUkVFV2pCWQpnZ3d3TWpCbU1UZzFZelkwTnpXQ0NXeHZZMkZzYUc5emRJSUVkVzVwZUlJS2RXNXBlSEJoWTJ0bGRJSUhZblZtClkyOXVib2NFZndBQUFZY1FBQUFBQUFBQUFBQUFBQUFBQUFBQUFZY0V3S2dRQW9jRUFBQUFBREFLQmdncWhrak8KUFFRREFnTkhBREJFQWlBd0dMY05qNXVZSkVwanhYR05OUnNFSzAwWmlSUUh2Qm50NHp6M0htWHBiZ0lnSWtvUQo3cHFvNGdWNGhiczdrSmt1bnk2bkxlNVg0ZzgxYjJQOW52ZnZ2bkk9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K' # base64 ~/.lnd/data/chain/bitcoin/testnet/admin.macaroon | tr -d '\n' LND_MACAROON_BASE64='AgEDbG5kAvgBAwoQsyI+PK+fyb7F2UyTeZ4seRIBMBoWCgdhZGRyZXNzEgRyZWFkEgV3cml0ZRoTCgRpbmZvEgRyZWFkEgV3cml0ZRoXCghpbnZvaWNlcxIEcmVhZBIFd3JpdGUaIQoIbWFjYXJvb24SCGdlbmVyYXRlEgRyZWFkEgV3cml0ZRoWCgdtZXNzYWdlEgRyZWFkEgV3cml0ZRoXCghvZmZjaGFpbhIEcmVhZBIFd3JpdGUaFgoHb25jaGFpbhIEcmVhZBIFd3JpdGUaFAoFcGVlcnMSBHJlYWQSBXdyaXRlGhgKBnNpZ25lchIIZ2VuZXJhdGUSBHJlYWQAAAYgMt90uD6v4truTadWCjlppoeJ4hZrL1SBb09Y+4WOiI0=' +LND_GRPC_HOST='localhost' +LND_GRPC_PORT='10009' +# Auto unlock LND password. Only used in development docker-compose environment. +# It will fail starting up the node without it. +# To disable auto unlock, comment out 'wallet-unlock-password-file=/tmp/pwd' from 'docker/lnd/lnd.conf' +AUTO_UNLOCK_PWD='1234' # CLN directory CLN_DIR='/cln/testnet/' -CLN_GRPC_HOST='localhost:9999' -CLN_GRPC_HOLD_HOST='localhost:9998' +CLN_GRPC_HOST='localhost' +CLN_GRPC_PORT='9999' +CLN_GRPC_HOLD_HOST='localhost' +CLN_GRPC_HOLD_PORT='9998' # Bitcoin Core Daemon RPC, used to validate addresses -BITCOIND_RPCURL = 'http://127.0.0.1:18332' +# mainnet: 8332, testnet: 18332, signet: 38332, regtest: 18443 +BITCOIND_RPCHOST = '127.0.0.1' +BITCOIND_RPCPORT = '18332' BITCOIND_RPCUSER = 'robodev' BITCOIND_RPCPASSWORD = 'robodev' @@ -33,14 +43,9 @@ POSTGRES_PORT='5432' USE_TOR=True TOR_PROXY='127.0.0.1:9050' -# Auto unlock LND password. Only used in development docker-compose environment. -# It will fail starting up the node without it. -# To disable auto unlock, comment out 'wallet-unlock-password-file=/tmp/pwd' from 'docker/lnd/lnd.conf' -AUTO_UNLOCK_PWD='1234' - -LND_GRPC_HOST='localhost:10009' - -REDIS_URL='redis://localhost:6379/1' +REDIS_HOST="localhost" +REDIS_PORT="6379" +REDIS_DB_NUMBER="1" # List of market price public APIs. If the currency is available in more than 1 API, will use median price. MARKET_PRICE_APIS = https://blockchain.info/ticker, https://api.yadio.io/exrates/BTC, https://bitpay.com/rates/BTC, https://criptoya.com/api/btc diff --git a/api/lightning/cln.py b/api/lightning/cln.py index c446a83a9..260706d64 100755 --- a/api/lightning/cln.py +++ b/api/lightning/cln.py @@ -29,8 +29,12 @@ server_cert = f.read() -CLN_GRPC_HOST = config("CLN_GRPC_HOST", cast=str, default="localhost:9999") -CLN_GRPC_HOLD_HOST = config("CLN_GRPC_HOLD_HOST", cast=str, default="localhost:9998") +CLN_GRPC_HOST = \ + str(config("CLN_GRPC_HOST", cast=str, default="localhost")) + \ + ":" + str(config("CLN_GRPC_PORT", cast=str, default="9999")) +CLN_GRPC_HOLD_HOST = \ + str(config("CLN_GRPC_HOLD_HOST", cast=str, default="localhost")) + \ + ":" + str(config("CLN_GRPC_HOLD_PORT", cast=str, default="9998")) DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True) MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500000) diff --git a/api/lightning/lnd.py b/api/lightning/lnd.py index 706b4a6b1..90f8fc833 100644 --- a/api/lightning/lnd.py +++ b/api/lightning/lnd.py @@ -42,7 +42,9 @@ except Exception: MACAROON = b64decode(config("LND_MACAROON_BASE64")) -LND_GRPC_HOST = config("LND_GRPC_HOST") +LND_GRPC_HOST = \ + str(config("LND_GRPC_HOST", cast=str, default="localhost")) + \ + ":" + str(config("LND_GRPC_PORT", cast=str, default="10009")) DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True) MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500_000) diff --git a/api/tests/test_utils.py b/api/tests/test_utils.py index 0cf427ca2..8971d35e2 100644 --- a/api/tests/test_utils.py +++ b/api/tests/test_utils.py @@ -35,7 +35,7 @@ def test_get_session(self, mock_session, mock_config): @patch("api.utils.config") @patch("api.utils.requests.post") def test_bitcoind_rpc(self, mock_post, mock_config): - mock_config.side_effect = ["url", "user", "password"] + mock_config.side_effect = ["host", "port", "user", "password"] mock_post.return_value.json.return_value = {"result": "response"} response = bitcoind_rpc("method", ["params"]) self.assertEqual(response, "response") diff --git a/api/utils.py b/api/utils.py index 66658db2b..06cbdd989 100644 --- a/api/utils.py +++ b/api/utils.py @@ -37,7 +37,9 @@ def bitcoind_rpc(method, params=None): :return: """ - BITCOIND_RPCURL = config("BITCOIND_RPCURL") + BITCOIND_RPCHOST = str(config("BITCOIND_RPCHOST", cast=str, default="127.0.0.1")) + BITCOIND_RPCPORT = str(config("BITCOIND_RPCPORT", cast=str, default="18332")) + BITCOIND_RPCURL = "http://" + BITCOIND_RPCHOST + ":" + BITCOIND_RPCPORT BITCOIND_RPCUSER = config("BITCOIND_RPCUSER") BITCOIND_RPCPASSWORD = config("BITCOIND_RPCPASSWORD") diff --git a/docker-compose.yml b/docker-compose.yml index c8334bdb8..e21ed7f33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,7 +115,9 @@ services: container_name: cele-worker-dev restart: always environment: - REDIS_URL: redis://localhost:6379 + REDIS_HOST: localhost + REDIS_PORT: 6379 + REDIS_DB_NUMBER: 1 SKIP_COLLECT_STATIC: "true" volumes: - .:/usr/src/robosats @@ -132,7 +134,9 @@ services: container_name: cele-beat-dev restart: always environment: - REDIS_URL: redis://localhost:6379 + REDIS_HOST: localhost + REDIS_PORT: 6379 + REDIS_DB_NUMBER: 1 SKIP_COLLECT_STATIC: "true" command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler volumes: diff --git a/docker-tests.yml b/docker-tests.yml index 975747e88..432d791a8 100644 --- a/docker-tests.yml +++ b/docker-tests.yml @@ -149,7 +149,8 @@ services: USE_TOR: False MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon' CLN_DIR: '/cln/regtest/' - BITCOIND_RPCURL: 'http://127.0.0.1:18443' + BITCOIND_RPCHOST: '127.0.0.1' + BITCOIND_RPCPORT: '18443' BITCOIND_RPCUSER: 'test' BITCOIND_RPCPASSWORD: 'test' env_file: @@ -190,7 +191,8 @@ services: # USE_TOR: False # MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon' # CLN_DIR: '/cln/regtest/' - # BITCOIND_RPCURL: 'http://127.0.0.1:18443' + # BITCOIND_RPCHOST: '127.0.0.1' + # BITCOIND_RPCPORT: '18443' # BITCOIND_RPCUSER: 'test' # BITCOIND_RPCPASSWORD: 'test' # SKIP_COLLECT_STATIC: "true" @@ -210,4 +212,4 @@ volumes: bitcoin: lnd: cln: - lndrobot: \ No newline at end of file + lndrobot: diff --git a/robosats/celery/__init__.py b/robosats/celery/__init__.py index c1a27eb60..313796660 100644 --- a/robosats/celery/__init__.py +++ b/robosats/celery/__init__.py @@ -3,11 +3,15 @@ import os from datetime import timedelta +from decouple import config from celery import Celery from celery.schedules import crontab # You can use rabbitmq instead here. -BASE_REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") +BASE_REDIS_URL = \ + "redis://" + str(config("REDIS_HOST", cast=str, default="localhost")) + \ + ":" + str(config("REDIS_PORT", cast=str, default="6379")) + \ + "/" + str(config("REDIS_DB_NUMBER", cast=str, default="1")) # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "robosats.settings") diff --git a/robosats/settings.py b/robosats/settings.py index 555061f06..4a762300f 100644 --- a/robosats/settings.py +++ b/robosats/settings.py @@ -231,7 +231,11 @@ "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": [config("REDIS_URL")], + "hosts": [ + "redis://" + str(config("REDIS_HOST", cast=str, default="localhost")) + \ + ":" + str(config("REDIS_PORT", cast=str, default="6379")) + \ + "/" + str(config("REDIS_DB_NUMBER", cast=str, default="1")) + ], }, }, } @@ -239,7 +243,10 @@ CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": config("REDIS_URL"), + "LOCATION": \ + "redis://" + str(config("REDIS_HOST", cast=str, default="localhost")) + \ + ":" + str(config("REDIS_PORT", cast=str, default="6379")) + \ + "/" + str(config("REDIS_DB_NUMBER", cast=str, default="1")), "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, } } From 76ad7e308e6f1cb3ea7b9c96117a7d6803ac5be4 Mon Sep 17 00:00:00 2001 From: jerryfletcher21 Date: Fri, 5 Jul 2024 01:29:13 +0200 Subject: [PATCH 2/2] Traditional environment for backend development --- .env-sample | 3 + .gitignore | 3 + api/utils.py | 4 +- api/views.py | 4 + requirements_dev.txt | 3 +- robosats/settings.py | 6 +- scripts/traditional/README.md | 222 +++ scripts/traditional/regtest-nodes | 1407 ++++++++++++++++++ scripts/traditional/regtest-services | 372 +++++ scripts/traditional/robosats.bash-completion | 281 ++++ setup.md | 6 + tests/utils/node.py | 25 +- tests/utils/pgp.py | 3 +- 13 files changed, 2324 insertions(+), 15 deletions(-) create mode 100644 scripts/traditional/README.md create mode 100755 scripts/traditional/regtest-nodes create mode 100755 scripts/traditional/regtest-services create mode 100755 scripts/traditional/robosats.bash-completion diff --git a/.env-sample b/.env-sample index 3ef472a6a..e69640f3b 100644 --- a/.env-sample +++ b/.env-sample @@ -47,6 +47,9 @@ REDIS_HOST="localhost" REDIS_PORT="6379" REDIS_DB_NUMBER="1" +LOG_TO_CONSOLE=False +LOGGER_LEVEL="WARNING" + # List of market price public APIs. If the currency is available in more than 1 API, will use median price. MARKET_PRICE_APIS = https://blockchain.info/ticker, https://api.yadio.io/exrates/BTC, https://bitpay.com/rates/BTC, https://criptoya.com/api/btc diff --git a/.gitignore b/.gitignore index 40a4ba981..d0f983aac 100755 --- a/.gitignore +++ b/.gitignore @@ -653,3 +653,6 @@ mobile/html/Web.bundle/assets* # Protocol Buffers api/lightning/*.proto + +# Traditional environment +regtest/ diff --git a/api/utils.py b/api/utils.py index 06cbdd989..c8b4a954c 100644 --- a/api/utils.py +++ b/api/utils.py @@ -387,7 +387,7 @@ def compute_avg_premium(queryset): def validate_pgp_keys(pub_key, enc_priv_key): """Validates PGP valid keys. Formats them in a way understandable by the frontend""" - gpg = gnupg.GPG() + gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None)) # Standardize format with linux linebreaks '\n'. Windows users submitting their own keys have '\r\n' breaking communication. enc_priv_key = enc_priv_key.replace("\r\n", "\n").replace("\\", "\n") @@ -441,7 +441,7 @@ def verify_signed_message(pub_key, signed_message): Verifies a signed cleartext PGP message. Returns whether the signature is valid (was made by the given pub_key) and the content of the message. """ - gpg = gnupg.GPG() + gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None)) # import the public key import_result = gpg.import_keys(pub_key) diff --git a/api/views.py b/api/views.py index a03dbd33d..3b30516db 100644 --- a/api/views.py +++ b/api/views.py @@ -16,6 +16,7 @@ from rest_framework.views import APIView from api.logics import Logics +from api.tasks import cache_market from api.models import ( Currency, LNPayment, @@ -146,6 +147,9 @@ def post(self, request): status.HTTP_400_BAD_REQUEST, ) + if len(Currency.objects.all()) == 0: + cache_market() + # Creates a new order order = Order( type=type, diff --git a/requirements_dev.txt b/requirements_dev.txt index e4c5aabc2..afb96ea71 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,5 @@ coverage==7.5.0 ruff==0.5.1 git+https://github.com/Reckless-Satoshi/drf-openapi-tester.git@soften-django-requirements -pre-commit==3.7.0 \ No newline at end of file +pre-commit==3.7.0 +python-dotenv[cli]==1.0.1 diff --git a/robosats/settings.py b/robosats/settings.py index 4a762300f..db5d75f42 100644 --- a/robosats/settings.py +++ b/robosats/settings.py @@ -59,7 +59,7 @@ SESSION_COOKIE_HTTPONLY = False # Logging settings -if os.environ.get("LOG_TO_CONSOLE"): +if config("LOG_TO_CONSOLE", cast=bool, default=False): LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -70,12 +70,12 @@ }, "root": { "handlers": ["console"], - "level": "WARNING", + "level": str(config("LOGGER_LEVEL", cast=str, default="WARNING")), }, "loggers": { "api.utils": { "handlers": ["console"], - "level": "WARNING", + "level": str(config("LOGGER_LEVEL", cast=str, default="WARNING")), }, }, } diff --git a/scripts/traditional/README.md b/scripts/traditional/README.md new file mode 100644 index 000000000..b60c08b90 --- /dev/null +++ b/scripts/traditional/README.md @@ -0,0 +1,222 @@ +# Robosats traditional environment + +Robosats backend development and testing without docker and containers. + +Binaries needed: +* postgresql (`postgres`, `initdb`, `psql`) +* redis (`redis-server`) +* bitcoin (`bitcoind`, `bitcoin-cli`) +* cln (`lightningd`, `lightning-cli`, `holdinvoice`) +* lnd (`lnd`, `lncli`) + +## Preparation + +Postgresql and redis can be found in all linux distros and bsds. + +Some distros do not put postgresql binaries in `PATH`, if this is the +case simply link them somewhere in your `PATH`. + +Example on debian: +``` +ln -sf /usr/lib/postgresql/16/bin/postgres ~/.local/bin/ +ln -sf /usr/lib/postgresql/16/bin/initdb ~/.local/bin/ +ln -sf /usr/lib/postgresql/16/bin/psql ~/.local/bin/ +``` + +Bitcoin nodes if not already installed need to be manually downloaded. +* bitcoin core binaries can be found here: https://bitcoincore.org/en/download +* cln binaries can be found here: https://github.com/ElementsProject/lightning/releases +* holdinvoice binary can be found here: https://github.com/daywalker90/holdinvoice/releases +* lnd binaries can be found here: https://github.com/lightningnetwork/lnd/releases + +Example preparation: +``` +$ python3 -m venv venv +$ . venv/bin/activate +$ pip install -r requirements_dev.txt +$ pip install -r requirements.txt + +$ mkdir regtest +$ mkdir regtest/programs +$ cd regtest/programs +$ mkdir bitcoin cln lnd +# download bitcoin, cln (and holdinvoice) and lnd binaries + +# if you want to use roboauto +# still in regtest/programs +$ git clone https://github.com/jerryfletcher21/roboauto +$ cd roboauto +$ pip install -r requirements.txt +$ pip install . +``` + +## env file + +``` +$ cp .env-sample .env +``` +Edit `.env`, both robosats and regtest scripts will read from there. + +Variables to change: +``` +LNVENDOR = "CLN" +# LNVENDOR = "LND" + +LND_DIR = "regtest/nodes/lnd-coord/" +MACAROON_PATH = "data/chain/bitcoin/regtest/admin.macaroon" +CLN_DIR = "regtest/nodes/cln-coord/regtest/" +BITCOIND_RPCPORT = "18443" +POSTGRES_DB = "robosats" +POSTGRES_USER = "robosats" +POSTGRES_PASSWORD = "robosats" +USE_TOR = False +LOG_TO_CONSOLE = True +LOGGER_LEVEL = "INFO" +``` + +Variables to add: +``` +DEVELOPMENT = True +TESTING = True + +LNVENDOR_ROBOT = "LND" +# LNVENDOR_ROBOT = "CLN" + +BITCOIND_BIN = "regtest/programs/bitcoin/bitcoin-27.1/bin/bitcoind" +BITCOIN_CLI_BIN = "regtest/programs/bitcoin/bitcoin-27.1/bin/bitcoin-cli" +LIGHTNINGD_BIN = "regtest/programs/cln/usr/bin/lightningd" +LIGHTNING_CLI_BIN = "regtest/programs/cln/usr/bin/lightning-cli" +HOLDINVOICE_PLUGIN_BIN = "regtest/programs/cln/holdinvoice" +LND_BIN = "regtest/programs/lnd/lnd-linux-amd64-v0.18.1-beta/lnd" +LNCLI_BIN = "regtest/programs/lnd/lnd-linux-amd64-v0.18.1-beta/lncli" +ROBOAUTO_GIT_DIR = "regtest/programs/roboauto" + +REGTEST_NODES_DIR = "regtest/nodes" +REGTEST_SERVICES_DIR = "regtest/services" +REGTEST_LOGS_DIR = "regtest/logs" +GNUPG_DIR = "regtest/services/gnupg" + +BITCOIND_TEST_RPCUSER = "robodev" +BITCOIND_TEST_RPCPASSWORD = "robodev" +CLN_TEST_COORD_LISTEN_PORT = 9785 +LND_TEST_COORD_LISTEN_PORT = 9885 +LND_TEST_COORD_MACAROON_PATH = "regtest/nodes/lnd-coord/data/chain/bitcoin/regtest/admin.macaroon" +LND_TEST_ROBOT_MACAROON_PATH = "regtest/nodes/lnd-user/data/chain/bitcoin/regtest/admin.macaroon" +LND_TEST_ROBOT_REST_PORT = 8180 +LND_TEST_COORD_REST_PORT = 8181 +``` + +Paths can be relative or absolute. Binaries should be paths, they are +not resolved with `PATH`. + +Roboauto can be disabled by not setting `ROBOAUTO_GIT_DIR` or setting it +to `false`. + +If some ports are already in use, change their value. + +To check which port are already in use, `netstat -tulnp` with root +privileges can be used. + +For example if there is alread an instance of postgresql running on the +default port, change `POSTGRES_PORT = "5433"`. + +## Usage + +For development and testing two scripts are provided: +* `regtest-services` for non bitcoin related services +* `regtest-nodes` for bitcoin and lightning nodes + +`regtest-services` sets up the database and manages the services. + +Everything is done locally, so no root privileges/service managers are +needed. + +`regtest-nodes` is a script that should be sourced, it sets up a regtest +environment, with bitcoin core, cln, lnd and roboauto, connecting them +and creating channels. It then exposes the functions `btc_reg`, +`cln_coord`, `cln_user`, `lnd_coord`, `lnd_user`, `ra_reg` to interact +with the nodes and roboauto. + +If the script is sourced in a `bash` shell, it will also source +completions for all the functions. + +`regtest-nodes` can also be run without arguments to simply expose the +functions to start and remove the nodes and to create the channels +between them, without setting up a specific environment. + +Setup: +``` +# after having changed .env file + +$ . venv/bin/activate + +# generate cln and lnd grpc +$ scripts/generate_grpc.sh + +$ scripts/traditional/regtest-services postgres-setup + +$ scripts/traditional/regtest-services postgres-database + +# postgres-database will create the database specified by +# POSTGRES_DB in .env, it can be run multiple times with +# different values of POSTGRES_DB for different databases +``` + +Testing: +``` +# edit .env setting LNVENDOR to either "CLN" or "LND" +# LNVENDOR_ROBOT while running tests should be set to "LND" + +# in the main window +$ . venv/bin/activate +$ . scripts/traditional/regtest-nodes test + +# in a secondary window +$ . venv/bin/activate +# can be stopped with Control-C +$ scripts/traditional/regtest-services test + +# back in the main window +$ python3 manage.py test + +# after having run the tests run +$ robosats_regtest_stop_and_remove_all +# to remove the nodes, they will be recreated when +# running the tests again + +# the tests should be run with a clean database so if you have already run +# the server and want to keep the database, either use a different value +# of POSTGRES_DB or use a different directory by moving +# regtest/services/postgres somewhere and the moving it back when you want +# to run the server again +``` + +Development: +``` +# edit .env setting LNVENDOR to either "CLN" or "LND" +# and LNVENDOR_ROBOT to either "CLN" or "LND" + +# in the main window +$ . venv/bin/activate +$ . scripts/traditional/regtest-nodes server + +# in a secondary window +$ . venv/bin/activate +# can be stopped with Control-C +$ scripts/traditional/regtest-services server + +# to see the output of the django runserver command +# in a third window +$ tail -f regtest/logs/runserver + +# if roboauto is active, use it to test the backend +# back in the main window +$ ra_reg --help +... +$ ra_reg create-order "$(ra_reg generate-robot --loc)" type=buy currency=btc min_amount=0.01 max_amount=0.02 payment_method="On-Chain BTC" premium=-1.5 +... +$ ra_reg take-order $(ra_reg generate-robot --loc) order-id +... +$ ra_reg escrow-pay RobotName +... +``` diff --git a/scripts/traditional/regtest-nodes b/scripts/traditional/regtest-nodes new file mode 100755 index 000000000..746b0ef16 --- /dev/null +++ b/scripts/traditional/regtest-nodes @@ -0,0 +1,1407 @@ +#!/bin/sh + +_command_exist() { + if command -v "$1" >/dev/null 2>&1; then + return 0 + else + echo "error: $1 command not found" >&2 + return 1 + fi +} + +_function_exist() { + if command -V "$1" 2>/dev/null | grep -qwi function >/dev/null 2>&1; then + return 0 + else + echo "error: function $1 not set" >&2 + return 1 + fi +} + +_create_dir() { + if [ ! -e "$1" ]; then + mkdir -p "$1" || return "$?" + elif [ ! -d "$1" ]; then + echo "error: $1 is not a directory" >&2 + return 1 + fi +} + +# if $2 is provided use it as default +# otherwise return error when not found +_get_env_var() { + if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then + if [ "$#" -ge 2 ]; then + env_var="$2" + else + echo "error: getting $1 from .env" >&2 + return 1 + fi + fi + printf "%s\n" "$env_var" + + return 0 +} + +# transform relative path into absolute and remove trailing slashes +_get_env_var_path() { + env_var="$(_get_env_var "$@")" || return "$?" + real_path="$(realpath -m "$env_var")" || return "$?" + printf "%s\n" "$real_path" + + return 0 +} + +_nodes_environment_set() { + if [ ! -f "scripts/traditional/regtest-nodes" ]; then + echo "error: source this script from the project root" >&2 + return 1 + fi + if [ ! -f ".env" ]; then + echo "error: .env is not present" >&2 + return 1 + fi + + BITCOIND_BIN="$(_get_env_var_path "BITCOIND_BIN")" || return "$?" + BITCOIN_CLI_BIN="$(_get_env_var_path "BITCOIN_CLI_BIN")" || return "$?" + LIGHTNINGD_BIN="$(_get_env_var_path "LIGHTNINGD_BIN")" || return "$?" + LIGHTNING_CLI_BIN="$(_get_env_var_path "LIGHTNING_CLI_BIN")" || return "$?" + HOLDINVOICE_PLUGIN_BIN="$(_get_env_var_path "HOLDINVOICE_PLUGIN_BIN")" || return "$?" + LND_BIN="$(_get_env_var_path "LND_BIN")" || return "$?" + LNCLI_BIN="$(_get_env_var_path "LNCLI_BIN")" || return "$?" + REGTEST_NODES_DIR="$(_get_env_var_path "REGTEST_NODES_DIR")" || return "$?" + REGTEST_LOGS_DIR="$(_get_env_var_path "REGTEST_LOGS_DIR")" || return "$?" + + ROBOAUTO_GIT_DIR="$(_get_env_var_path "ROBOAUTO_GIT_DIR" false)" || return "$?" + + BITCOIN_REGTEST_RPC_PORT="$(_get_env_var "BITCOIND_RPCPORT")" || return "$?" + BITCOIN_REGTEST_RPC_USER="$(_get_env_var "BITCOIND_RPCUSER")" || return "$?" + BITCOIN_REGTEST_RPC_PASS="$(_get_env_var "BITCOIND_RPCPASSWORD")" || return "$?" + CLN_COORD_GRPC_PORT="$(_get_env_var "CLN_GRPC_PORT")" || return "$?" + CLN_COORD_HOLD_PORT="$(_get_env_var "CLN_GRPC_HOLD_PORT")" || return "$?" + LND_COORD_RPC_PORT="$(_get_env_var "LND_GRPC_PORT")" || return "$?" + CLN_COORD_LISTEN_PORT="$(_get_env_var "CLN_TEST_COORD_LISTEN_PORT")" || return "$?" + LND_COORD_LISTEN_PORT="$(_get_env_var "LND_TEST_COORD_LISTEN_PORT")" || return "$?" + LND_COORD_REST_PORT="$(_get_env_var "LND_TEST_COORD_REST_PORT")" || return "$?" + LND_USER_REST_PORT="$(_get_env_var "LND_TEST_ROBOT_REST_PORT")" || return "$?" + + bitcoind_test_rpcuser="$(_get_env_var "BITCOIND_TEST_RPCUSER")" || return "$?" + if [ "$bitcoind_test_rpcuser" != "$BITCOIN_REGTEST_RPC_USER" ]; then + printf "%s%s\n" \ + "error: set BITCOIND_RPCUSER and " \ + "BITCOIND_TEST_RPCUSER to the same value" \ + >&2 + return 1 + fi + bitcoind_test_rpcpass="$(_get_env_var "BITCOIND_TEST_RPCPASSWORD")" || return "$?" + if [ "$bitcoind_test_rpcpass" != "$BITCOIN_REGTEST_RPC_PASS" ]; then + printf "%s%s\n" \ + "error: set BITCOIND_RPCPASSWORD and " \ + "BITCOIND_TEST_RPCPASSWORD to the same value" \ + >&2 + return 1 + fi + + LNVENDOR_COORD="$(_get_env_var LNVENDOR)" || return "$?" + LNVENDOR_COORD="$(printf "%s\n" "$LNVENDOR_COORD" | tr '[:upper:]' '[:lower:]')" + case "$LNVENDOR_COORD" in + cln|lnd) ;; + *) + echo "error: LNVENDOR can be cln or lnd" >&2 + return 1 + ;; + esac + LNVENDOR_USER="$(_get_env_var LNVENDOR_ROBOT)" || return "$?" + LNVENDOR_USER="$(printf "%s\n" "$LNVENDOR_USER" | tr '[:upper:]' '[:lower:]')" + case "$LNVENDOR_USER" in + cln|lnd) ;; + *) + echo "error: LNVENDOR_ROBOT can be cln or lnd" >&2 + return 1 + ;; + esac + + if [ -z "$BITCOIND_BIN" ]; then + BITCOIND_BIN="bitcoind" + fi + if ! _command_exist "$BITCOIND_BIN"; then + return 1 + fi + + if [ -z "$BITCOIN_CLI_BIN" ]; then + BITCOIN_CLI_BIN="bitcoin-cli" + fi + if ! _command_exist "$BITCOIN_CLI_BIN"; then + return 1 + fi + + if [ -z "$LIGHTNINGD_BIN" ]; then + LIGHTNINGD_BIN="lightningd" + fi + if ! _command_exist "$LIGHTNINGD_BIN"; then + return 1 + fi + + if [ -z "$LIGHTNING_CLI_BIN" ]; then + LIGHTNING_CLI_BIN="lightning-cli" + fi + if ! _command_exist "$LIGHTNING_CLI_BIN"; then + return 1 + fi + + if [ -z "$HOLDINVOICE_PLUGIN_BIN" ]; then + echo "error: $HOLDINVOICE_PLUGIN_BIN not set" >&2 + return 1 + fi + if [ ! -f "$HOLDINVOICE_PLUGIN_BIN" ]; then + echo "error: $HOLDINVOICE_PLUGIN_BIN plugin not found" >&2 + return 1 + fi + + if [ -z "$LND_BIN" ]; then + LND_BIN="lnd" + fi + if ! _command_exist "$LND_BIN"; then + return 1 + fi + + if [ -z "$LNCLI_BIN" ]; then + LNCLI_BIN="lncli" + fi + if ! _command_exist "$LNCLI_BIN"; then + return 1 + fi + + if [ -z "$REGTEST_NODES_DIR" ]; then + echo "error: REGTEST_NODES_DIR not set" >&2 + return 1 + fi + _create_dir "$REGTEST_NODES_DIR" || return "$?" + + if [ -z "$REGTEST_LOGS_DIR" ]; then + echo "error: REGTEST_LOGS_DIR not set" >&2 + return 1 + fi + _create_dir "$REGTEST_LOGS_DIR" || return "$?" + + BITCOIN_DIR="$REGTEST_NODES_DIR/bitcoin" + CLN_COORD_DIR="$REGTEST_NODES_DIR/cln-coord" + CLN_USER_DIR="$REGTEST_NODES_DIR/cln-user" + LND_COORD_DIR="$REGTEST_NODES_DIR/lnd-coord" + LND_USER_DIR="$REGTEST_NODES_DIR/lnd-user" + ROBOAUTO_DIR="$REGTEST_NODES_DIR/roboauto" + + cln_coor_dir_env="$(_get_env_var_path "CLN_DIR")" || return "$?" + if [ "$cln_coor_dir_env" != "$CLN_COORD_DIR/regtest" ]; then + echo "error: CLN_DIR not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + lnd_coord_dir_env="$(_get_env_var_path "LND_DIR")" || return "$?" + if [ "$lnd_coord_dir_env" != "$LND_COORD_DIR" ]; then + echo "error: LND_DIR not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + + lnd_test_coord_macaroon_path="$( + _get_env_var_path "LND_TEST_COORD_MACAROON_PATH" + )" || return "$?" + if [ \ + "$lnd_test_coord_macaroon_path" != \ + "$LND_COORD_DIR/data/chain/bitcoin/regtest/admin.macaroon" \ + ]; then + echo "error: LND_TEST_COORD_MACAROON_PATH not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + lnd_test_user_macaroon_path="$( + _get_env_var_path "LND_TEST_ROBOT_MACAROON_PATH" + )" || return "$?" + if [ \ + "$lnd_test_user_macaroon_path" != \ + "$LND_USER_DIR/data/chain/bitcoin/regtest/admin.macaroon" \ + ]; then + echo "error: LND_TEST_ROBOT_MACAROON_PATH not consistent with REGTEST_NODES_DIR" >&2 + return 1 + fi + + # defaults + + BITCOIN_REGTEST_RPC_PORT="${BITCOIN_REGTEST_RPC_PORT:-18443}" + BITCOIN_REGTEST_RPC_USER="${BITCOIN_REGTEST_RPC_USER:-robodev}" + BITCOIN_REGTEST_RPC_PASS="${BITCOIN_REGTEST_RPC_PASS:-robodev}" + BITCOIN_REGTEST_ZMQ_BLOCK_PORT="${BITCOIN_REGTEST_ZMQ_BLOCK_PORT:-28432}" + BITCOIN_REGTEST_ZMQ_TX_PORT="${BITCOIN_REGTEST_ZMQ_TX_PORT:-28433}" + + CLN_COORD_LISTEN_PORT="${CLN_COORD_LISTEN_PORT:-9835}" + CLN_COORD_GRPC_PORT="${CLN_COORD_GRPC_PORT:-9999}" + CLN_COORD_HOLD_PORT="${CLN_COORD_HOLD_PORT:-9998}" + + CLN_USER_LISTEN_PORT="${CLN_USER_LISTEN_PORT:-9935}" + + LND_COORD_LISTEN_PORT="${LND_COORD_LISTEN_PORT:-9885}" + LND_COORD_RPC_PORT="${LND_COORD_RPC_PORT:-10009}" + LND_COORD_REST_PORT="${LND_COORD_REST_PORT:-8081}" + + LND_USER_LISTEN_PORT="${LND_USER_LISTEN_PORT:-9895}" + LND_USER_RPC_PORT="${LND_USER_RPC_PORT:-10010}" + LND_USER_REST_PORT="${LND_USER_REST_PORT:-8080}" +} + +_pgrep_bitcoin_regtest() { + pgrep -f "$BITCOIND_BIN -datadir=$BITCOIN_DIR -regtest -daemon" >/dev/null +} + +_bitcoin_regtest_check_started() { + if ! _pgrep_bitcoin_regtest; then + echo "error: bitcoin regtest not started" >&2 + return 1 + fi + if ! _function_exist btc_reg; then + return 1 + fi + return 0 +} + +bitcoin_regtest_mine() { + _bitcoin_regtest_check_started || return "$?" + + if [ "$#" -lt 2 ]; then + return 1 + fi + if [ "$2" = "new" ]; then + if ! new_address="$(btc_reg getnewaddress)"; then + echo "error: could not create new address" >&2 + return 1 + fi + else + new_address="$2" + fi + if ! new_block_hash="$( + btc_reg -named generatetoaddress nblocks="$1" address="$new_address" | + jq -r '.[-1]' + )"; then + echo "error: could not generate to $new_address" >&2 + return 1 + fi + while [ "$(btc_reg getbestblockhash)" != "$new_block_hash" ]; do + echo "waiting for $1 blocks to be mined..." + sleep 1 + done + unset new_block_hash + echo "mined $1 blocks to $new_address" + unset new_address +} + +bitcoin_regtest_start() { + _create_dir "$BITCOIN_DIR" || return "$?" + + echo "writing bitcoin regtest config" + cat << EOF > "$BITCOIN_DIR/bitcoin.conf" +txindex=1 +rest=1 +server=1 +logips=1 +listenonion=0 +zmqpubrawblock=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_BLOCK_PORT +zmqpubrawtx=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_TX_PORT +[regtest] +rpcport=$BITCOIN_REGTEST_RPC_PORT +rpcuser=$BITCOIN_REGTEST_RPC_USER +rpcpassword=$BITCOIN_REGTEST_RPC_PASS +EOF + + if ! _pgrep_bitcoin_regtest; then + echo "starting bitcoin regtest" + if ! "$BITCOIND_BIN" -datadir="$BITCOIN_DIR" -regtest -daemon >/dev/null; then + echo "error: starting bitcoind regtest daemon" >&2 + return 1 + fi + else + echo "bitcoin regtest already started" + fi + + btc_reg() { + "$BITCOIN_CLI_BIN" \ + -datadir="$BITCOIN_DIR" \ + -regtest \ + -rpcuser="$BITCOIN_REGTEST_RPC_USER" \ + -rpcpassword="$BITCOIN_REGTEST_RPC_PASS" \ + "$@" + } + + # wait for bitcoin to start + while ! btc_reg ping >/dev/null 2>&1; do + echo "waiting for bitcoind regtes to start..." + sleep 1 + done + + # check if default wallet exists + if + ! btc_reg listwalletdir | + jq -r '.wallets[] | .name' | + grep "^default$" >/dev/null 2>&1 + then + if ! btc_reg -named createwallet \ + wallet_name=default \ + disable_private_keys=false \ + blank=false \ + avoid_reuse=false \ + descriptors=true \ + load_on_startup=true \ + external_signer=false >/dev/null + then + echo "error: could not create default wallet" >&2 + return 1 + fi + echo "default bitcoind wallet successfully created" + fi + + # check if default wallet is loaded + if + ! btc_reg listwallets | + jq -r '.[]' | + grep "^default$" >/dev/null 2>&1 + then + if ! btc_reg loadwallet default; then + echo "error: could not load default wallet" >&2 + return 1 + fi + echo "default bitcoind wallet successfully loaded" + fi + + bitcoin_regtest_mine 1 "new" || return "$?" + while [ "$(btc_reg getblockchaininfo | jq -r '.initialblockdownload')" != false ]; do + echo "waiting for bitcoind regtes to download blocks..." + sleep 1 + done + + echo "bitcoin regtest started, data directory is $BITCOIN_DIR" + echo "btc_reg function set" +} + +_bitcoin_regtest_stop() { + if ! _pgrep_bitcoin_regtest; then + echo "bitcoin regtest already stopped" + else + if + ! "$BITCOIN_CLI_BIN" \ + -datadir="$BITCOIN_DIR" \ + -regtest \ + stop >/dev/null + then + echo "error: bitcoin regtest not stopped" >&2 + fi + + while _pgrep_bitcoin_regtest; do + echo "waiting for bitcoin regtest to stop..." + sleep 1 + done + echo "bitcoin regtest stopped" + fi +} + +bitcoin_regtest_stop_and_remove() { + _bitcoin_regtest_stop || return "$?" + + if [ ! -e "$BITCOIN_DIR" ]; then + echo "bitcoin directory not present" + else + if ! rm -rf "$BITCOIN_DIR"; then + echo "error: removing bitcoin directory $BITCOIN_DIR" + return 1 + fi + echo "removed bitcoin directory $BITCOIN_DIR" + fi +} + +_pgrep_cln_coord() { + pgrep -f \ + "$LIGHTNINGD_BIN --lightning-dir=$CLN_COORD_DIR --regtest --daemon" \ + >/dev/null +} + +cln_coord_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$CLN_COORD_DIR" || return "$?" + + echo "writing cln coordinator config" + cat << EOF > "$CLN_COORD_DIR/config" +bitcoin-cli=$BITCOIN_CLI_BIN +bitcoin-datadir=$BITCOIN_DIR +bitcoin-rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoin-rpcpassword=$BITCOIN_REGTEST_RPC_PASS +bitcoin-rpcport=$BITCOIN_REGTEST_RPC_PORT +alias=cln-coordinator +rgb=100000 +fee-base=0 +fee-per-satoshi=0 +min-capacity-sat=990000 +ignore-fee-limits=true +funding-confirms=1 +max-concurrent-htlcs=64 +max-dust-htlc-exposure-msat=1000000000 +cltv-delta=144 +cltv-final=144 +bitcoin-retry-timeout=120 +database-upgrade=true +log-file=lightning.log +addr=localhost:$CLN_COORD_LISTEN_PORT +grpc-port=$CLN_COORD_GRPC_PORT +important-plugin=$HOLDINVOICE_PLUGIN_BIN +grpc-hold-port=$CLN_COORD_HOLD_PORT +disable-plugin=clnrest +disable-plugin=wss-proxy +EOF + + if ! _pgrep_cln_coord; then + echo "starting cln coordinator" + if + ! "$LIGHTNINGD_BIN" \ + --lightning-dir="$CLN_COORD_DIR" \ + --regtest \ + --daemon \ + >/dev/null + then + echo "error: starting cln coordinator daemon" >&2 + return 1 + fi + else + echo "cln coordinator already started" + fi + + cln_coord() { + "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_COORD_DIR" \ + --regtest \ + "$@" + } + + while [ \ + "$(cln_coord getinfo | jq -r '.blockheight')" -ne \ + "$(btc_reg getblockcount)" \ + ]; do + echo "waiting for cln coordinator to sync with the chain..." + sleep 1 + done + + echo "cln coordinator started, data directory is $CLN_COORD_DIR" + echo "cln_coord function set" +} + +_cln_coord_stop() { + if ! _pgrep_cln_coord; then + echo "cln coordinator already stopped" + else + if + ! "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_COORD_DIR" \ + --regtest \ + stop >/dev/null + then + echo "error: cln coordinator not stopped" >&2 + fi + + while _pgrep_cln_coord; do + echo "waiting for cln coordinator to stop..." + sleep 1 + done + echo "cln coordinator stopped" + fi +} + +cln_coord_stop_and_remove() { + _cln_coord_stop || return "$?" + + if [ ! -e "$CLN_COORD_DIR" ]; then + echo "cln coordinator directory not present" + else + if ! rm -rf "$CLN_COORD_DIR"; then + echo "error: removing cln coordinator directory $CLN_COORD_DIR" + return 1 + fi + echo "removed cln coordinator directory $CLN_COORD_DIR" + fi +} + +_pgrep_cln_user() { + pgrep -f \ + "$LIGHTNINGD_BIN --lightning-dir=$CLN_USER_DIR --regtest --daemon" \ + >/dev/null +} + +cln_user_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$CLN_USER_DIR" || return "$?" + + echo "writing cln user config" + cat << EOF > "$CLN_USER_DIR/config" +bitcoin-cli=$BITCOIN_CLI_BIN +bitcoin-datadir=$BITCOIN_DIR +bitcoin-rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoin-rpcpassword=$BITCOIN_REGTEST_RPC_PASS +bitcoin-rpcport=$BITCOIN_REGTEST_RPC_PORT +alias=cln-user +rgb=200000 +fee-base=0 +fee-per-satoshi=0 +min-capacity-sat=990000 +ignore-fee-limits=true +funding-confirms=1 +max-concurrent-htlcs=64 +max-dust-htlc-exposure-msat=1000000000 +cltv-delta=144 +cltv-final=144 +bitcoin-retry-timeout=120 +database-upgrade=true +log-file=lightning.log +addr=localhost:$CLN_USER_LISTEN_PORT +disable-plugin=clnrest +disable-plugin=wss-proxy +EOF + + if ! _pgrep_cln_user; then + echo "starting cln user" + if + ! "$LIGHTNINGD_BIN" \ + --lightning-dir="$CLN_USER_DIR" \ + --regtest \ + --daemon \ + >/dev/null + then + echo "error: starting cln user daemon" >&2 + return 1 + fi + else + echo "cln user already started" + fi + + cln_user() { + "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_USER_DIR" \ + --regtest \ + "$@" + } + + while [ \ + "$(cln_user getinfo | jq -r '.blockheight')" -ne \ + "$(btc_reg getblockcount)" \ + ]; do + echo "waiting for cln user to sync with the chain..." + sleep 1 + done + + echo "cln user started, data directory is $CLN_USER_DIR" + echo "cln_user function set" +} + +_cln_user_stop() { + if ! _pgrep_cln_user; then + echo "cln user already stopped" + else + if + ! "$LIGHTNING_CLI_BIN" \ + --lightning-dir="$CLN_USER_DIR" \ + --regtest \ + stop >/dev/null + then + echo "error: cln user not stopped" >&2 + fi + + while _pgrep_cln_user; do + echo "waiting for cln user to stop..." + sleep 1 + done + echo "cln user stopped" + fi +} + +cln_user_stop_and_remove() { + _cln_user_stop || return "$?" + + if [ ! -e "$CLN_USER_DIR" ]; then + echo "cln user directory not present" + else + if ! rm -rf "$CLN_USER_DIR"; then + echo "error: removing cln user directory $CLN_USER_DIR" + return 1 + fi + echo "removed cln user directory $CLN_USER_DIR" + fi +} + +_pgrep_lnd_coord() { + pgrep -f "$LND_BIN --lnddir=$LND_COORD_DIR --bitcoin.regtest" >/dev/null +} + +lnd_coord_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$LND_COORD_DIR" || return "$?" + + echo "writing lnd coordinator config" + cat << EOF > "$LND_COORD_DIR/lnd.conf" +[Application Options] +listen=localhost:$LND_COORD_LISTEN_PORT +rpclisten=localhost:$LND_COORD_RPC_PORT +restlisten=localhost:$LND_COORD_REST_PORT +no-rest-tls=true +tlsdisableautofill=true +noseedbackup=true +maxpendingchannels=16 +minchansize=20000 +alias=lnd-coordinator +color=#300000 + +[Bitcoin] +bitcoin.node=bitcoind +bitcoin.defaultchanconfs=1 +bitcoin.basefee=0 +bitcoin.feerate=0 + +[Bitcoind] +bitcoind.dir=$BITCOIN_DIR +bitcoind.config=$BITCOIN_DIR/bitcoin.conf +bitcoind.rpchost=localhost:$BITCOIN_REGTEST_RPC_PORT +bitcoind.rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoind.rpcpass=$BITCOIN_REGTEST_RPC_PASS +bitcoind.zmqpubrawblock=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_BLOCK_PORT +bitcoind.zmqpubrawtx=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_TX_PORT + +[protocol] +protocol.wumbo-channels=true +EOF + + if ! _pgrep_lnd_coord; then + echo "starting lnd coordinator" + "$LND_BIN" \ + --lnddir="$LND_COORD_DIR" \ + --bitcoin.regtest \ + >"$REGTEST_LOGS_DIR/lnd-coord" 2>&1 & + else + echo "lnd coordinator already started" + fi + + lnd_coord() { + "$LNCLI_BIN" \ + --lnddir="$LND_COORD_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_COORD_RPC_PORT" \ + "$@" + } + + while [ "$(lnd_coord getinfo 2>/dev/null | jq -r '.synced_to_chain')" != true ]; do + echo "waiting for lnd coordinator to sync with the chain..." + sleep 1 + done + + # while [ "$(lnd_coord getinfo 2>/dev/null | jq -r '.synced_to_graph')" != true ]; do + # echo "waiting for lnd coordinator to sync with the graph..." + # sleep 1 + # done + + echo "lnd coordinator started, data directory is $LND_COORD_DIR" + echo "lnd_coord function set" +} + +_lnd_coord_stop() { + if ! _pgrep_lnd_coord; then + echo "lnd coordinator already stopped" + else + if + ! "$LNCLI_BIN" \ + --lnddir="$LND_COORD_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_COORD_RPC_PORT" \ + stop >/dev/null + then + echo "error: lnd coordinator not stopped" >&2 + fi + + while _pgrep_lnd_coord; do + echo "waiting for lnd coordinator to stop..." + sleep 1 + done + echo "lnd coordinator stopped" + fi +} + +lnd_coord_stop_and_remove() { + _lnd_coord_stop || return "$?" + + if [ ! -e "$LND_COORD_DIR" ]; then + echo "lnd coordinator directory not present" + else + if ! rm -rf "$LND_COORD_DIR"; then + echo "error: removing lnd coordinator directory $LND_COORD_DIR" + return 1 + fi + echo "removed lnd coordinator directory $LND_COORD_DIR" + fi +} + +_pgrep_lnd_user() { + pgrep -f "$LND_BIN --lnddir=$LND_USER_DIR --bitcoin.regtest" >/dev/null +} + +lnd_user_start() { + _bitcoin_regtest_check_started || return "$?" + + _create_dir "$LND_USER_DIR" || return "$?" + + echo "writing lnd user config" + cat << EOF > "$LND_USER_DIR/lnd.conf" +[Application Options] +listen=localhost:$LND_USER_LISTEN_PORT +rpclisten=localhost:$LND_USER_RPC_PORT +restlisten=localhost:$LND_USER_REST_PORT +no-rest-tls=true +tlsdisableautofill=true +noseedbackup=true +maxpendingchannels=16 +minchansize=20000 +alias=lnd-user +color=#400000 + +[Bitcoin] +bitcoin.node=bitcoind +bitcoin.defaultchanconfs=1 +bitcoin.basefee=0 +bitcoin.feerate=0 + +[Bitcoind] +bitcoind.dir=$BITCOIN_DIR +bitcoind.config=$BITCOIN_DIR/bitcoin.conf +bitcoind.rpchost=localhost:$BITCOIN_REGTEST_RPC_PORT +bitcoind.rpcuser=$BITCOIN_REGTEST_RPC_USER +bitcoind.rpcpass=$BITCOIN_REGTEST_RPC_PASS +bitcoind.zmqpubrawblock=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_BLOCK_PORT +bitcoind.zmqpubrawtx=tcp://127.0.0.1:$BITCOIN_REGTEST_ZMQ_TX_PORT + +[protocol] +protocol.wumbo-channels=true +EOF + + if ! _pgrep_lnd_user; then + echo "starting lnd user" + "$LND_BIN" \ + --lnddir="$LND_USER_DIR" \ + --bitcoin.regtest \ + >"$REGTEST_LOGS_DIR/lnd-user" 2>&1 & + else + echo "lnd user already started" + fi + + lnd_user() { + "$LNCLI_BIN" \ + --lnddir="$LND_USER_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_USER_RPC_PORT" \ + "$@" + } + + while [ "$(lnd_user getinfo 2>/dev/null | jq -r '.synced_to_chain')" != true ]; do + echo "waiting for lnd user to sync with the chain..." + sleep 1 + done + + # while [ "$(lnd_user getinfo 2>/dev/null | jq -r '.synced_to_graph')" != true ]; do + # echo "waiting for lnd user to sync with the graph..." + # sleep 1 + # done + + echo "lnd user started, data directory is $LND_USER_DIR" + echo "lnd_user function set" +} + +_lnd_user_stop() { + if ! _pgrep_lnd_user; then + echo "lnd user already stopped" + else + if + ! "$LNCLI_BIN" \ + --lnddir="$LND_USER_DIR" \ + --network regtest \ + --rpcserver localhost:"$LND_USER_RPC_PORT" \ + stop >/dev/null + then + echo "error: lnd user not stopped" >&2 + fi + + while _pgrep_lnd_user; do + echo "waiting for lnd user to stop..." + sleep 1 + done + echo "lnd user stopped" + fi +} + +lnd_user_stop_and_remove() { + _lnd_user_stop || return "$?" + + if [ ! -e "$LND_USER_DIR" ]; then + echo "lnd user directory not present" + else + if ! rm -rf "$LND_USER_DIR"; then + echo "error: removing lnd user directory $LND_USER_DIR" + return 1 + fi + echo "removed lnd user directory $LND_USER_DIR" + fi +} + +_mine_blocks_coordinator() { + coord_node="$1" + case "$coord_node" in + cln) + if ! coord_address="$( + cln_coord -k newaddr addresstype=p2tr | jq -r '.p2tr' + )"; then + echo "error: generating cln coordinator address" >&2 + return 1 + fi + ;; + lnd) + if ! coord_address="$( + lnd_coord newaddress p2tr | jq -r '.address' + )"; then + echo "error: generating lnd coordinator address" >&2 + return 1 + fi + ;; + esac + + bitcoin_regtest_mine 101 "$coord_address" || return "$?" + unset user_address +} + +_robosats_regtest_channel_create_cln_user() { + if [ "$#" -lt 1 ]; then + echo "error: insert coordinator node" >&2 + return 1 + fi + coord_node="$1" + shift 1 + + _bitcoin_regtest_check_started || return "$?" + + case "$coord_node" in + cln) + if ! _pgrep_cln_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist cln_coord; then + return 1 + fi + coord_id="$(cln_coord getinfo | jq -r '.id')" + coord_port="$CLN_COORD_LISTEN_PORT" + ;; + lnd) + if ! _pgrep_lnd_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist lnd_coord; then + return 1 + fi + coord_id="$(lnd_coord getinfo | jq -r '.identity_pubkey')" + coord_port="$LND_COORD_LISTEN_PORT" + ;; + *) + echo "error: coordinator node can only be cln and lnd" >&2 + return 1 + ;; + esac + + if ! _pgrep_cln_user; then + echo "error: cln user not started" >&2 + return 1 + fi + if ! _function_exist cln_user; then + return 1 + fi + + if ! cln_user connect "$coord_id"@localhost:"$coord_port" >/dev/null; then + echo "error: connection lightning nodes" >&2 + return 1 + fi + echo "lightning nodes connected" + + # check if channel not already present + if [ "$( + cln_user listpeerchannels "$coord_id" | + jq -r '.channels | length' + )" -ge 1 ]; then + echo "lightning nodes already have a channel, not opening a new one" + else + echo "mining blocks to coordinator $coord_node" + _mine_blocks_coordinator "$coord_node" || return "$?" + + previous_output_number="$(cln_user listfunds | jq -r '.outputs | length')" + + if ! user_address="$(cln_user -k newaddr addresstype=p2tr | jq -r '.p2tr')"; then + echo "error: generating user address" >&2 + return 1 + fi + echo "mining blocks to user cln" + bitcoin_regtest_mine 101 "$user_address" || return "$?" + unset user_address + + while [ \ + "$(cln_user listfunds | jq -r '.outputs | length')" -le \ + "$previous_output_number" \ + ]; do + echo "waiting for cln user to see the new blocks..." + sleep 5 + done + while [ "$(cln_user listfunds | jq -r '.outputs[0].status')" != "confirmed" ]; do + echo "waiting for cln user funds to mature..." + sleep 1 + done + + unset previous_output_number + + if ! cln_user -k fundchannel \ + id="$coord_id" \ + amount=1btc \ + feerate=10000perkb \ + announce=true \ + >/dev/null + then + echo "error: funding lightning channel" >&2 + return 1 + fi + echo "lightning channel created" + + bitcoin_regtest_mine 10 "new" || return "$?" + while [ "$( + cln_user listpeerchannels "$coord_id" | + jq -r '.channels | length' + )" -lt 1 ]; do + echo "waiting for channel to confirm..." + sleep 1 + done + echo "lightning channel opened" + fi + + while [ "$( + cln_user listpeerchannels "$coord_id" | + jq -r '.channels[].state' + )" != "CHANNELD_NORMAL" ]; do + echo "waiting for channel to be active..." + sleep 5 + done + echo "lightning channel is active" + + unset coord_port + unset coord_id + unset coord_node +} + +_robosats_regtest_channel_create_lnd_user() { + if [ "$#" -lt 1 ]; then + echo "error: insert coordinator node" >&2 + return 1 + fi + coord_node="$1" + shift 1 + + _bitcoin_regtest_check_started || return "$?" + + case "$coord_node" in + cln) + if ! _pgrep_cln_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist cln_coord; then + return 1 + fi + coord_id="$(cln_coord getinfo | jq -r '.id')" + coord_port="$CLN_COORD_LISTEN_PORT" + ;; + lnd) + if ! _pgrep_lnd_coord; then + echo "error: cln coordinator not started" >&2 + return 1 + fi + if ! _function_exist lnd_coord; then + return 1 + fi + coord_id="$(lnd_coord getinfo | jq -r '.identity_pubkey')" + coord_port="$LND_COORD_LISTEN_PORT" + ;; + *) + echo "error: coordinator node can only be cln and lnd" >&2 + return 1 + ;; + esac + + if ! _pgrep_lnd_user; then + echo "error: lnd user not started" >&2 + return 1 + fi + if ! _function_exist lnd_user; then + return 1 + fi + + already_connected=false + for pub_key in $(lnd_user listpeers | jq -r '.peers[].pub_key'); do + if [ "$pub_key" = "$coord_id" ]; then + already_connected=true + break + fi + done + unset pub_key + if [ "$already_connected" = false ]; then + if ! lnd_user connect "$coord_id"@localhost:"$coord_port" >/dev/null; then + echo "error: connection lightning nodes" >&2 + return 1 + fi + fi + unset already_connected + echo "lightning nodes connected" + + # check if channel not already present + if [ "$( + lnd_user listchannels --peer "$coord_id" | + jq -r '.channels | length' + )" -ge 1 ]; then + echo "lightning nodes already have a channel, not opening a new one" + else + echo "mining blocks to coordinator $coord_node" + _mine_blocks_coordinator "$coord_node" || return "$?" + + previous_output_number="$(lnd_user listunspent | jq -r '.utxos | length')" + + if ! user_address="$(lnd_user newaddress p2tr | jq -r '.address')"; then + echo "error: generating user address" >&2 + return 1 + fi + echo "mining blocks to user lnd" + bitcoin_regtest_mine 101 "$user_address" || return "$?" + unset user_address + + while [ \ + "$(lnd_user listunspent | jq -r '.utxos | length')" -le \ + "$previous_output_number" \ + ]; do + echo "waiting for lnd user to see the new blocks..." + sleep 5 + done + while [ "$(lnd_user listunspent | jq -r '.utxos[0].confirmations')" -lt 100 ]; do + echo "waiting for lnd user funds to mature..." + sleep 1 + done + + unset previous_output_number + + if ! lnd_user openchannel \ + --node_key "$coord_id" \ + --local_amt "100000000" \ + --sat_per_vbyte "10" \ + --min_confs "1" \ + --channel_type "anchors" \ + >/dev/null + then + echo "error: funding lightning channel" >&2 + return 1 + fi + echo "lightning channel created" + + bitcoin_regtest_mine 10 "new" || return "$?" + while [ "$( + lnd_user listchannels --peer "$coord_id" | + jq -r '.channels | length' + )" -lt 1 ]; do + echo "waiting for channel to confirm..." + sleep 1 + done + echo "lightning channel opened" + fi + + while [ "$( + lnd_user listchannels --peer "$coord_id" | + jq -r '.channels[].active' + )" != true ]; do + echo "waiting for channel to be active..." + sleep 5 + done + echo "lightning channel is active" + + unset coord_port + unset coord_id + unset coord_node +} + +robosats_regtest_channel_create() { + if [ "$#" -lt 1 ]; then + echo "error: insert coordinator node" >&2 + return 1 + fi + coord_node="$1" + shift 1 + + if [ "$#" -lt 1 ]; then + echo "error: insert user node" >&2 + return 1 + fi + user_node="$1" + shift 1 + + case "$coord_node" in + cln|lnd) ;; + *) + echo "error: $coord_node should be cln or lnd" >&2 + return 1 + ;; + esac + + case "$user_node" in + cln) + _robosats_regtest_channel_create_cln_user "$coord_node" + ;; + lnd) + _robosats_regtest_channel_create_lnd_user "$coord_node" + ;; + *) + echo "error: $user_node should be cln or lnd" >&2 + return 1 + ;; + esac + + unset user_node + unset coord_node +} + +roboauto_regtest_setup() { + if [ "$ROBOAUTO_GIT_DIR" = false ]; then + echo "error: roboauto is disable, set ROBOAUTO_GIT_DIR in .env to activate it" >&2 + return 1 + fi + + _command_exist roboauto || return "$?" + + _create_dir "$ROBOAUTO_DIR" || return "$?" + + ra_reg() { + roboauto \ + --config-dir "$REGTEST_NODES_DIR/roboauto" \ + --data-dir "$REGTEST_NODES_DIR/roboauto" \ + "$@" + } + + echo "writing roboauto regtest config" + cat << EOF > "$ROBOAUTO_DIR/config.ini" +[federation] +exp = None +sau = None +tos = None +tbl = None +bve = None +loc = "http://127.0.0.1:8000" +EOF + + case "$LNVENDOR_USER" in + cln) + roboauto_cln_script="$ROBOAUTO_GIT_DIR/data/lightning-node-core-lightning" + if [ ! -f "$roboauto_cln_script" ]; then + echo "error: roboauto cln script not found in $ROBOAUTO_GIT_DIR" >&2 + return 1 + fi + + search_data=$(cat << EOF +lightning-cli "\$@" +EOF + ) + new_data=$(cat << EOF +"$LIGHTNING_CLI_BIN" \ +--lightning-dir="$CLN_USER_DIR" \ +--regtest \ +"\$@" +EOF + ) + sed "s|$search_data|$new_data|" \ + "$roboauto_cln_script" > "$ROBOAUTO_DIR/lightning-node" || \ + return "$?" + + echo "roboauto cln node script set up" + ;; + lnd) + roboauto_lnd_script="$ROBOAUTO_GIT_DIR/data/lightning-node-lnd" + if [ ! -f "$roboauto_lnd_script" ]; then + echo "error: roboauto lnd script not found in $ROBOAUTO_GIT_DIR" >&2 + return 1 + fi + + search_data=$(cat << EOF +lncli "\$@" +EOF + ) + new_data=$(cat << EOF +"$LNCLI_BIN" \ +--lnddir="$LND_USER_DIR" \ +--network regtest \ +--rpcserver localhost:"$LND_USER_RPC_PORT" \ +"\$@" +EOF + ) + sed "s|$search_data|$new_data|" \ + "$roboauto_lnd_script" > "$ROBOAUTO_DIR/lightning-node" || \ + return "$?" + + echo "roboauto lnd node script set up" + ;; + esac + if ! chmod u+x "$ROBOAUTO_DIR/lightning-node"; then + echo "error: changing file mode on roboauto lightning-node" >&2 + return 1 + fi +} + +roboauto_regtest_remove() { + if ! rm -rf "$ROBOAUTO_DIR"; then + echo "error: removing roboauto directory $ROBOAUTO_DIR" >&2 + return 1 + fi +} + +robosats_regtest_stop_all() { + _lnd_user_stop + _lnd_coord_stop + _cln_user_stop + _cln_coord_stop + _bitcoin_regtest_stop +} + +robosats_regtest_stop_and_remove_all() { + roboauto_regtest_remove + lnd_user_stop_and_remove + lnd_coord_stop_and_remove + cln_user_stop_and_remove + cln_coord_stop_and_remove + bitcoin_regtest_stop_and_remove +} + +_node_test_setup() { + bitcoin_regtest_start || return "$?" + + bitcoin_regtest_mine 1 "new" || return "$?" + + printf "\n" + + case "$LNVENDOR_COORD" in + cln) + cln_coord_start || return "$?" + ;; + lnd) + lnd_coord_start || return "$?" + ;; + esac + + printf "\n" + + case "$LNVENDOR_USER" in + cln) + cln_user_start || return "$?" + ;; + lnd) + lnd_user_start || return "$?" + ;; + esac +} + +_node_server_setup() { + _node_test_setup || return "$?" + + printf "\n" + + robosats_regtest_channel_create \ + "$LNVENDOR_COORD" "$LNVENDOR_USER" || return "$?" + + if [ "$ROBOAUTO_GIT_DIR" != false ]; then + printf "\n" + + roboauto_regtest_setup + fi +} + +_robosats_regtest_info_print() { + cat << EOF +regtest nodes directory is $REGTEST_NODES_DIR +available command: + +bitcoin_regtest_start +bitcoin_regtest_stop_and_remove +bitcoin_regtest_mine number-of-blocks
|new +cln_coord_start +cln_coord_stop_and_remove +cln_user_start +cln_user_stop_and_remove +lnd_coord_start +lnd_coord_stop_and_remove +lnd_user_start +lnd_user_stop_and_remove +roboauto_regtest_setup +roboauto_regtest_remove +robosats_regtest_stop_all +robosats_regtest_stop_and_remove_all +robosats_regtest_channel_create cln|lnd cln|lnd +EOF +} + +_nodes_main() { + _nodes_environment_set || return "$?" + + # if shell is bash and script is sourced source also bash completions + if [ -n "$BASH_VERSION" ]; then + case "$0" in + /*|./*|../*) ;; + *) + if [ -f "scripts/traditional/robosats.bash-completion" ]; then + # shellcheck disable=SC1091 + . scripts/traditional/robosats.bash-completion + fi + if + [ "$ROBOAUTO_GIT_DIR" != false ] && + [ -f "$ROBOAUTO_GIT_DIR/completions/roboauto.bash-completion" ] + then + # shellcheck disable=SC1091 + # shellcheck disable=SC3044 + . "$ROBOAUTO_GIT_DIR/completions/roboauto.bash-completion" && + complete -F __roboauto_completion ra_reg + fi + ;; + esac + fi + + if [ "$#" -lt 1 ]; then + _robosats_regtest_info_print || return "$?" + else + case "$1" in + -h|--help) + cat << EOF +regtest-nodes [server|test] +EOF + return 0 + ;; + test) + _robosats_regtest_info_print || return "$?" + printf "\n" + _node_test_setup || return "$?" + ;; + server) + _robosats_regtest_info_print || return "$?" + printf "\n" + _node_server_setup || return "$?" + ;; + *) + echo "error: action $1 not recognized" >&2 + return 1 + ;; + esac + fi +} + +_nodes_main "$@" diff --git a/scripts/traditional/regtest-services b/scripts/traditional/regtest-services new file mode 100755 index 000000000..e1b70e313 --- /dev/null +++ b/scripts/traditional/regtest-services @@ -0,0 +1,372 @@ +#!/bin/sh + +_command_exist() { + if command -v "$1" >/dev/null 2>&1; then + return 0 + else + echo "error: $1 command not found" >&2 + return 1 + fi +} + +_create_dir() { + if [ ! -e "$1" ]; then + mkdir -p "$1" || return "$?" + if [ "$#" -ge 2 ]; then + if ! chmod "$2" "$1"; then + echo "error: setting chmod $2 of $1" >&2 + return 1 + fi + fi + elif [ ! -d "$1" ]; then + echo "error: $1 is not a directory" >&2 + return 1 + fi +} + +_get_env_var() { + if ! env_var="$(dotenv -f ".env" get "$1" 2>/dev/null)"; then + echo "error: getting $1 from .env" >&2 + return 1 + fi + printf "%s\n" "$env_var" + + return 0 +} + +# transform relative path into absolute and remove trailing slashes +_get_env_var_path() { + env_var="$(_get_env_var "$1")" || return "$?" + real_path="$(realpath -m "$env_var")" || return "$?" + printf "%s\n" "$real_path" + + return 0 +} + +_services_environment_set() { + # docker-compose.yml + CELERY_WORKER_COMMAND="celery -A robosats worker --loglevel=INFO --concurrency 4 --max-tasks-per-child=4 --max-memory-per-child=200000" + CELERY_BEAT_COMMAND="celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" + + REGTEST_SERVICES_DIR="$(_get_env_var_path "REGTEST_SERVICES_DIR")" || return "$?" + REGTEST_LOGS_DIR="$(_get_env_var_path "REGTEST_LOGS_DIR")" || return "$?" + _create_dir "$REGTEST_SERVICES_DIR" || return "$?" + _create_dir "$REGTEST_LOGS_DIR" || return "$?" + + POSTGRES_DIR="$REGTEST_SERVICES_DIR/postgres" + REDIS_DIR="$REGTEST_SERVICES_DIR/redis" + GNUPG_DIR="$(_get_env_var_path "GNUPG_DIR")" || return "$?" + + POSTGRES_DB="$(_get_env_var "POSTGRES_DB")" || return "$?" + POSTGRES_USER="$(_get_env_var "POSTGRES_USER")" || return "$?" + POSTGRES_PASS="$(_get_env_var "POSTGRES_PASSWORD")" || return "$?" + POSTGRES_PORT="$(_get_env_var "POSTGRES_PORT")" || return "$?" + + REDIS_PORT="$(_get_env_var "REDIS_PORT")" || return "$?" + + # defaults + + POSTGRES_DB="${POSTGRES_DB:-postgres}" + POSTGRES_USER="${POSTGRES_USER:-postgres}" + POSTGRES_PASS="${POSTGRES_PASS:-example}" + POSTGRES_PORT="${POSTGRES_PORT:-5432}" + + REDIS_PORT="${REDIS_REGTEST_PORT:-6379}" +} + +postgres_action() { + if [ "$#" -lt 1 ]; then + echo "error: insert postgres action" >&2 + return 1 + fi + action="$1" + shift 1 + case "$action" in + setup|database) ;; + *) + echo "error: wrong action" >&2 + return 1 + ;; + esac + + if ! _command_exist postgres; then + return 1 + fi + if ! _command_exist initdb; then + return 1 + fi + if ! _command_exist psql; then + return 1 + fi + + case "$action" in + setup) + if [ -e "$POSTGRES_DIR" ]; then + echo "postgres directory $POSTGRES_DIR already exists" + return 0 + fi + _create_dir "$POSTGRES_DIR" "0700" || return "$?" + + if ! initdb -D "$POSTGRES_DIR"; then + echo "error: running initdb" >&2 + return 1 + fi + + cat << EOF > "$POSTGRES_DIR/postgresql.conf" +port = $POSTGRES_PORT +unix_socket_directories = '$POSTGRES_DIR' +EOF + ;; + database) + if [ ! -d "$POSTGRES_DIR" ]; then + printf "%s%s\n" \ + "error: $POSTGRES_DIR is not a directory, " \ + "should run postgres-setup" \ + >&2 + return 1 + fi + ;; + esac + + postgres_setup_log_file="$REGTEST_LOGS_DIR/postgres-setup-log" + echo "starting postgres, setup log file is $postgres_setup_log_file" + postgres -D "$POSTGRES_DIR" >>"$postgres_setup_log_file" 2>&1 & + postgres_pid="$!" + + _postgres_shut_down() { + echo "shutting down postgres" + if ! kill "$postgres_pid"; then + echo "error: shutting down postgres" >&2 + return 1 + fi + } + + sleep 5 + + case "$action" in + setup) + echo "setting up postgres user $POSTGRES_USER" + psql_stdin=$(cat << EOF +CREATE ROLE $POSTGRES_USER WITH LOGIN PASSWORD '$POSTGRES_PASS'; +ALTER ROLE $POSTGRES_USER CREATEDB; +EOF + ) + ;; + database) + psql_stdin=$(cat << EOF +CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER; +EOF + ) + ;; + esac + printf "%s\n" "$psql_stdin" | + psql -h localhost -p "$POSTGRES_PORT" -U "$USER" -d postgres + + sleep 5 + + case "$action" in + database) + if ! DJANGO_SUPERUSER_USERNAME="$(_get_env_var "ESCROW_USERNAME")"; then + _postgres_shut_down || return "$?" + return 1 + fi + # shellcheck disable=SC2034 + DJANGO_SUPERUSER_PASSWORD="password" + DJANGO_SUPERUSER_EMAIL="superuser@email.com" + + if ! python3 manage.py migrate; then + _postgres_shut_down || return "$?" + return 1 + fi + + if ! python3 manage.py createsuperuser \ + --noinput \ + --username "$DJANGO_SUPERUSER_USERNAME" \ + --email "$DJANGO_SUPERUSER_EMAIL" + then + _postgres_shut_down || return "$?" + return 1 + fi + ;; + esac + + _postgres_shut_down || return "$?" + + return 0 +} + +cleanup_signal() { + printf "\n" + printf "%s\n" "Caught $1 signal, sending it to services..." + + pkill -"$2" -f "$CELERY_BEAT_COMMAND" + pkill -"$2" -f "$CELERY_WORKER_COMMAND" + pkill -"$2" -f "python3 manage.py follow_invoices" + pkill -"$2" -f "python3 manage.py clean_orders" + pkill -"$2" -f "python3 manage.py runserver 0.0.0.0:8000" + pkill -"$2" -f "redis-server \*:${REDIS_PORT}" + pkill -"$2" -f "postgres -D $POSTGRES_DIR" + + exit 0 +} + +cleanup_int() { + printf "\n" + printf "%s\n" "Caught INT signal, shutting down services..." + + pkill -INT -f "$CELERY_BEAT_COMMAND" + pkill -INT -f "$CELERY_WORKER_COMMAND" + pkill -TERM -f "python3 manage.py follow_invoices" + pkill -TERM -f "python3 manage.py clean_orders" + pkill -TERM -f "python3 manage.py runserver 0.0.0.0:8000" + # pkill -INT -f "python3 manage.py follow_invoices" + # pkill -INT -f "python3 manage.py clean_orders" + # pkill -INT -f "python3 manage.py runserver 0.0.0.0:8000" + pkill -INT -f "redis-server \*:${REDIS_PORT}" + pkill -INT -f "postgres -D $POSTGRES_DIR" + + exit 0 +} + +main_loop() { + if [ "$#" -lt 1 ]; then + echo "error: insert main loop action" >&2 + return 1 + fi + case "$1" in + test) + is_test=true + ;; + server) + is_test=false + ;; + *) + echo "error: $1 is invalid" >&2 + return 1 + ;; + esac + shift 1 + + if ! _command_exist postgres; then + return 1 + fi + if ! _command_exist redis-server; then + return 1 + fi + if ! _command_exist celery; then + return 1 + fi + + if [ ! -d "$POSTGRES_DIR" ]; then + printf "%s%s\n" \ + "error: $POSTGRES_DIR is not a directory, " \ + "should run postgres-setup and postgres-database" \ + >&2 + return 1 + fi + + if ! pgrep -a bitcoind >/dev/null 2>&1 || { + ! pgrep -a lightningd >/dev/null 2>&1 && + ! pgrep -a lnd >/dev/null 2>&1 + }; then + printf "%s%s\n" \ + "error: bitcoin or lightning not running, " \ + "make sure to run this script after running regtest-nodes" \ + >&2 + return 1 + fi + + _create_dir "$REDIS_DIR" || return "$?" + _create_dir "$GNUPG_DIR" "0700" || return "$?" + + trap "cleanup_signal HUP" HUP + trap "cleanup_signal QUIT" QUIT + trap "cleanup_signal TERM" TERM + trap "cleanup_int" INT + + while true; do + if ! pgrep -f "postgres -D $POSTGRES_DIR" >/dev/null; then + echo "starting postgres" + postgres -D "$POSTGRES_DIR" >> "$REGTEST_LOGS_DIR/postgres" 2>&1 & + fi + + if ! pgrep -f "redis-server \*:${REDIS_PORT}" >/dev/null; then + echo "starting redis" + printf "%s\n%s\n%s\n" \ + "dir $REDIS_DIR" \ + "port $REDIS_PORT" \ + "maxclients 1024" | + redis-server - >> "$REGTEST_LOGS_DIR/redis" 2>&1 & + fi + + if [ "$is_test" = false ]; then + if ! pgrep -f "python3 manage.py runserver 0.0.0.0:8000" >/dev/null; then + echo "starting runserver" + python3 manage.py runserver 0.0.0.0:8000 \ + >> "$REGTEST_LOGS_DIR/runserver" 2>&1 & + fi + + if ! pgrep -f "python3 manage.py clean_orders" >/dev/null; then + echo "starting clean-orders" + python3 manage.py clean_orders \ + >> "$REGTEST_LOGS_DIR/clean-orders" 2>&1 & + fi + + if ! pgrep -f "python3 manage.py follow_invoices" >/dev/null; then + echo "starting follow-invoices" + python3 manage.py follow_invoices \ + >> "$REGTEST_LOGS_DIR/follow-invoices" 2>&1 & + fi + + if ! pgrep -f "$CELERY_WORKER_COMMAND" >/dev/null; then + echo "starting celery worker" + $CELERY_WORKER_COMMAND >> "$REGTEST_LOGS_DIR/celery-worker" 2>&1 & + fi + + if ! pgrep -f "$CELERY_BEAT_COMMAND" >/dev/null; then + echo "starting celery beat" + $CELERY_BEAT_COMMAND >> "$REGTEST_LOGS_DIR/celery-beat" 2>&1 & + fi + fi + + sleep 2 + done +} + +_services_main() { + if [ "$#" -lt 1 ]; then + echo "error: insert action" >&2 + return 1 + fi + action="$1" + shift 1 + + case "$action" in + -h|--help) + cat << EOF +regtest-services postgres-setup|postgres-database|server|test +EOF + return 0 + ;; + esac + + _services_environment_set || return "$?" + + case "$action" in + postgres-setup) + postgres_action "setup" + ;; + postgres-database) + postgres_action "database" + ;; + server|test) + main_loop "$action" + ;; + *) + echo "error: action $action not recognized" >&2 + return 1 + ;; + esac +} + +_services_main "$@" diff --git a/scripts/traditional/robosats.bash-completion b/scripts/traditional/robosats.bash-completion new file mode 100755 index 000000000..2b3dee15a --- /dev/null +++ b/scripts/traditional/robosats.bash-completion @@ -0,0 +1,281 @@ +#!/usr/bin/env bash + +# adapted from +# https://github.com/bitcoin/bitcoin/blob/master/contrib/completions/bash/bitcoin-cli.bash +_bitcoin_cli() { + local cur prev words=() cword + local bitcoin_cli + + # save and use original argument to invoke bitcoin-cli for -help, help and RPC + # as bitcoin-cli might not be in $PATH + bitcoin_cli="$1" + + if ! command -v "$bitcoin_cli" >/dev/null 2>&1; then + return 0 + fi + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + if ((cword > 5)); then + case ${words[cword-5]} in + sendtoaddress) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 4)); then + case ${words[cword-4]} in + importaddress|listtransactions|setban) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + signrawtransactionwithkey|signrawtransactionwithwallet) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "ALL NONE SINGLE ALL|ANYONECANPAY NONE|ANYONECANPAY SINGLE|ANYONECANPAY" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 3)); then + case ${words[cword-3]} in + addmultisigaddress) + return 0 + ;; + getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + if ((cword > 2)); then + case ${words[cword-2]} in + addnode) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "add remove onetry" -- "$cur" ) ) + return 0 + ;; + setban) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) ) + return 0 + ;; + fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + esac + fi + + case "$prev" in + backupwallet|dumpwallet|importwallet) + _filedir + return 0 + ;; + getaddednodeinfo|getrawmempool|lockunspent) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) + return 0 + ;; + getbalance|getnewaddress|listtransactions|sendmany) + return 0 + ;; + esac + + # determine already specified args necessary for RPC + local rpcargs=() + local i + for i in ${COMP_LINE}; do + case "$i" in + -conf=*|-datadir=*|-rpc*|-chain=*|-testnet|-signet|-regtest) + rpcargs=( "${rpcargs[@]}" "$i" ) + ;; + esac + done + + case "$cur" in + -conf=*) + cur="${cur#*=}" + _filedir + return 0 + ;; + -datadir=*) + cur="${cur#*=}" + _filedir -d + return 0 + ;; + -rpcwallet=*) + cur="${cur#*=}" + wallets="$($bitcoin_cli "${rpcargs[@]}" listwallets | jq -r '.[]')" + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$wallets" -- "$cur" ) ) + return 0 + ;; + -*=*) # prevent nonsense completions + return 0 + ;; + *) + local helpopts commands completions + + # only parse -help if senseful + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + helpopts=$($bitcoin_cli -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' ) + fi + + # only parse help if senseful + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + commands=$($bitcoin_cli "${rpcargs[@]}" help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }') + fi + + completions="$helpopts $commands generatetoaddress" + + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$completions" -- "$cur" ) ) + + # Prevent space if an argument is desired + local word + for word in "${COMPREPLY[@]}"; do + case "$word" in + *=) + compopt -o nospace + break + ;; + esac + done + return 0 + ;; + esac +} && +complete -F _bitcoin_cli btc_reg + +# adapted from +# https://github.com/ElementsProject/lightning/blob/master/contrib/lightning-cli.bash-completion +_lightning_cli() { + local command_name="$1" + # local current_word="$2" + local previous_word="$3" + + local lightning_cli + + # lightning_cli might not be in $PATH + lightning_cli="$command_name" + + if ! command -v "$lightning_cli" >/dev/null 2>&1; then + return 0 + fi + + if [ "${COMP_CWORD}" -eq 1 ]; then + complete_opt=true + else + case "$previous_word" in + --help|-h) complete_opt=false ;; + help|-*) complete_opt=true ;; + *) complete_opt=false ;; + esac + fi + + if [ "$complete_opt" = true ]; then + # shellcheck disable=SC2034 + local cur prev words=() cword + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + case "$cur" in + -*=*) # prevent nonsense completions + return 0 + ;; + *) + local helpopts globalcmds + + # get the global options, starting with -- + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + globalcmds="$( + $lightning_cli --help 2>&1 | + tr '|' '\n' | + sed -n -e 's/ .*//' -e 's/\(-[-a-z0-9A-Z]*\).*/\1/p' + )" + fi + + # get the regular commands + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + helpopts="$( + $lightning_cli help 2>/dev/null | + sed -n 's/^\([a-z][a-z_-]*\).*/\1/p' | + sed '$ d' + )" + fi + + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$helpopts $globalcmds" -X "*," -- "$cur" ) ) + ;; + esac + else + _minimal + fi +} && +complete -F _lightning_cli cln_coord && +complete -F _lightning_cli cln_user + +# adapted from +# https://github.com/lightningnetwork/lnd/blob/master/contrib/lncli.bash-completion +_lncli() { + local cur prev words=() cword + local lncli + + # lncli might not be in $PATH + lncli="$1" + + if ! command -v "$lncli" >/dev/null 2>&1; then + return 0 + fi + + COMPREPLY=() + _get_comp_words_by_ref -n = cur prev words cword + + case "$prev" in + # example of further completion + newaddress) + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "p2wkh np2wkh" -- "$cur" ) ) + return 0 + ;; + esac + + case "$cur" in + -*=*) # prevent nonsense completions + return 0 + ;; + esac + + if [ "$cword" -eq 1 ] || { + [ "$cword" -eq 2 ] && [ "$prev" = "help" ] + }; then + local helpopts globalcmds completions + + # get the global options, starting with -- + if [[ -z "$cur" || "$cur" =~ ^- ]]; then + globalcmds=$($lncli help 2>&1 | awk '$1 ~ /^-/ { sub(/,/, ""); print $1}') + fi + + # get the regular commands + if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then + helpopts=$($lncli help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }' ) + fi + + completions="$helpopts $globalcmds help" + + # shellcheck disable=SC2207 + COMPREPLY=( $( compgen -W "$completions" -X "*," -- "$cur" ) ) + fi +} && +complete -F _lncli lnd_coord && +complete -F _lncli lnd_user diff --git a/setup.md b/setup.md index b47c4789d..d1fa5e49c 100644 --- a/setup.md +++ b/setup.md @@ -111,3 +111,9 @@ You will need these commands also often or eventually: `docker exec -it lnd-dev lncli -network=testnet payinvoice --allow_self_payment` **RoboSats development site should be accessible on 127.0.0.1:8000** + +# Backend Development + +## Traditional environment without docker + +See [scripts/traditional/README.md](scripts/traditional/README.md) diff --git a/tests/utils/node.py b/tests/utils/node.py index d5cb1289e..daf3981f0 100644 --- a/tests/utils/node.py +++ b/tests/utils/node.py @@ -16,17 +16,24 @@ def get_node(name="robot"): We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user) """ if name == "robot": + admin_macaroon_file = config( + "LND_TEST_ROBOT_MACAROON_PATH", cast=str, + default="/lndrobot/data/chain/bitcoin/regtest/admin.macaroon" + ) macaroon = codecs.encode( - open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), - "hex", + open(admin_macaroon_file, "rb").read(), "hex", ) - port = 8080 + port = config("LND_TEST_ROBOT_REST_PORT", cast=int, default=8080) elif name == "coordinator": + admin_macaroon_file = config( + "LND_TEST_COORD_MACAROON_PATH", cast=str, + default="/lnd/data/chain/bitcoin/regtest/admin.macaroon" + ) macaroon = codecs.encode( - open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), "hex" + open(admin_macaroon_file, "rb").read(), "hex" ) - port = 8081 + port = config("LND_TEST_COORD_REST_PORT", cast=int, default=8081) return {"port": port, "headers": {"Grpc-Metadata-macaroon": macaroon}} @@ -158,10 +165,10 @@ def set_up_regtest_network(): # Coordinator is either LND or CLN. Robot user is always LND. if LNVENDOR == "LND": coordinator_node_id = get_lnd_node_id("coordinator") - coordinator_port = 9735 + coordinator_port = config("LND_TEST_COORD_LISTEN_PORT", cast=int, default=9735) elif LNVENDOR == "CLN": coordinator_node_id = get_cln_node_id() - coordinator_port = 9737 + coordinator_port = config("CLN_TEST_COORD_LISTEN_PORT", cast=int, default=9737) print("Coordinator Node ID: ", coordinator_node_id) @@ -258,8 +265,10 @@ def generate_blocks(address, num_blocks): "method": "generatetoaddress", "params": [num_blocks, address], } + rpc_user = config("BITCOIND_TEST_RPCUSER", cast=str, default="test") + rpc_pass = config("BITCOIND_TEST_RPCPASSWORD", cast=str, default="test") response = requests.post( - "http://localhost:18443", json=data, auth=HTTPBasicAuth("test", "test") + "http://localhost:18443", json=data, auth=HTTPBasicAuth(rpc_user, rpc_pass) ) return response.json() diff --git a/tests/utils/pgp.py b/tests/utils/pgp.py index a269ef7e6..ede35d872 100644 --- a/tests/utils/pgp.py +++ b/tests/utils/pgp.py @@ -1,8 +1,9 @@ +from decouple import config import gnupg def sign_message(message, private_key_path, passphrase_path): - gpg = gnupg.GPG() + gpg = gnupg.GPG(gnupghome=config("GNUPG_DIR", default=None)) with open(private_key_path, "r") as f: private_key = f.read()