Merge pull request #377 from district0x/config-via-env

Load & serve config from filesystem during runtime (SERVER_CONFIG_PATH, UI_CONFIG_PATH)

.github/workflows/docker-image.yml

@@ -41,7 +41,7 @@ jobs:
           repository: district0x/ethlance-config
           path: ethlance-config
           token: ${{ secrets.ETHLANCE_CONFIG_PAT }}
-          ref: docker_build
+          ref: master
 
       - name: Run ganache service
         run: |
@@ -106,19 +106,6 @@ jobs:
       - name: Checkout ethlance code
         uses: actions/checkout@v4
 
-      - name: Checkout ethlance-configs repo
-        uses: actions/checkout@v4
-        with:
-          repository: district0x/ethlance-config
-          path: ethlance-config
-          token: ${{ secrets.ETHLANCE_CONFIG_PAT }}
-          ref: docker_build
-
-      - name: Copy configs
-        run: |
-          cp -r ethlance-config/config .
-          ls -lah config/
-
       - name: Restore Top level cache directory for Ethlance
         uses: actions/cache@v3
         id: yarn-cache-top-level
@@ -132,10 +119,6 @@ jobs:
         run: |
           yarn install
 
-      - name: Deploy contracts for ${{ matrix.env }}
-        run: |
-          ETHLANCE_ENV=${{ matrix.env }} npx truffle migrate --network ethlance.mad.is-testnet --reset
-
       - name: Set up QEMU
         id: setup-qemu
         uses: docker/setup-qemu-action@v2
@@ -155,6 +138,19 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
+      - name: Checkout ethlance-configs repo
+        uses: actions/checkout@v4
+        with:
+          repository: district0x/ethlance-config
+          path: ethlance-config
+          token: ${{ secrets.ETHLANCE_CONFIG_PAT }}
+          ref: master
+
+      - name: Copy configs
+        run: |
+          cp -r ethlance-config/config .
+          ls -lah config/
+
       - name: Build and push UI
         uses: docker/build-push-action@v4
         with:
@@ -175,4 +171,4 @@ jobs:
           file: docker-builds/server/Dockerfile
           build-args: BUILD_ENV=${{ matrix.env }}
           push: true
-          tags: ${{ secrets.ECR_REGISTRY }}/ethlance-server:latest-${{ matrix.env }}
+          tags: ${{ secrets.ECR_REGISTRY }}/ethlance-server:latest-${{ matrix.env }}

.gitignore

@@ -13,4 +13,3 @@ designs/deploy
 tags
 temp
 release
-config

README.md

@@ -72,6 +72,49 @@ ipfs config --json Gateway.HTTPHeaders.Access-Control-Allow-Headers '["X-Request
 ipfs config --json Gateway.Writable true
 ```
 
+# Building & deployment
+
+## 1. Smart contracts
+The ethlance smart contracts should (for now) be deployed manually. As a result of the compilation process, the environment specific clojure files with contract addresses get written (e.g. `shared/src/ethlance/shared/smart_contracts_qa.cljs` for QA) and must be committed to git to make them available for deployment of browser & server applications.
+
+During clojure application deployment (browser, server), the contracts must only be compiled, which will generate JSON ABI files under `<ethlance-root>/resources/public/contracts/build/`
+
+Compilation:
+1. `ETHLANCE_ENV=qa npx truffle compile` (replacing the network with one suitable for the env)
+2. This generates ABI JSON files under `<ethlance-root>/resources/public/contracts/build/`
+3. Server app needs to access them during runtime:
+  - configured via `[:smart-contracts :contracts-build-path]`
+4. UI (browser app) needs them available: should be served by Nginx (or your web server of choice)
+  - get loaded from urls like `http://d0x-vm:6500/contracts/build/TestToken.json`
+
+## 2. Server (<ethlance-root>/server)
+To build Server (consists mainly of graphql API):
+1. The following ENV variables need to be set:
+  - `export ETHLANCE_ENV=qa`
+2. Compile with `clj -A:dev:shadow-clj release dev-server`
+  - will result a single file in `out/ethlance_server.js` (and accompanying `ethlance_server.js.map` source map)
+3. Before running it the smart contract ABI JSON files need to be in a location defined in the EDN file `ETHLNCE_CONFIG_PATH` at EDN path `[:smart-contracts :contracts-build-path]`
+  - if it's a relative path, it gets resolved in relation to where the server process gets started
+
+Running server:
+1. Earlier the server config was compiled into the generated JS file. Now it will be loaded during runtime (at application startup) from the variable specified under `:config :env-name`. As for now it is `ETHLANCE_CONFIG_PATH`.
+2. Also earlier the UI (browser app) config was baked into the compiled JS file for the UI. Now it gets served via `/config` endpoint from a location pointed to ENV variable `UI_CONFIG_PATH` (contents loaded at run time from the file system). Thus, starting the server:
+  - `export ETHLANCE_ENV=qa`
+  - `export SERVER_CONFIG_PATH=/path/to/server-config-qa.edn `
+  - `export UI_CONFIG_PATH=/path/to/ui-config-qa.edn`
+  - `node out/ethlance_server.js`
+
+## 3. Browser (<ethlance-root>/ui)
+1. The following ENV variables need to be set:
+  - `export ETHLANCE_ENV=qa`
+2. Compile with `clj -A:dev:shadow-clj release dev-ui`
+  - the generated JS file (single) will be under `<ethlance-root>/ui/resources/public/main.js`
+3. To serve the web page, configure the web server to serve the files under `<ethlance-root>/ui/resources/public`
+  - there are some other files, like CSS, JS and index.html that the web server needs to serve too
+4. The web server must also serve the ABI JSON files generated in _1. Smart contracts_
+  - for that, they can be copied from the build step or re-compiled and moved to where the web server can serve them
+  - the expected HTTP path will be `/contracts/build/<ABI JSON FILE>.json` (e.g. `/contracts/build/Ethlance.json`)
+
 # Contributing
 
 Anyone is welcome to contribute to the ethlance project, here are some brief guidelines:

contracts/Ethlance.sol

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-3.0
-pragma solidity ^0.8.21;
+pragma solidity ^0.8.0;
 pragma experimental ABIEncoderV2;
 
 import "./EthlanceStructs.sol";

contracts/EthlanceStructs.sol

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-3.0
-pragma solidity ^0.8.21;
+pragma solidity ^0.8.0;
 pragma experimental ABIEncoderV2;
 
 import "@openzeppelin/contracts/token/ERC20/IERC20.sol";

contracts/Job.sol

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-3.0
-pragma solidity ^0.8.21;
+pragma solidity ^0.8.0;
 pragma experimental ABIEncoderV2;
 
 import "./EthlanceStructs.sol";

contracts/JobHelpers.sol

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-3.0
-pragma solidity ^0.8.21;
+pragma solidity ^0.8.0;
 pragma experimental ABIEncoderV2;
 
 import "./EthlanceStructs.sol";

contracts/Migrations.sol

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: MIT
-pragma solidity ^0.8.21;
+pragma solidity ^0.8.0;
 
 contract Migrations {
   address public owner = msg.sender;

deploy-commands.sh

@@ -1,41 +1,53 @@
 ETHLANCE_SOURCE_ROOT=/home/madis/code/district0x/ethlance
-ETHLANCE_ENV=qa
+export ETHLANCE_ENV=qa
 DEPLOY_TARGET=mad.is:~/www/ethlance
 
 ## Smart contracts
 cd $ETHLANCE_SOURCE_ROOT
 ETHLANCE_ENV=qa npx truffle migrate --network ethlance.mad.is-testnet --reset
-#
+
 #####################
 ## UI: prepare assets
+#####################
 cd $ETHLANCE_SOURCE_ROOT/ui && yarn
 rm -rf .cpcache/* .shadow-cljs/*
-cd $ETHLANCE_SOURCE_ROOT/ui && ETHLANCE_ENV=qa npx shadow-cljs -A:local-deps release dev-ui
+export ETHLANCE_CONFIG_PATH=/home/madis/code/district0x/ethlance-config/config/ui-config-qa.edn
+export SMART_CONTRACTS_BUILD_PATH=/home/madis/code/district0x/ethlance/resources/public/contracts/build
+export SMART_CONTRACTS=/home/madis/code/district0x/ethlance/shared/src/ethlance/shared/smart_contracts_qa.cljs
+cd $ETHLANCE_SOURCE_ROOT/ui && ETHLANCE_ENV=qa npx shadow-cljs release dev-ui
+# To compile with local dependencies defined under :local-deps alias
+ETHLANCE_ENV=qa clj -A:dev:shadow-cljs:local-deps release dev-ui
+
 cd $ETHLANCE_SOURCE_ROOT/ui && ./node_modules/less/bin/lessc resources/public/less/main.less resources/public/css/main.css --verbose
 
 ## UI: copy assets
 rsync --progress -ru resources/public/{css,images,index.html,js} ../resources/public/contracts $DEPLOY_TARGET/ui
 
 ##################
 ## Server: compile
+##################
 ETHLANCE_SERVER_ROOT=/home/madis/temp/ethlance/server
 
 cd $ETHLANCE_SOURCE_ROOT/server
 # Need to clear shadow-cljs cache to force the slurp macro to be evaluated again
 # because the clojure file didn't change but the .edn config file did
 # Alternatively the ethlance.server.core must be touched after changing config
-rm -rf .shadow-cljs/* .cpcache/*
+rm -rf .shadow-cljs/* .cpcache/* out/*
 
 # Release deployment
-ETHLANCE_ENV=qa npx shadow-cljs -A:local-deps release dev-server
+export ETHLANCE_CONFIG_PATH=/home/madis/code/district0x/ethlance-config/config/server-config-qa.edn
+ETHLANCE_ENV=qa npx shadow-cljs release dev-server
+# To compile including local changes to  libraries:
+ETHLANCE_ENV=qa clj -A:dev:shadow-cljs:local-deps release dev-server
 rsync --progress -ru out/ethlance_server.js out/ethlance_server.js.map package.json $DEPLOY_TARGET/server
 
 ## Start server (on remote machine/container)
 cd $ETHLANCE_SERVER_ROOT && yarn
 cd $ETHLANCE_SERVER_ROOT && node ethlance_server.js
 
 ## Server Postgres
-# $ sudo -u postgres psql
-# > CREATE DATABASE ethlance;
-# > CREATE USER ethlanceuser WITH ENCRYPTED PASSWORD 'pass';
-# > GRANT ALL PRIVILEGES ON DATABASE ethlance TO ethlanceuser;
+sudo -u postgres psql <<EOS
+CREATE DATABASE ethlance;
+CREATE USER ethlanceuser WITH ENCRYPTED PASSWORD 'pass';
+GRANT ALL PRIVILEGES ON DATABASE ethlance TO ethlanceuser;
+EOS

docker-builds/base/Dockerfile

@@ -19,4 +19,4 @@ RUN git clone https://github.com/district0x/d0x-libs
 
 ## Deploy Smart contracts
 WORKDIR /build/ethlance
-RUN yarn && ETHLANCE_ENV=qa npx truffle migrate --network ethlance.mad.is-testnet --reset
+# RUN yarn && ETHLANCE_ENV=qa npx truffle migrate --network ethlance.mad.is-testnet --reset

docker-builds/server/Dockerfile

@@ -2,44 +2,59 @@ FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:
 ARG BUILD_ENV=qa
 USER root
 
+ENV ETHLANCE_COMPONENT="server"
 ENV ETHLANCE_SOURCE_ROOT=/build/ethlance
 ENV ETHLANCE_SERVER_ROOT=/build/ethlance/server
 ENV ETHLANCE_ENV=$BUILD_ENV
 ENV DEPLOY_TARGET=/deploy/
 
+ENV SMART_CONTRACTS_PATH="/build/ethlance/shared/src/ethlance/shared/smart_contracts_${BUILD_ENV}.cljs"
+ENV SMART_CONTRACTS_BUILD_PATH="/build/ethlance/resources/public/contracts/build"
+ENV ETHLANCE_CONFIG_PATH="/build/ethlance/config/${ETHLANCE_COMPONENT}-config-${BUILD_ENV}.edn"
+ENV UI_CONFIG_PATH="/build/ethlance/config/ui-config-${BUILD_ENV}.edn"
+
 WORKDIR /build/ethlance
 COPY . .
 
-RUN cat shared/src/ethlance/shared/smart_contracts_${ETHLANCE_ENV}.cljs
+RUN cat "${SMART_CONTRACTS_PATH}"
 
 WORKDIR /build/ethlance-config
 COPY config .
 
 WORKDIR /build
-RUN git clone https://github.com/district0x/d0x-libs
+# RUN git clone https://github.com/district0x/d0x-libs
 
 FROM init as build_server
+WORKDIR $ETHLANCE_SOURCE_ROOT
+RUN yarn install && ETHLANCE_ENV="${BUILD_ENV}" npx truffle compile
 
 # Release deployment
 WORKDIR $ETHLANCE_SOURCE_ROOT/ui 
 RUN yarn && \
-    ETHLANCE_ENV=qa npx shadow-cljs release dev-ui && \
+    ETHLANCE_ENV="${BUILD_ENV}" npx shadow-cljs release dev-ui && \
     ./node_modules/less/bin/lessc resources/public/less/main.less resources/public/css/main.css --verbose
 
 # Release deployment
-WORKDIR ${ETHLANCE_SOURCE_ROOT}/server
+WORKDIR "${ETHLANCE_SOURCE_ROOT}/server"
 RUN yarn && npx shadow-cljs release dev-server
 
+ARG BUILD_ENV=qa
+ARG ETHLANCE_ENV=qa
 FROM node:18
 ENV ETHLANCE_SOURCE_ROOT=/deploy
 ENV ETHLANCE_SERVER_ROOT=/deploy/server
 ENV ETHLANCE_ENV=qa
 ENV DEPLOY_TARGET=/deploy
-WORKDIR ${ETHLANCE_SERVER_ROOT}
+ENV UI_CONFIG_PATH="/deploy/resources/config/ui-config-${BUILD_ENV}.edn"
+ENV SERVER_CONFIG_PATH="/deploy/resources/config/server-config-${BUILD_ENV}.edn"
+WORKDIR "${ETHLANCE_SERVER_ROOT}"
 
+COPY --from=build_server /build/ethlance-config/ui-config-${ETHLANCE_ENV}.edn /deploy/resources/config/ui-config-${ETHLANCE_ENV}.edn
+COPY --from=build_server /build/ethlance-config/server-config-${ETHLANCE_ENV}.edn /deploy/resources/config/server-config-${ETHLANCE_ENV}.edn
 COPY --from=build_server /build/ethlance/resources /deploy/resources
 COPY --from=build_server /build/ethlance/server/node_modules /deploy/server/node_modules/
 COPY --from=build_server /build/ethlance/server/package.json /build/ethlance/server/out/ethlance_server* /deploy/server/
 COPY --from=build_server /build/ethlance/resources/public/contracts /deploy/ui/contracts/
 COPY --from=build_server /build/ethlance/ui/resources/public/* /deploy/ui/
+RUN echo -n "{:last-processed-block 9622279}" > ethlance-events.log
 CMD [ "node", "ethlance_server.js" ]

docker-builds/ui/Dockerfile

@@ -2,11 +2,16 @@ FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:
 ARG BUILD_ENV=qa
 USER root
 
+ENV ETHLANCE_COMPONENT="ui"
 ENV ETHLANCE_SOURCE_ROOT=/build/ethlance
 ENV ETHLANCE_SERVER_ROOT=/build/ethlance/server
 ENV ETHLANCE_ENV=$BUILD_ENV
 ENV DEPLOY_TARGET=/deploy/
 
+ENV SMART_CONTRACTS_PATH="/build/ethlance/shared/src/ethlance/shared/smart_contracts_${BUILD_ENV}.cljs"
+ENV SMART_CONTRACTS_BUILD_PATH="/build/ethlance/resources/public/contracts/build"
+ENV ETHLANCE_CONFIG_PATH="/build/ethlance/config/${ETHLANCE_COMPONENT}-config-${BUILD_ENV}.edn"
+
 WORKDIR /build/ethlance
 COPY . .
 
@@ -15,14 +20,18 @@ RUN cat shared/src/ethlance/shared/smart_contracts_${ETHLANCE_ENV}.cljs
 WORKDIR /build/ethlance-config
 COPY config .
 
-WORKDIR /build
-RUN git clone https://github.com/district0x/d0x-libs
+# WORKDIR /build
+# RUN git clone https://github.com/district0x/d0x-libs
 
 
 FROM init as build_stage
+ENV ETHLANCE_ENV=qa
+WORKDIR $ETHLANCE_SOURCE_ROOT
+RUN yarn install && ETHLANCE_ENV="${BUILD_ENV}" npx truffle compile
+
 WORKDIR $ETHLANCE_SOURCE_ROOT/ui
-RUN yarn && \
-    ETHLANCE_ENV=qa npx shadow-cljs release dev-ui && \
+RUN yarn install && \
+    ETHLANCE_ENV="${BUILD_ENV}" npx shadow-cljs release dev-ui && \
     ./node_modules/less/bin/lessc resources/public/less/main.less resources/public/css/main.css --verbose
 
 FROM nginx:alpine

package.json

@@ -6,17 +6,18 @@
     "deps": "yarn install --frozen-lockfile"
   },
   "dependencies": {
-    "@openzeppelin/contracts": "4.1.0",
     "@ganache/console.log": "0.4.0",
+    "@openzeppelin/contracts": "4.1.0",
     "@truffle/hdwallet-provider": "^1.3.1"
   },
   "devDependencies": {
     "jsedn": "0.4.1",
+    "solc": "0.8.21",
     "truffle": "5.11.5",
-    "solc": "0.8.21"
+    "truffle-plugin-verify": "^0.6.7"
   },
   "resolutions": {
-    "solc": "0.8.21"
+    "solc": "0.8.19"
   },
   "engines": {
     "node": "^18.17.0"