Remove vendor dependencies and cleanup scripts.

.github/workflows/ci-docker.yml

@@ -32,12 +32,11 @@ jobs:
           # Added platforms to ensure this image will work on Linux and OSx with Mx processors
           platforms: linux/amd64,linux/arm64
           build-args: |
-            CI=true
             NODE_ENV=production
             CHAIN_ID=poktroll
 
       # development mode
-      - name: Build Develop
+      - name: Build Development
         uses: docker/build-push-action@v6
         if: ${{ inputs.env != 'production' }}
         with:
@@ -47,6 +46,5 @@ jobs:
           # Added platforms to ensure this image will work on Linux and OSx with Mx processors
           platforms: linux/amd64,linux/arm64
           build-args: |
-            CI=true
-            NODE_ENV=test
+            NODE_ENV=development
             CHAIN_ID=poktroll

.github/workflows/ci-host.yml

@@ -36,15 +36,6 @@ jobs:
       - name: Install dependencies
         run: yarn
 
-      - name: Vendor Clean & Clean Cache
-        run: |
-          ./scripts/vendor.sh clean
-          ./scripts/vendor.sh clean-cache
-
-      - name: Vendor Setup
-        run: |
-          ./scripts/vendor.sh setup
-
       - name: Format
         run: yarn run format:ci
 

README.md

@@ -18,10 +18,9 @@ To learn more about SubQuery, [see their docs](https://academy.subquery.network)
   - [Explore via postgres](#explore-via-postgres)
 - [Getting Started](#getting-started)
   - [tl;dr local development (if not your first time)](#tldr-local-development-if-not-your-first-time)
-  - [1. Ensure submodules are updated](#1-ensure-submodules-are-updated)
-  - [2. Install dependencies](#2-install-dependencies)
-  - [3. Generate types](#3-generate-types)
-  - [4. Run](#4-run)
+  - [1. Install dependencies](#1-install-dependencies)
+  - [2. Generate types](#2-generate-types)
+  - [3. Run](#3-run)
     - [Localnet ONLY](#localnet-only)
     - [4.1 Debugging, errors running \& building](#41-debugging-errors-running--building)
     - [4.2 Using a pre-built image](#42-using-a-pre-built-image)
@@ -69,27 +68,21 @@ yarn poktroll:proxy:start
 yarn run docker:start:development
 ```
 
-### 1. Ensure submodules are updated
-
-```shell
-git submodule update --init --recursive
-```
-
-### 2. Install dependencies
+### 1. Install dependencies
 
 ```shell
 yarn install
 ```
 
-### 3. Generate types
+### 2. Generate types
 
 Types will need to be regenerated any time the `graphql.schema` is changed.
 
 ```shell
 yarn run codegen
 ```
 
-### 4. Run
+### 3. Run
 
 Dotenv files will be automatically created after the `yarn install` thanks to the `postinstall` script.
 After that, feel free to modify them as you wish.

docker/dev-node.dockerfile

@@ -13,34 +13,36 @@ ENV CI=$CI
 # typescript is added here because is wrongly used on some of the workspaces, just by the name
 # without the use of npm exec, yarn exec or any other to ensure they are looking into the node_modules
 RUN apt-get update \
-    && apt-get install -y tree git postgresql-client tini curl jq yq \
+    && apt-get install -y tree git postgresql-client tini curl jq \
     # The following was necessary to install in order to add support for building
     # the docker container on an M1 chip (i.e. ARM64)
     make build-essential pkg-config python3 libusb-1.0-0-dev libudev-dev \
     && npm i -g typescript
 
+# add specific version of yq because depending on the operative system it could get another implementation or version
+# than produce error on shell scripts later.
+RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -o /usr/bin/yq &&\
+            chmod +x /usr/bin/yq
+
 WORKDIR /app
 
 # Copy the minimum required to run install and vendor:setup
 # preventing this step need to be re-build everytime due to change on dev files
 # but if for X reason you update a vendor package this step CACHE will be dropped
 # by docker and fully rebuild
 COPY package.json yarn.lock .yarnrc.yml /app/
-COPY vendor /app/vendor
 COPY scripts /app/scripts
 COPY .yarn /app/.yarn
 
 # Install dev dependencies
 RUN yarn install
 
-## Build forked vendor packages
-RUN ./scripts/vendor.sh clean
-RUN if [ "$CI" = "true" ]; then ./scripts/vendor.sh clean-cache; else echo "Not in CI"; fi
-RUN ./scripts/vendor.sh setup
-
 # Copy everything because we use nodemon on development to build and watch.
 COPY . /app
 
+# Required after this version https://github.com/subquery/subql-cosmos/releases/tag/node-cosmos/4.0.0
+ENV TZ=utc
+
 # Allow execution for every shell script at scripts folder
 RUN find /app/scripts -type f -name "*.sh" -exec chmod +x {} \;
 

docker/node.dockerfile

@@ -1,6 +1,5 @@
 FROM node:22.5-slim AS builder
 
-ARG CI=false
 ARG NODE_ENV=production
 ARG ENDPOINT
 ARG CHAIN_ID=poktroll
@@ -24,18 +23,12 @@ WORKDIR /app
 # but if for X reason you update a vendor package this step CACHE will be dropped
 # by docker and fully rebuild
 COPY package.json yarn.lock .yarnrc.yml /app/
-COPY vendor /app/vendor
 COPY scripts /app/scripts
 COPY .yarn /app/.yarn
 
 # Install dev dependencies
 RUN yarn install
 
-## Build forked vendor packages
-RUN ./scripts/vendor.sh clean
-RUN if [ "$CI" = "true" ]; then ./scripts/vendor.sh clean-cache; else echo "Not in CI"; fi
-RUN ./scripts/vendor.sh setup
-
 # Copy files
 COPY ./project.ts ./schema.graphql ./tsconfig.json ./.eslintrc.js ./.eslintignore /app/
 COPY src /app/src
@@ -53,16 +46,21 @@ RUN addgroup -g 1001 app && adduser -D -h /home/app -u 1001 -G app app
 ARG NODE_ENV=production
 ARG ENDPOINT
 ARG CHAIN_ID=poktroll
-ARG CI=false
 
 ENV NODE_ENV=$NODE_ENV
 ENV ENDPOINT=$ENDPOINT
 ENV CHAIN_ID=$CHAIN_ID
-ENV CI=$CI
 
 # Add system dependencies
 RUN apk update
-RUN apk add git postgresql14-client tini curl jq yq
+RUN apk add git postgresql14-client tini curl jq
+# The following was necessary to install in order to add support for building
+# the docker container on an M1 chip (i.e. ARM64)
+RUN apk add g++ make py3-pip
+# add specific version of yq because depending on the operative system it could get another implementation or version
+# than produce error on shell scripts later.
+RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -o /usr/bin/yq &&\
+        chmod +x /usr/bin/yq
 
 # Switch to user "app"
 WORKDIR /home/app
@@ -73,27 +71,30 @@ COPY .yarn /home/app/.yarn
 COPY scripts /home/app/scripts
 
 # Install dependencies for production
-# Starting from yarn 2.0, the --production flag is indeed deprecated.
-# Yarn 2 introduced improvements to allow more precise installations.
-RUN yarn workspaces focus --production
+# NOTE: in case a yarn install fail, please add --inline-build to produce a more verbose installation logs, that could
+# help you debugging what is going wrong.
+RUN yarn install \
+    # Starting from yarn 2.0, the --production flag is indeed deprecated.
+    # Yarn 2 introduced improvements to allow more precise installations.
+    # The install now is needed because there are some dependencies that need to be build and that is achieve on install \
+    # the with this command the dependencies are reduced to only leave production one.
+    && yarn workspaces focus --production
 
 # Add the dependencies
 COPY --from=builder /app/project.ts /app/schema.graphql /app/tsconfig.json /home/app/
 
 # Include build artefacts in final image
 COPY --from=builder /app/dist /home/app/dist
-COPY --from=builder /app/vendor /home/app/vendor
 COPY --from=builder /app/project.yaml /home/app/
 COPY --from=builder /app/proto /home/app/proto
 COPY --from=builder /app/scripts /home/app/scripts
 
-# TODO_MAINNET(@bryanchriswhite): Add the .gmrc once migrations are available.
-#COPY ./.gmrc /app/.gmrc
-
 # Allow execution for every shell script at scripts folder
 RUN find /home/app/scripts -type f -name "*.sh" -exec chmod +x {} \;
 
 # Set user as app
 USER app
+# Required after this version https://github.com/subquery/subql-cosmos/releases/tag/node-cosmos/4.0.0
+ENV TZ=utc
 
 ENTRYPOINT ["/sbin/tini", "--", "/home/app/scripts/node-entrypoint.sh"]

nodemon.json

@@ -1,24 +1,23 @@
 {
-  "verbose": true,
+  "verbose": false,
   "watch": [
     "src/*",
     "proto/*",
     "project.ts",
     "schema.graphql",
-    "vendor/*",
-    ".env",
-    ".env.develop"
+    ".env.development"
   ],
   "ignore": [
-    "src/types/",
+    "src/types/**/*",
     "project.yaml",
     "dist/*",
     "docs/*",
     "docker/*",
     "node_modules/**/*",
-    "*/node_modules/**/*"
+    "*/node_modules/**/*",
+    ".yarn/**/*"
   ],
   "ext": "ts,proto,graphql",
-  "legacyWatch": false,
+  "legacyWatch": true,
   "delay": 2000
 }

package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "preinstall": "npx only-allow yarn",
     "postinstall": "yarn run env:prepare",
+    "precommit": "./scripts/before-commit.sh",
     "codegen": "yarn exec subql codegen",
     "build": "./scripts/build.sh",
     "watch:build": "./scripts/build.sh no-lint",
@@ -15,12 +16,6 @@
     "format:ci": "yarn exec pretty-quick --verbose --check --pattern 'src/*.{js,ts,jsx,tsx}' --branch origin/main",
     "test": "yarn run build && subql-node-cosmos test",
     "test:ci": "env node vendor/subql-cosmos/packages/node/bin/run test",
-    "vendor:setup": "./scripts/vendor.sh setup",
-    "vendor:clean-cache": "./scripts/vendor.sh clean-cache",
-    "vendor:install": "./scripts/vendor.sh install",
-    "vendor:build": "./scripts/vendor.sh build",
-    "vendor:lint": "./scripts/vendor.sh lint",
-    "vendor:clean": "./scripts/vendor.sh clean",
     "env:prepare": "./scripts/prepare-dotenv.sh",
     "docker:compose": "./scripts/docker-compose.sh",
     "poktroll:proxy:start": "./scripts/proxy-tunnel.sh start",
@@ -62,9 +57,8 @@
   "license": "MIT",
   "devDependencies": {
     "@cosmjs/stargate": "^0.28.9",
-    "@subql/cli": "latest",
-    "@subql/node-cosmos": "latest",
-    "@subql/testing": "latest",
+    "@subql/cli": "5.1.1",
+    "@subql/testing": "2.2.1",
     "@typescript-eslint/eslint-plugin": "^5.10.2",
     "@typescript-eslint/parser": "^5.10.2",
     "dotenv": "latest",
@@ -81,8 +75,9 @@
     "typescript": "^5.2.2"
   },
   "dependencies": {
-    "@subql/types-cosmos": "latest",
-    "@types/node": "^17.0.21",
+    "@subql/node-cosmos": "4.0.1",
+    "@subql/types-cosmos": "3.5.0",
+    "@types/node": "^22.0.0",
     "pino": "^7.8.0",
     "ts-proto": "^1.112.1",
     "tslib": "^2.3.1"

project.ts

@@ -23,7 +23,7 @@ const project: CosmosProject = {
   runner: {
     node: {
       name: "@subql/node-cosmos",
-      version: ">=3.0.0",
+      version: ">=4.0.0",
     },
     query: {
       name: "@subql/query",

scripts/before-commit.sh

@@ -0,0 +1,21 @@
+### Added this script to allow locally build both version for amd64 and arm64 as a way to test it locally what will
+### be done on CI.
+
+### IMPORTANT: you should have `buildx` properly configured on your host.
+### https://github.com/docker/buildx
+
+set -e
+
+. scripts/shared.sh
+
+info_log "Building development for AMD64"
+docker buildx build -f docker/dev-node.dockerfile -t pocket-network/pocketdex:development-amd64 --platform=linux/amd64 .
+
+info_log "Building development for ARM64"
+docker buildx build -f docker/dev-node.dockerfile -t pocket-network/pocketdex:development-arm64 --platform=linux/arm64 .
+
+info_log "Building production for AMD64"
+docker buildx build -f docker/node.dockerfile -t pocket-network/pocketdex:development-amd64 --platform=linux/amd64 .
+
+info_log "Building production for ARM64"
+docker buildx build -f docker/node.dockerfile -t pocket-network/pocketdex:development-arm64 --platform=linux/arm64 .

scripts/node-entrypoint.sh

@@ -17,7 +17,7 @@ then
   info_log "Running Tests only"
   params=$(get_params)
   scripts/build.sh no-lint
-  cmd="$cmd node ./vendor/subql-cosmos/packages/node/bin/run $params $@"
+  cmd="$cmd node ./node_modules/@subql/node-cosmos/bin/run $params $@"
 else
   # Add btree_gist extension to support historical mode - after the db reset from `graphile-migrate reset --erase`
   export PGPASSWORD=$DB_PASS
@@ -40,7 +40,7 @@ EOF
   if [ "$NODE_ENV" = "development" ]
   then
     # call the first command if WATCH is true
-    info_log "NODE_ENV is set to 'development'. Installing nodemon and Running with it..."
+    info_log "NODE_ENV is set to 'development'. Running with Hot-Reload..."
 
     exec="./scripts/watch-exec.sh $@"
     if ! jq --arg value "$exec" '. + {"exec": $value }' /app/nodemon.json > /tmp/temp.json; then
@@ -57,7 +57,7 @@ EOF
     update_project
     params=$(get_params)
     # run the main node
-    cmd="$cmd node ./vendor/subql-cosmos/packages/node/bin/run $params $@"
+    cmd="$cmd node ./node_modules/@subql/node-cosmos/bin/run $params $@"
   fi
 fi
 

scripts/shared.sh

@@ -27,7 +27,7 @@ update_project() {
 
   if [ ! -z "${CHAIN_ID}" ]; then
       info_log "[Config Update] Chain ID: ${CHAIN_ID}"
-      yq -i '.network.chainId = env(CHAIN_ID)' project.yaml
+      yq -i '.network.chainId = strenv(CHAIN_ID)' project.yaml
   fi
 
   if [ ! -z "${ENDPOINT}" ]; then