PoP stats service

.github/CODEOWNERS

@@ -38,3 +38,6 @@
 
 # GoReleaser config
 .goreleaser.yaml @hemilabs/dev @joshuasing
+
+# PoP payouts reporting service
+/popstats/ @hemilabs/dev @gabmontes

popstats/.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+redis-data

popstats/.gitignore

@@ -0,0 +1,3 @@
+.env
+node_modules
+redis-data

popstats/.nvmrc

@@ -0,0 +1 @@
+22

popstats/Dockerfile

@@ -0,0 +1,38 @@
+# Copyright (c) 2024 Hemi Labs, Inc.
+# Use of this source code is governed by the MIT License,
+# which can be found in the LICENSE file.
+
+FROM node:22-alpine@sha256:4162c8a0f1fef9d3b003eb1fd3d8a26db46815288832aa453d829f4129d4dfd3
+
+# Build metadata
+ARG VERSION
+ARG VCS_REF
+ARG BUILD_DATE
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+  org.opencontainers.image.authors="Hemi Labs" \
+  org.opencontainers.image.url="https://github.com/hemilabs/heminetwork" \
+  org.opencontainers.image.source="https://github.com/hemilabs/heminetwork" \
+  org.opencontainers.image.version=$VERSION \
+  org.opencontainers.image.revision=$VCS_REF \
+  org.opencontainers.image.vendor="Hemi Labs" \
+  org.opencontainers.image.licenses="MIT" \
+  org.opencontainers.image.title="PoP stats" \
+  org.label-schema.build-date=$BUILD_DATE \
+  org.label-schema.name="PoP stats" \
+  org.label-schema.url="https://github.com/hemilabs/heminetwork" \
+  org.label-schema.vcs-url="https://github.com/hemilabs/heminetwork" \
+  org.label-schema.vcs-ref=$VCS_REF \
+  org.label-schema.vendor="Hemi Labs" \
+  org.label-schema.version=$VERSION \
+  org.label-schema.schema-version="1.0"
+
+WORKDIR /home/node/app
+
+COPY package.json pnpm-lock.yaml .
+RUN corepack enable
+RUN pnpm install --prod
+COPY . .
+
+USER node
+
+CMD [ "node", "server.js"]

popstats/README.md

@@ -0,0 +1,55 @@
+# PoP stats
+
+This is a service that identifies all the PoP payout transactions, summarizes, calculates the incentive program points,
+and reports those to Absinthe.
+
+## How it works
+
+This service is prepared to be run once a day as a cron job.
+On each run, it gets all the HEMI token mint operations (transfers coming from the null address), filters those
+originated by the depositor address (0x8888...8888), summarizes the events by recipient address, does some data
+formatting and sends it to Absinthe.
+
+In order to walk through the history more efficiently, the service processes the blocks in chunks of 7200, which is
+roughly 1 day's worth of Hemi Network blocks.
+Then it reports the results before taking care of the next chunk.
+The intended side-effect is to report points once per day, even during the initial run where many days need to be
+reported.
+
+In addition, within each chunk, the network is queried for those transfer events in sub-chunks of 1200 blocks (4 hours)
+to minimize the impact on the RPC nodes. Additional rate limits were set with the same goal.
+
+The state of the process is stored in a single key in a Redis database so the service knows where to start at the
+beginning of each run.
+That state is the number of the last blocks processed and reported to Absinthe.
+
+The data sent to Absinthe includes the address receiving the PoP payouts, the amount of points granted and some metadata
+just intended for tracking/auditing purposes.
+
+## How to run locally
+
+Set the following environment variables in a `.env` file:
+
+- ABSINTHE_API_URL and ABSINTHE_API_KEY: URL of the Absinthe GraphQL API and authentication JWT
+- REDIS_URL: Full URI of the Redis database
+
+Optionally, start a Redis instance and use `redis://host.docker.internal:6379` as the Redis URL:
+
+```sh
+docker run -d -p 6379:6379 -v ./redis-data:/data --name pop-stats-redis --rm redis:7-alpine redis-server --save 60 1
+```
+
+Finally, build the Docker image and start a container:
+
+```sh
+docker build -t hemilabs/pop-stats:latest .
+docker run -it --env-file .env --name pop-stats --rm hemilabs/pop-stats:latest
+```
+
+The process will then start identifying and reporting all PoP payouts, then will exit.
+
+Redis can be easily stopped by running:
+
+```sh
+docker stop $(docker ps -aqf \"name=pop-stats-redis\")
+```

popstats/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "hemi-pop-stats",
+  "version": "1.0.0",
+  "description": "Report PoP payouts to Absinthe",
+  "license": "MIT",
+  "author": {
+    "name": "Gabriel Montes"
+  },
+  "dependencies": {
+    "before-exit": "1.0.0",
+    "hemi-viem": "1.1.0",
+    "lodash": "^4.17.21",
+    "p-filter": "^4.1.0",
+    "p-limit": "^6.1.0",
+    "p-throttle": "^6.2.0",
+    "promise-mem": "^1.0.2",
+    "redis": "4.6.15",
+    "viem": "2.17.3"
+  },
+  "devDependencies": {
+    "eslint": "^8.57.0",
+    "eslint-config-bloq": "^4.1.0"
+  },
+  "engines": {
+    "node": ">=12.0.0"
+  },
+  "private": true,
+  "packageManager": "[email protected]+sha512.140036830124618d624a2187b50d04289d5a087f326c9edfc0ccd733d76c4f52c3a313d4fc148794a2a9d81553016004e6742e8cf850670268a7387fc220c903",
+  "type": "module",
+  "eslintConfig": {
+    "extends": [
+      "bloq",
+      "bloq/node"
+    ],
+    "root": true,
+    "rules": {
+      "no-console": "off",
+      "no-process-exit": "off",
+      "quotes": "off"
+    }
+  },
+  "prettier": {}
+}