Saves docker

TrueBlocks · Feb 21, 2024 · cb6c8d9 · cb6c8d9
1 parent b7a5a99
commit cb6c8d9
Show file tree

Hide file tree

Showing 10 changed files with 498 additions and 0 deletions.
diff --git a/docker/analysis/build/Dockerfile b/docker/analysis/build/Dockerfile
@@ -0,0 +1,20 @@
+FROM python:3.10.5-slim
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and copy dependencies
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/venv/bin:$PATH"
+
+RUN apt update && apt install -y parallel \
+    && python3 -m ensurepip \
+    && python -m venv /venv \
+    && . /venv/bin/activate
+
+# Copy files from builder
+COPY --from=tokenomics:latest /root/tokenomics.io/scripts/networks /app
+
+RUN pip install -r /app/requirements.fil
+ENV PATH="/venv/bin:$PATH"
+
+COPY analysis.entrypoint.sh .
+ENTRYPOINT bash ./analysis.entrypoint.sh
diff --git a/docker/analysis/build/analysis.entrypoint.sh b/docker/analysis/build/analysis.entrypoint.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+if [ -z "$WEBSITES" ]
+then
+    echo "This script requires WEBSITES env variable to be set as a comma-separated list"
+    exit 1
+fi
+
+update_per_file_data() {
+    FOLDER=$1
+    CHAIN=$2
+
+    EXPORTS_DIR="/exports/$FOLDER/$CHAIN"
+
+    # Turn filenames into addresses
+    find $EXPORTS_DIR/statements -type f -name *.csv | xargs basename -a -s .csv > /tmp/addresses
+
+    mkdir -p ${EXPORTS_DIR}/neighbors/networks
+    cd ${EXPORTS_DIR}/neighbors/networks
+    cat /tmp/addresses | parallel --jobs 200% "python /app/neighbor_networks.py"
+}
+
+update_project() {
+    FOLDER=$1
+    CHAINS=${2:-mainnet}
+    FMT=${3:-csv}
+
+    for CHAIN in ${CHAINS//,/ }
+    do
+        echo "Folder: " $FOLDER
+        echo "Chain:  " $CHAIN
+
+        update_per_file_data $FOLDER $CHAIN
+    done
+}
+
+start() {
+    for WEBSITE in ${WEBSITES//,/ }
+    do
+        echo "Building neighbor network for ${WEBSITE}"
+        WEBSITE_UP=`echo $WEBSITE | tr '[:lower:]' '[:upper:]'`
+        varname="NOMICS_${WEBSITE_UP}_CHAINS"
+        CHAINS="${!varname}"
+
+        update_project $WEBSITE $CHAINS
+    done
+}
+
+echo "Images will be built in 20 minutes"
+sleep 1200
+
+while true
+do
+    start
+    echo "Images built. Will wait 50 minutes"
+    sleep 3000
+done
diff --git a/docker/cron_tasks/15min/update_webistes.sh b/docker/cron_tasks/15min/update_webistes.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ -z "$WEBSITES" ]
+then
+    echo "This script requires WEBSITES env variable to be set as a comma-separated list"
+    exit 1
+fi
+
+if [ -z "$HTML_DIR" ]
+then
+    echo "This script requires HTML_DIR env variable to be set"
+    exit 1
+fi
+
+if [ -z "$NOMICS_DIR" ]
+then
+    echo "This script requires NOMICS_DIR env variable to be set"
+    exit 1
+fi
+
+bash $NOMICS_DIR/scripts/update.sh
+STATUS=$?
+
+if [ $STATUS -gt 0 ]
+then
+    echo "Error while updating the data, exiting"
+    exit 1
+fi
+
+bash $NOMICS_DIR/scripts/build.sh $WEBSITES $HTML_DIR
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -0,0 +1,133 @@
+name: Tokenomics
+x-logging: &default-logging
+  options:
+    max-size: "10m"
+    max-file: "3"
+services:
+  core:
+    image: trueblocks/core:develop
+    logging: *default-logging
+    ports:
+      - "8080:8080"
+    env_file: .env
+    volumes:
+      - Index:/index
+      - Cache:/cache
+
+  monitor_giveth_mainnet:
+    depends_on:
+      - core
+    restart: unless-stopped
+    image: trueblocks/monitors-watch:develop
+    logging: *default-logging
+    env_file: .env
+    environment:
+      MONITORS_WATCH_FILE: |-
+        --appearances
+        --articulate --cache --cache_traces
+        --logs --relevant
+        --neighbors
+        --statements
+    volumes:
+      - Index:/index
+      - Cache:/cache
+      - giveth_exports:/exports
+      - ../giveth:/addresses:ro
+
+  monitor_giveth_gnosis:
+    depends_on:
+      - core
+    restart: unless-stopped
+    image: trueblocks/monitors-watch:develop
+    logging: *default-logging
+    env_file: .env
+    environment:
+      MONITORS_WATCH_ARGS: --chain gnosis
+      MONITORS_WATCH_FILE: |-
+        --appearances
+        --articulate --cache --cache_traces
+        --logs --relevant
+        --neighbors
+        --statements
+    volumes:
+      - Index:/index
+      - Cache:/cache
+      - giveth_exports:/exports
+      - ../giveth:/addresses:ro
+
+  monitor_gitcoin:
+    depends_on:
+      - core
+    restart: unless-stopped
+    image: trueblocks/monitors-watch:develop
+    logging: *default-logging
+    env_file: .env
+    environment:
+      MONITORS_WATCH_FILE: |-
+        --appearances
+        --articulate --cache --cache_traces
+        --logs --relevant --emitter 0xdf869fad6db91f437b59f1edefab319493d4c4ce --emitter 0x7d655c57f71464b6f83811c55d84009cd9f5221c --emitter 0xf2354570be2fb420832fb7ff6ff0ae0df80cf2c6 --emitter 0x3342e3737732d879743f2682a3953a730ae4f47c --emitter 0x3ebaffe01513164e638480404c651e885cca0aa4
+        --neighbors
+        --statements
+    volumes:
+      - Index:/index
+      - Cache:/cache
+      - gitcoin_exports:/exports
+      - ../gitcoin:/addresses:ro
+
+  tokenomics:
+    depends_on:
+      - core
+      - monitor_giveth_mainnet
+      - monitor_giveth_gnosis
+      - monitor_gitcoin
+    image: tokenomics:latest
+    logging: *default-logging
+    build:
+      context: ./tokenomics/build
+      args:
+        UPSTREAM_VER: docker
+    env_file: .env
+    deploy:
+      restart_policy:
+        condition: on-failure
+    volumes:
+      - giveth_exports:/root/tokenomics.io/giveth/exports
+      - gitcoin_exports:/root/tokenomics.io/gitcoin/exports
+      - Output:/html
+
+  analysis:
+    depends_on:
+      - core
+      - monitor_gitcoin
+    image: analysis:latest
+    logging: *default-logging
+    build: ./analysis/build
+    env_file: .env
+    deploy:
+      restart_policy:
+        condition: on-failure
+    volumes:
+      - giveth_exports:/exports/giveth
+      - gitcoin_exports:/exports/gitcoin
+      - Output:/html
+
+  lighttpd:
+    image: sebp/lighttpd
+    logging: *default-logging
+    volumes:
+      - Output:/var/www/localhost/htdocs:ro
+      - ./lighttpd:/etc/lighttpd
+    ports:
+      - "8090:80"
+    tty: true
+
+
+volumes:
+  Index:
+  Cache:
+  Output:
+  giveth_exports:
+  giveth_addresses:
+  gitcoin_exports:
+  gitcoin_addresses:
diff --git a/docker/env.example b/docker/env.example
@@ -0,0 +1,20 @@
+# No need to change these:
+HTML_DIR=/html
+CHIFRA_URL=core:8080
+MONITORS_WATCH_ARGS=--fmt csv
+
+# You may want to change these:
+
+# How long to wait between data updates
+NOMICS_SLEEP=1800 # 30 mins
+
+WEBSITES=gitcoin
+NOMICS_GITCOIN_CHAIN=mainnet
+
+# WEBSITES=gitcoin,giveth
+# NOMICS_GIVETH_CHAIN=mainnet,gnosis
+# NOMICS_GITCOIN_CHAIN=mainnet
+
+TB_SETTINGS_ETHERSCANKEY=your-key
+
+TB_CHAINS_MAINNET_RPCPROVIDER=http://erigon.dappnode:8545
diff --git a/docker/lighttpd/lighttpd.conf b/docker/lighttpd/lighttpd.conf
@@ -0,0 +1,34 @@
+# Copied from https://github.com/spujadas/lighttpd-docker/blob/master/etc/lighttpd/lighttpd.conf
+
+var.basedir  = "/var/www/localhost"
+var.logdir   = "/var/log/lighttpd"
+var.statedir = "/var/lib/lighttpd"
+
+server.modules = (
+    "mod_rewrite",
+    "mod_access",
+    "mod_accesslog"
+)
+
+include "mime-types.conf"
+
+server.username      = "lighttpd"
+server.groupname     = "lighttpd"
+
+server.document-root = var.basedir + "/htdocs"
+server.pid-file      = "/run/lighttpd.pid"
+
+server.errorlog      = "/dev/pts/0"
+server.indexfiles    = ("index.html")
+
+server.follow-symlink = "enable"
+static-file.exclude-extensions = (".php", ".pl", ".cgi", ".fcgi")
+accesslog.filename   = "/dev/pts/0"
+url.access-deny = ("~", ".inc")
+
+url.rewrite-once = (
+ "^/([a-z_-]+)/?$" => "/index.html"
+)
+
+# Allow listing */data/ dir
+$HTTP["url"] =~ "[a-z_-]+/data($|/)" { server.dir-listing = "enable" }
diff --git a/docker/lighttpd/mime-types.conf b/docker/lighttpd/mime-types.conf
@@ -0,0 +1,77 @@
+###############################################################################
+# Default mime-types.conf for Gentoo.
+# include'd from lighttpd.conf.
+# $Header: /var/cvsroot/gentoo-x86/www-servers/lighttpd/files/conf/mime-types.conf,v 1.4 2010/03/14 21:45:18 bangert Exp $
+###############################################################################
+
+# {{{ mime types
+mimetype.assign             = (
+  ".svg"          =>      "image/svg+xml",
+  ".svgz"         =>      "image/svg+xml",
+  ".pdf"          =>      "application/pdf",
+  ".sig"          =>      "application/pgp-signature",
+  ".spl"          =>      "application/futuresplash",
+  ".class"        =>      "application/octet-stream",
+  ".ps"           =>      "application/postscript",
+  ".torrent"      =>      "application/x-bittorrent",
+  ".dvi"          =>      "application/x-dvi",
+  ".gz"           =>      "application/x-gzip",
+  ".pac"          =>      "application/x-ns-proxy-autoconfig",
+  ".swf"          =>      "application/x-shockwave-flash",
+  ".tar.gz"       =>      "application/x-tgz",
+  ".tgz"          =>      "application/x-tgz",
+  ".tar"          =>      "application/x-tar",
+  ".zip"          =>      "application/zip",
+  ".dmg"          =>      "application/x-apple-diskimage",
+  ".mp3"          =>      "audio/mpeg",
+  ".m3u"          =>      "audio/x-mpegurl",
+  ".wma"          =>      "audio/x-ms-wma",
+  ".wax"          =>      "audio/x-ms-wax",
+  ".ogg"          =>      "application/ogg",
+  ".wav"          =>      "audio/x-wav",
+  ".gif"          =>      "image/gif",
+  ".jpg"          =>      "image/jpeg",
+  ".jpeg"         =>      "image/jpeg",
+  ".png"          =>      "image/png",
+  ".xbm"          =>      "image/x-xbitmap",
+  ".xpm"          =>      "image/x-xpixmap",
+  ".xwd"          =>      "image/x-xwindowdump",
+  ".css"          =>      "text/css",
+  ".html"         =>      "text/html",
+  ".htm"          =>      "text/html",
+  ".js"           =>      "text/javascript",
+  ".asc"          =>      "text/plain",
+  ".c"            =>      "text/plain",
+  ".h"            =>      "text/plain",
+  ".cc"           =>      "text/plain",
+  ".cpp"          =>      "text/plain",
+  ".hh"           =>      "text/plain",
+  ".hpp"          =>      "text/plain",
+  ".conf"         =>      "text/plain",
+  ".log"          =>      "text/plain",
+  ".text"         =>      "text/plain",
+  ".txt"          =>      "text/plain",
+  ".diff"         =>      "text/plain",
+  ".patch"        =>      "text/plain",
+  ".ebuild"       =>      "text/plain",
+  ".eclass"       =>      "text/plain",
+  ".rtf"          =>      "application/rtf",
+  ".bmp"          =>      "image/bmp",
+  ".tif"          =>      "image/tiff",
+  ".tiff"         =>      "image/tiff",
+  ".ico"          =>      "image/x-icon",
+  ".dtd"          =>      "text/xml",
+  ".xml"          =>      "text/xml",
+  ".mpeg"         =>      "video/mpeg",
+  ".mpg"          =>      "video/mpeg",
+  ".mov"          =>      "video/quicktime",
+  ".qt"           =>      "video/quicktime",
+  ".avi"          =>      "video/x-msvideo",
+  ".asf"          =>      "video/x-ms-asf",
+  ".asx"          =>      "video/x-ms-asf",
+  ".wmv"          =>      "video/x-ms-wmv",
+  ".bz2"          =>      "application/x-bzip",
+  ".tbz"          =>      "application/x-bzip-compressed-tar",
+  ".tar.bz2"      =>      "application/x-bzip-compressed-tar"
+ )
+# }}}