From cb6c8d9cde216e466854d7624f81826a23a8a090 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Wed, 21 Feb 2024 07:40:51 -0500 Subject: [PATCH] Saves docker --- docker/analysis/build/Dockerfile | 20 +++ docker/analysis/build/analysis.entrypoint.sh | 58 ++++++++ docker/cron_tasks/15min/update_webistes.sh | 30 ++++ docker/docker-compose.yml | 133 ++++++++++++++++++ docker/env.example | 20 +++ docker/lighttpd/lighttpd.conf | 34 +++++ docker/lighttpd/mime-types.conf | 77 ++++++++++ docker/tokenomics/build/Dockerfile | 37 +++++ .../tokenomics/build/tokenomics.entrypoint.sh | 76 ++++++++++ docker/tokenomics/build/trueBlocks.toml | 13 ++ 10 files changed, 498 insertions(+) create mode 100644 docker/analysis/build/Dockerfile create mode 100644 docker/analysis/build/analysis.entrypoint.sh create mode 100644 docker/cron_tasks/15min/update_webistes.sh create mode 100644 docker/docker-compose.yml create mode 100644 docker/env.example create mode 100644 docker/lighttpd/lighttpd.conf create mode 100644 docker/lighttpd/mime-types.conf create mode 100644 docker/tokenomics/build/Dockerfile create mode 100644 docker/tokenomics/build/tokenomics.entrypoint.sh create mode 100644 docker/tokenomics/build/trueBlocks.toml diff --git a/docker/analysis/build/Dockerfile b/docker/analysis/build/Dockerfile new file mode 100644 index 0000000..327a903 --- /dev/null +++ b/docker/analysis/build/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.10.5-slim +ENV DEBIAN_FRONTEND=noninteractive + +# Install Python and copy dependencies +ENV PYTHONUNBUFFERED=1 +ENV PATH="/venv/bin:$PATH" + +RUN apt update && apt install -y parallel \ + && python3 -m ensurepip \ + && python -m venv /venv \ + && . /venv/bin/activate + +# Copy files from builder +COPY --from=tokenomics:latest /root/tokenomics.io/scripts/networks /app + +RUN pip install -r /app/requirements.fil +ENV PATH="/venv/bin:$PATH" + +COPY analysis.entrypoint.sh . +ENTRYPOINT bash ./analysis.entrypoint.sh \ No newline at end of file diff --git a/docker/analysis/build/analysis.entrypoint.sh b/docker/analysis/build/analysis.entrypoint.sh new file mode 100644 index 0000000..71a8eac --- /dev/null +++ b/docker/analysis/build/analysis.entrypoint.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e + +if [ -z "$WEBSITES" ] +then + echo "This script requires WEBSITES env variable to be set as a comma-separated list" + exit 1 +fi + +update_per_file_data() { + FOLDER=$1 + CHAIN=$2 + + EXPORTS_DIR="/exports/$FOLDER/$CHAIN" + + # Turn filenames into addresses + find $EXPORTS_DIR/statements -type f -name *.csv | xargs basename -a -s .csv > /tmp/addresses + + mkdir -p ${EXPORTS_DIR}/neighbors/networks + cd ${EXPORTS_DIR}/neighbors/networks + cat /tmp/addresses | parallel --jobs 200% "python /app/neighbor_networks.py" +} + +update_project() { + FOLDER=$1 + CHAINS=${2:-mainnet} + FMT=${3:-csv} + + for CHAIN in ${CHAINS//,/ } + do + echo "Folder: " $FOLDER + echo "Chain: " $CHAIN + + update_per_file_data $FOLDER $CHAIN + done +} + +start() { + for WEBSITE in ${WEBSITES//,/ } + do + echo "Building neighbor network for ${WEBSITE}" + WEBSITE_UP=`echo $WEBSITE | tr '[:lower:]' '[:upper:]'` + varname="NOMICS_${WEBSITE_UP}_CHAINS" + CHAINS="${!varname}" + + update_project $WEBSITE $CHAINS + done +} + +echo "Images will be built in 20 minutes" +sleep 1200 + +while true +do + start + echo "Images built. Will wait 50 minutes" + sleep 3000 +done \ No newline at end of file diff --git a/docker/cron_tasks/15min/update_webistes.sh b/docker/cron_tasks/15min/update_webistes.sh new file mode 100644 index 0000000..c4317ed --- /dev/null +++ b/docker/cron_tasks/15min/update_webistes.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ -z "$WEBSITES" ] +then + echo "This script requires WEBSITES env variable to be set as a comma-separated list" + exit 1 +fi + +if [ -z "$HTML_DIR" ] +then + echo "This script requires HTML_DIR env variable to be set" + exit 1 +fi + +if [ -z "$NOMICS_DIR" ] +then + echo "This script requires NOMICS_DIR env variable to be set" + exit 1 +fi + +bash $NOMICS_DIR/scripts/update.sh +STATUS=$? + +if [ $STATUS -gt 0 ] +then + echo "Error while updating the data, exiting" + exit 1 +fi + +bash $NOMICS_DIR/scripts/build.sh $WEBSITES $HTML_DIR \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..af7831f --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,133 @@ +name: Tokenomics +x-logging: &default-logging + options: + max-size: "10m" + max-file: "3" +services: + core: + image: trueblocks/core:develop + logging: *default-logging + ports: + - "8080:8080" + env_file: .env + volumes: + - Index:/index + - Cache:/cache + + monitor_giveth_mainnet: + depends_on: + - core + restart: unless-stopped + image: trueblocks/monitors-watch:develop + logging: *default-logging + env_file: .env + environment: + MONITORS_WATCH_FILE: |- + --appearances + --articulate --cache --cache_traces + --logs --relevant + --neighbors + --statements + volumes: + - Index:/index + - Cache:/cache + - giveth_exports:/exports + - ../giveth:/addresses:ro + + monitor_giveth_gnosis: + depends_on: + - core + restart: unless-stopped + image: trueblocks/monitors-watch:develop + logging: *default-logging + env_file: .env + environment: + MONITORS_WATCH_ARGS: --chain gnosis + MONITORS_WATCH_FILE: |- + --appearances + --articulate --cache --cache_traces + --logs --relevant + --neighbors + --statements + volumes: + - Index:/index + - Cache:/cache + - giveth_exports:/exports + - ../giveth:/addresses:ro + + monitor_gitcoin: + depends_on: + - core + restart: unless-stopped + image: trueblocks/monitors-watch:develop + logging: *default-logging + env_file: .env + environment: + MONITORS_WATCH_FILE: |- + --appearances + --articulate --cache --cache_traces + --logs --relevant --emitter 0xdf869fad6db91f437b59f1edefab319493d4c4ce --emitter 0x7d655c57f71464b6f83811c55d84009cd9f5221c --emitter 0xf2354570be2fb420832fb7ff6ff0ae0df80cf2c6 --emitter 0x3342e3737732d879743f2682a3953a730ae4f47c --emitter 0x3ebaffe01513164e638480404c651e885cca0aa4 + --neighbors + --statements + volumes: + - Index:/index + - Cache:/cache + - gitcoin_exports:/exports + - ../gitcoin:/addresses:ro + + tokenomics: + depends_on: + - core + - monitor_giveth_mainnet + - monitor_giveth_gnosis + - monitor_gitcoin + image: tokenomics:latest + logging: *default-logging + build: + context: ./tokenomics/build + args: + UPSTREAM_VER: docker + env_file: .env + deploy: + restart_policy: + condition: on-failure + volumes: + - giveth_exports:/root/tokenomics.io/giveth/exports + - gitcoin_exports:/root/tokenomics.io/gitcoin/exports + - Output:/html + + analysis: + depends_on: + - core + - monitor_gitcoin + image: analysis:latest + logging: *default-logging + build: ./analysis/build + env_file: .env + deploy: + restart_policy: + condition: on-failure + volumes: + - giveth_exports:/exports/giveth + - gitcoin_exports:/exports/gitcoin + - Output:/html + + lighttpd: + image: sebp/lighttpd + logging: *default-logging + volumes: + - Output:/var/www/localhost/htdocs:ro + - ./lighttpd:/etc/lighttpd + ports: + - "8090:80" + tty: true + + +volumes: + Index: + Cache: + Output: + giveth_exports: + giveth_addresses: + gitcoin_exports: + gitcoin_addresses: diff --git a/docker/env.example b/docker/env.example new file mode 100644 index 0000000..675d35a --- /dev/null +++ b/docker/env.example @@ -0,0 +1,20 @@ +# No need to change these: +HTML_DIR=/html +CHIFRA_URL=core:8080 +MONITORS_WATCH_ARGS=--fmt csv + +# You may want to change these: + +# How long to wait between data updates +NOMICS_SLEEP=1800 # 30 mins + +WEBSITES=gitcoin +NOMICS_GITCOIN_CHAIN=mainnet + +# WEBSITES=gitcoin,giveth +# NOMICS_GIVETH_CHAIN=mainnet,gnosis +# NOMICS_GITCOIN_CHAIN=mainnet + +TB_SETTINGS_ETHERSCANKEY=your-key + +TB_CHAINS_MAINNET_RPCPROVIDER=http://erigon.dappnode:8545 diff --git a/docker/lighttpd/lighttpd.conf b/docker/lighttpd/lighttpd.conf new file mode 100644 index 0000000..a5281ef --- /dev/null +++ b/docker/lighttpd/lighttpd.conf @@ -0,0 +1,34 @@ +# Copied from https://github.com/spujadas/lighttpd-docker/blob/master/etc/lighttpd/lighttpd.conf + +var.basedir = "/var/www/localhost" +var.logdir = "/var/log/lighttpd" +var.statedir = "/var/lib/lighttpd" + +server.modules = ( + "mod_rewrite", + "mod_access", + "mod_accesslog" +) + +include "mime-types.conf" + +server.username = "lighttpd" +server.groupname = "lighttpd" + +server.document-root = var.basedir + "/htdocs" +server.pid-file = "/run/lighttpd.pid" + +server.errorlog = "/dev/pts/0" +server.indexfiles = ("index.html") + +server.follow-symlink = "enable" +static-file.exclude-extensions = (".php", ".pl", ".cgi", ".fcgi") +accesslog.filename = "/dev/pts/0" +url.access-deny = ("~", ".inc") + +url.rewrite-once = ( + "^/([a-z_-]+)/?$" => "/index.html" +) + +# Allow listing */data/ dir +$HTTP["url"] =~ "[a-z_-]+/data($|/)" { server.dir-listing = "enable" } \ No newline at end of file diff --git a/docker/lighttpd/mime-types.conf b/docker/lighttpd/mime-types.conf new file mode 100644 index 0000000..0b75bd4 --- /dev/null +++ b/docker/lighttpd/mime-types.conf @@ -0,0 +1,77 @@ +############################################################################### +# Default mime-types.conf for Gentoo. +# include'd from lighttpd.conf. +# $Header: /var/cvsroot/gentoo-x86/www-servers/lighttpd/files/conf/mime-types.conf,v 1.4 2010/03/14 21:45:18 bangert Exp $ +############################################################################### + +# {{{ mime types +mimetype.assign = ( + ".svg" => "image/svg+xml", + ".svgz" => "image/svg+xml", + ".pdf" => "application/pdf", + ".sig" => "application/pgp-signature", + ".spl" => "application/futuresplash", + ".class" => "application/octet-stream", + ".ps" => "application/postscript", + ".torrent" => "application/x-bittorrent", + ".dvi" => "application/x-dvi", + ".gz" => "application/x-gzip", + ".pac" => "application/x-ns-proxy-autoconfig", + ".swf" => "application/x-shockwave-flash", + ".tar.gz" => "application/x-tgz", + ".tgz" => "application/x-tgz", + ".tar" => "application/x-tar", + ".zip" => "application/zip", + ".dmg" => "application/x-apple-diskimage", + ".mp3" => "audio/mpeg", + ".m3u" => "audio/x-mpegurl", + ".wma" => "audio/x-ms-wma", + ".wax" => "audio/x-ms-wax", + ".ogg" => "application/ogg", + ".wav" => "audio/x-wav", + ".gif" => "image/gif", + ".jpg" => "image/jpeg", + ".jpeg" => "image/jpeg", + ".png" => "image/png", + ".xbm" => "image/x-xbitmap", + ".xpm" => "image/x-xpixmap", + ".xwd" => "image/x-xwindowdump", + ".css" => "text/css", + ".html" => "text/html", + ".htm" => "text/html", + ".js" => "text/javascript", + ".asc" => "text/plain", + ".c" => "text/plain", + ".h" => "text/plain", + ".cc" => "text/plain", + ".cpp" => "text/plain", + ".hh" => "text/plain", + ".hpp" => "text/plain", + ".conf" => "text/plain", + ".log" => "text/plain", + ".text" => "text/plain", + ".txt" => "text/plain", + ".diff" => "text/plain", + ".patch" => "text/plain", + ".ebuild" => "text/plain", + ".eclass" => "text/plain", + ".rtf" => "application/rtf", + ".bmp" => "image/bmp", + ".tif" => "image/tiff", + ".tiff" => "image/tiff", + ".ico" => "image/x-icon", + ".dtd" => "text/xml", + ".xml" => "text/xml", + ".mpeg" => "video/mpeg", + ".mpg" => "video/mpeg", + ".mov" => "video/quicktime", + ".qt" => "video/quicktime", + ".avi" => "video/x-msvideo", + ".asf" => "video/x-ms-asf", + ".asx" => "video/x-ms-asf", + ".wmv" => "video/x-ms-wmv", + ".bz2" => "application/x-bzip", + ".tbz" => "application/x-bzip-compressed-tar", + ".tar.bz2" => "application/x-bzip-compressed-tar" + ) +# }}} \ No newline at end of file diff --git a/docker/tokenomics/build/Dockerfile b/docker/tokenomics/build/Dockerfile new file mode 100644 index 0000000..63ac7f2 --- /dev/null +++ b/docker/tokenomics/build/Dockerfile @@ -0,0 +1,37 @@ +# Latest golang as builder +FROM golang:1.18-alpine as builder + +# Install build depenedencies +RUN apk --no-cache add make git gcc linux-headers libc-dev + +# Set workdir +WORKDIR /root + +# Try to get upstream version (default master) +ARG UPSTREAM_VER=master + +# Clone and build tokenomics tool +RUN git clone -b "${UPSTREAM_VER}" --single-branch --progress --depth 1 \ + https://github.com/TrueBlocks/tokenomics.io.git && \ + cd tokenomics.io && make + +# +# Switch to alpine container +FROM alpine:3.16 + +# Install binary dependencies and nice to haves +RUN apk --no-cache add gzip bash curl jq yarn nano + +# Copy files from builder +COPY --from=builder /root/tokenomics.io/nomics /usr/local/bin +COPY --from=builder /root/tokenomics.io /root/tokenomics.io + +# Copy config file +COPY trueBlocks.toml /root/.local/share/trueblocks/ + +WORKDIR /root/tokenomics.io + +# Run entrypoint script +# Default sleep time is 30 mins (= 1800 seconds) +ENV NOMICS_SLEEP=1800 +ENTRYPOINT bash docker/tokenomics/build/tokenomics.entrypoint.sh diff --git a/docker/tokenomics/build/tokenomics.entrypoint.sh b/docker/tokenomics/build/tokenomics.entrypoint.sh new file mode 100644 index 0000000..ffa06e9 --- /dev/null +++ b/docker/tokenomics/build/tokenomics.entrypoint.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +export NOMICS_DIR=/root/tokenomics.io + +echo "Updating websites" + +# bash /etc/periodic/15min/update_webistes.sh + +# if [ $? -gt 0 ] +# then +# echo "Error while updating websites, exiting" +# exit 1 +# fi + +# echo "Done. Cron will be updating the site every 15 minutes" +# # This runs forever +# tail -f /dev/null + +if [ -z "$WEBSITES" ] +then + echo "This script requires WEBSITES env variable to be set as a comma-separated list" + exit 1 +fi + +if [ -z "$HTML_DIR" ] +then + echo "This script requires HTML_DIR env variable to be set" + exit 1 +fi + +if [ -z "$NOMICS_DIR" ] +then + echo "This script requires NOMICS_DIR env variable to be set" + exit 1 +fi + +start() { + bash $NOMICS_DIR/scripts/update.sh + STATUS=$? + + if [ $STATUS -gt 0 ] + then + echo "Error while updating the data" + exit 1 + fi +} + +build_website() { + # If the website has already been built, we don't need to do anything + if [ -f /html/index.html ] + then + return + fi + + echo "Building website" + + cd $NOMICS_DIR/ui + yarn + yarn build + + # Do not copy development data mocks + echo "Copying over the artifacts..." + cp -v `find build/ -type f` /html + cp -rv build/static /html + cd - + + echo "Website built" +} + +build_website + +while true +do + start + sleep $NOMICS_SLEEP +done diff --git a/docker/tokenomics/build/trueBlocks.toml b/docker/tokenomics/build/trueBlocks.toml new file mode 100644 index 0000000..7c1b079 --- /dev/null +++ b/docker/tokenomics/build/trueBlocks.toml @@ -0,0 +1,13 @@ +[chains] + +[chains.mainnet] +chainId = "1" +pinGateway = "https://ipfs.unchainedindex.io/ipfs/" +rpcProvider = "http://erigon.dappnode:8545" +symbol = "ETH" + +[chains.gnosis] +chainId = "100" +pinGateway = "https://gnosis.unchainedindex.io/ipfs/" +rpcProvider = "https://rpc.gnosischain.com" +symbol = "XDAI"