added upload limit and released new version

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.3.0] - 2024-04-23
+
+### Added
+- limit uploads to 300Mb by default (MAX_SIZE=300)
+
 ## [0.2.0] - 2024-04-23
 
 ### Added

env.example

@@ -5,7 +5,7 @@ SERVER_NAME="criticalmaas.ncsa.illinois.edu"
 
 # system name and version
 SYSTEM_NAME="ncsa"
-SYSTEM_VERSION="0.2.0"
+SYSTEM_VERSION="0.3.0"
 
 # email address for registration with letsencrypt
 TRAEFIK_ACME_EMAIL="[email protected]"

server/Dockerfile

@@ -5,7 +5,7 @@ EXPOSE 8080
 
 ENV PYTHONUNBUFFERED=1 \
     SYSTEM_NAME="ncsa" \
-    SYSTEM_VERSION="0.2.0" \
+    SYSTEM_VERSION="0.3.0" \
     CDR_TOKEN="" \
     CDR_KEEP_EVENT="no" \
     CALLBACK_SECRET="this_is_a_really_silly_secret" \

uploader/Dockerfile

@@ -2,7 +2,8 @@ FROM python:3.11
 
 # environemnt variables
 ENV RABBITMQ_URI="amqp://guest:guest@localhost:5672/%2F" \
-    MODELS="golden_muscat flat_iceberg"
+    MODELS="golden_muscat flat_iceberg" \
+    MAX_SIZE=300
 
 WORKDIR /src
 

uploader/uploader.py

@@ -10,6 +10,7 @@
 rabbitmq_uri = os.getenv("RABBITMQ_URI", "amqp://guest:guest@localhost:5672/%2F")
 cdr_url = "https://api.cdr.land"
 cdr_token = os.getenv("CDR_TOKEN", "")
+max_size = int(os.getenv("MAX_SIZE", "300"))
 
 
 class Worker(threading.Thread):
@@ -24,6 +25,9 @@ def run(self):
             data = json.loads(self.body)
             file = os.path.join("/output", data['cdr_output'])
             logging.debug(f"Uploading data for {data['cog_id']} from {file}")
+            # only upload if less than certain size
+            if os.path.getsize(file) > max_size * 1024 * 1024:  # size in bytes
+                raise ValueError(f"File {file} is larger than {max_size}MB, skipping upload.")
             headers = {'Authorization': f'Bearer {cdr_token}', 'Content-Type': 'application/json'}
             with open(file, 'rb') as f:
                 cdr_data = json.load(f)

version.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION=0.2.0
+VERSION=0.3.0
 
 sed -i~ "s#SYSTEM_VERSION=.*#SYSTEM_VERSION=\"${VERSION}\" \\\\#" server/Dockerfile