v0.4.0 (#183)

v0.4.0 Release

.env.dist.compose

@@ -7,11 +7,11 @@
 
 # odtp db instance in the mongo db: "odtp"
 ODTP_MONGO_DB=
-# s3 bucket name: "odtp" 
+# s3 bucket name: "odtp"
 ODTP_BUCKET_NAME=
 
 # s3 access and secret key
-ODTP_ACCESS_KEY=     
+ODTP_ACCESS_KEY=
 ODTP_SECRET_KEY=
 
 # your github token
@@ -34,9 +34,11 @@ MONGODB_PATH=
 ODTP_DASHBOARD_PORT=
 ODTP_DASHBOARD_JSON_EDITOR=
 
-# Log level
-ODTP_LOG_LEVEL=
-RUN_LOG_LEVEL=
+# Log Level General
+ODTP_LOG_LEVEL=ERROR
+
+# Log Level when running executions
+RUN_LOG_LEVEL=INFO
 
 # App Path
 APP_PATH=

.env.dist.compose.dev

@@ -0,0 +1,48 @@
+# environment variables for installation with docker compose
+# with the dev version: compose.dev.yml
+# -----------------------------------------------------------
+# fill these variables in case you want to install odtp with
+# docker compose
+
+# local setup and compose
+
+# odtp db instance in the mongo db: "odtp"
+ODTP_MONGO_DB=
+# s3 bucket name: "odtp"
+ODTP_BUCKET_NAME=
+
+# s3 access and secret key
+ODTP_ACCESS_KEY=
+ODTP_SECRET_KEY=
+
+# your github token
+GITHUB_TOKEN=
+
+# mongodb user and password
+MONGO_DB_USER=
+MONGO_DB_PASSWORD=
+
+# mongoexpress user and password
+MONGO_EXPRESS_USER=
+MONGO_EXPRESS_PASSWORD=
+
+# absolute path for docker volumes
+ODTP_PATH=
+MINIO_PATH=
+MONGODB_PATH=
+
+# Dashboard parameters
+ODTP_DASHBOARD_PORT=
+ODTP_DASHBOARD_JSON_EDITOR=
+
+# Log Level General
+ODTP_LOG_LEVEL=ERROR
+
+# Log Level when running executions
+RUN_LOG_LEVEL=INFO
+
+# App Path
+APP_PATH=
+
+# Install the package in editable mode.
+PIP_INSTALL_ARGS="--editable"

.env.dist.local

@@ -29,6 +29,8 @@ ODTP_DASHBOARD_JSON_EDITOR=
 # Working directory for user projects
 ODTP_PATH=
 
-# Log level
-ODTP_LOG_LEVEL=
-RUN_LOG_LEVEL=
+# Log Level General
+ODTP_LOG_LEVEL=ERROR
+
+# Log Level when running executions
+RUN_LOG_LEVEL=INFO

.gitignore

@@ -1,6 +1,7 @@
 __pycache__/
 .DS_Store
 .env*
-!.env*.dist
+!.env.dist.*
 .nicegui
 .local
+odtp/odtp.log

CHANGELOG.md

@@ -1,5 +1,23 @@
 ## Changelog
 
+- v0.4.0
+    - gui: the general workflow for the user was improved (new Items are selected automatically for example)
+    - gui: validation has been implemented in the forms to make sure data that is entered will be valid
+    - gui: the run page for executions has been improved in the gui: logs are now also available from the gui when a workflow runs
+    - gui: on the dashboard homepage you can now directly see whether all services including github are connected
+    - gui: the workarea in the dashboard was removed and replace by an info section on top of the pages
+    - logging: improve logging at GUI/CLI: run commands are now logged to a file, while everything else is still logged to the command line: this is so run commands can be easier debugged by knowing what was run in docker and how that run was triggered 
+    - logging: the logs of the execution runs are now exposed as volumes, so that will be directly available when the component starts and will remain in the project path even after a component exited prematurely
+    - execution: execution and step timestamps
+    - cli: printing table from mongodb colleciton
+    - cli: delete methods for execution and related items. 
+    - components: avoid rebuilding image if component is available
+    - compose: a dev version for docker compose has been added to facilitate development
+
+- v0.3.1 
+    - hotfixes for bugs
+    - removal of unused dependencies and adition of pandas
+
 - v0.3.0 dashboard refactoring
     - dashboard was refactored since code files were getting too long
     - homepage of the dashboard got an upgrade and also contains now connection checks

compose.dev.yml

@@ -28,7 +28,7 @@ services:
       ODTP_DASHBOARD_RELOAD: ${ODTP_DASHBOARD_RELOAD}
       ODTP_DASHBOARD_JSON_EDITOR: ${ODTP_DASHBOARD_JSON_EDITOR}
       ODTP_LOG_LEVEL: ${ODTP_LOG_LEVEL}
-      RUN_LOG_LEVEL: ${ODTP_LOG_LEVEL}
+      RUN_LOG_LEVEL: ${RUN_LOG_LEVEL}
 
     stdin_open: true
     tty: true

compose.yml

@@ -23,7 +23,7 @@ services:
       ODTP_DASHBOARD_PORT: ${ODTP_DASHBOARD_PORT}
       ODTP_DASHBOARD_JSON_EDITOR: ${ODTP_DASHBOARD_JSON_EDITOR}
       ODTP_LOG_LEVEL: ${ODTP_LOG_LEVEL}
-      RUN_LOG_LEVEL: ${ODTP_LOG_LEVEL}
+      RUN_LOG_LEVEL: ${RUN_LOG_LEVEL}
 
     stdin_open: true
     tty: true

odtp/cli/component.py

@@ -3,6 +3,7 @@
 """
 import typer
 from typing_extensions import Annotated
+import logging
 
 from odtp.run import DockerManager
 import odtp.helpers.git as odtp_git
@@ -11,6 +12,9 @@
 
 app = typer.Typer()
 
+log = logging.getLogger(__name__)
+
+
 ## Adding listing so we can have multiple flags
 from typing import List
 
@@ -34,10 +38,10 @@ def prepare(
         )
         componentManager.prepare_component()
     except Exception as e:
-        print(f"ERROR: Prepare component failed: {e}") 
+        log.error(f"ERROR: Prepare component failed: {e}") 
         raise typer.Abort()           
     else:
-        print("SUCCESS: image for the component has been build")
+        log.info("SUCCESS: image for the component has been build")
 
 @app.command()
 def run(
@@ -78,17 +82,12 @@ def run(
             instance_name=instance_name
         )
     except Exception as e:
-        print(f"ERROR: Run of component failed: {e}") 
+        log.error(f"ERROR: Run of component failed: {e}") 
         raise typer.Abort()           
     else:
-        print("SUCCESS: container for the component has been started")
+        log.info("SUCCESS: container for the component has been started")
 
 
-#### TODO: Stop Component
-@app.command()
-def stop():
-    pass
-
 
 @app.command()
 def delete_instance(
@@ -98,7 +97,7 @@ def delete_instance(
 ):
     componentManager = DockerManager()
     componentManager.delete_component(instance_name=instance_name)
-    print("Container deleted")
+    log.info("Container deleted")
 
 
 @app.command()
@@ -109,7 +108,7 @@ def delete_image(
 ):
     componentManager = DockerManager(image_name=image_name)
     componentManager.delete_image()
-    print("Image deleted")
+    log.info("Image deleted")
 
 
 if __name__ == "__main__":

odtp/cli/db.py

@@ -3,16 +3,18 @@
 """
 import typer
 from typing_extensions import Annotated
+import logging 
 
 import odtp.mongodb.db as db
 import odtp.helpers.utils as odtp_utils
 
 app = typer.Typer()
 
+log = logging.getLogger(__name__)
 
 @app.command()
 def get(
-    collection: str = typer.Option(..., "--collection", help="Specify the collection"),
+    collection: str = typer.Argument(...,help="Specify the collection"),
     id: Annotated[str, typer.Option(help="Specify the id")] = None,
 ):
     if id:
@@ -21,6 +23,11 @@ def get(
         db_output = db.get_collection(collection=collection)
     odtp_utils.print_output_as_json(db_output)
 
+@app.command()
+def ls(collection_name: str):
+    """List all documents in a collection"""
+    db_output = db.get_collection(collection=collection_name)
+    odtp_utils.output_as_pretty_table(db_output, collection_name)
 
 @app.command()
 def showAll():
@@ -76,21 +83,21 @@ def delete_document(
     id: str = typer.Option(help="Specify the id")
 ):
     db.delete_document_by_id(document_id=id, collection=collection)
-    print(f"Document with ID {id} was deleted")
+    log.info(f"Document with ID {id} was deleted")
 
 
 @app.command()
 def delete_collection(
     collection: str = typer.Option(..., "--collection", help="Specify the collection"),
 ):
     db.delete_collection(collection=collection)
-    print(f"Collection {collection} was deleted.")
+    log.info(f"Collection {collection} was deleted.")
 
 
 @app.command()
 def deleteAll():
     db.delete_all()
-    print("All collection deleted.")
+    log.info("All collection deleted.")
 
 
 if __name__ == "__main__":

odtp/cli/execution.py

@@ -1,19 +1,26 @@
 """
 This scripts contains odtp subcommands for 'execution'
 """
-
+import sys
 import typer
 from typing_extensions import Annotated
+import logging
 
 import odtp.mongodb.db as db
 import odtp.helpers.parse as odtp_parse
 from odtp.workflow import WorkflowManager
 from directory_tree import display_tree
 import odtp.helpers.environment as odtp_env
+from odtp.storage import s3Manager
 from nicegui import ui
+import os
 
 app = typer.Typer()
 
+log = logging.getLogger(__name__)
+
+log = logging.getLogger(__name__)
+
 ## Adding listing so we can have multiple flags
 from typing import List
 
@@ -46,10 +53,14 @@ def prepare(
         flowManager = WorkflowManager(execution, project_path, secrets)
         flowManager.prepare_workflow()
     except Exception as e:
-        print(f"ERROR: Prepare execution failed: {e}") 
+        msg = f"ERROR: Prepare execution failed: {e}"
+        log.exception(msg) 
+        print(msg)
         raise typer.Abort()           
     else:
-        print("SUCCESS: images for the execution have been build")    
+        msg = "SUCCESS: images for the execution have been build"
+        log.info(msg)
+        print(msg)
 
 
 @app.command()
@@ -84,27 +95,75 @@ def run(
         flowManager = WorkflowManager(execution, project_path, secrets)
         flowManager.run_workflow()
     except Exception as e:
-        print(f"ERROR: Run execution failed: {e}")       
+        msg = f"ERROR: Prepare execution failed: {e}"
+        log.exception(msg)
+        print(msg)     
         raise typer.Abort()
     else:
-        print("SUCCESS: containers for the execution have been run")      
+        msg = "SUCCESS: containers for the execution have been run"
+        log.info(msg)
+        print(msg)
+
+
+@app.command()
+def streamlogs(
+    project_path: str = typer.Option(
+        ..., "--project-path", help="Specify the path for the execution"
+    ),
+    step_nr: str = typer.Option(
+        ..., "--step-nr", help="Specify the step for the execution"
+    ),
+):
+    try:
+        log_file_path = f"{project_path}/{step_nr}_*/odtp-logs/*"
+        os.system(f"tail -f {log_file_path}")     
+    except KeyboardInterrupt:
+        sys.exit()
 
 
 @app.command()
-def output(
+def delete(
+    execution_name: str = typer.Option(
+        None, "--execution-name", help="Specify the name of the execution"
+    ),
     execution_id: str = typer.Option(
-        ..., "--execution-id", help="Specify the ID of the execution"
+        None, "--execution-id", help="Specify the ID of the execution"
     ),
     project_path: str = typer.Option(
-        ..., "--project-path", help="Specify the path for the execution"
+        None, "--project-path", help="Specify the path for the execution"
     ),
-): 
+    keep_project_path: bool = typer.Option(
+        True, "--keep-project-path", help="Keep the project directory after deleting contents"
+    ),
+):
     try:
-        display_tree(project_path)
-    except Exception as e:
-        print(f"ERROR: Output printing failed: {e}")       
-        raise typer.Abort()
+        if execution_id is None and execution_name is None:
+            raise typer.Exit("Please provide either --execution-name or --execution-id")
+
+        if execution_name:
+            execution_id = db.get_document_id_by_field_value("title", execution_name, db.collection_executions)
+
+        # S3
+        s3_keys = db.get_all_outputs_s3_keys(execution_id)
+        s3M = s3Manager()
+        s3M.deletePaths(s3_keys)
 
+        # DB
+        db.delete_execution(execution_id)
+
+        # Folders
+        if project_path:
+            odtp_env.delete_folder(project_path, keep_project_path=keep_project_path) 
+
+    except Exception as e:
+        msg = f"ERROR: Delete execution failed: {e}"
+        log.exception(msg)
+        print(msg)       
+        raise typer.Abort()  
+    else:
+        msg = "SUCCESS: execution has been deleted"
+        log.info(msg)
+        print(msg)
 
 if __name__ == "__main__":
     app()