First prototype of Application Quality Pipeline

.gitignore

@@ -1 +1,2 @@
-venv/
+venv/
+__pycache__

workflow/README.md

@@ -0,0 +1,39 @@
+# Workflow Module: Code Quality Analysis and Reporting
+
+This module is a Common Workflow Language (CWL) workflow designed to automate code quality analysis for Python repositories. It is used in a Django application and executed using [pycalrissian](https://terradue.github.io/pycalrissian/), which runs the workflow in a Kubernetes environment. The workflow analyzes a Git repository using **pylint** and **flake8**, generates reports, and sends them to a specified database via POST requests.
+
+## Inputs
+
+The workflow requires the following inputs:
+
+- **`repo_url`**: The URL to a Git repository that will be cloned and analyzed.
+- **`server_url`**: The address of the database where the pylint and flake8 reports will be saved.
+- **`pipeline_id`**: A unique identifier used for organizing report files in the database.
+- **`run_id`**: A run-specific identifier to distinguish between different runs within the same pipeline.
+
+## Workflow Overview
+
+1. **Clone Repository**: The specified `repo_url` is cloned.
+2. **Run pylint and flake8**: 
+   - Two sub-workflows run pylint and flake8 on the cloned repository.
+   - Both generate code quality reports.
+3. **Post Reports to Database**: Reports are sent to the `server_url` database using POST requests, with `pipeline_id` and `run_id` used to manage report storage.
+
+## Running the Workflow
+
+This workflow is normally executed within a Django app using **pycalrissian**, which handles the orchestration in a Kubernetes environment. Users do not typically run it directly. The app takes care of passing the inputs (`repo_url`, `server_url`, `pipeline_id`, `run_id`), executing the workflow, and processing the reports.
+
+### Manual Execution (Using cwltool)
+
+If manual execution is needed, you can run the workflow using **[cwltool](https://www.commonwl.org/user_guide/introduction/quick-start.html#installing-a-cwl-runner)** by providing inputs through the command line. However, you must modify the tool files to comment out or remove the `baseCommand` lines due to **cwltool** handling Docker ENTRYPOINT differently than Calrissian.
+
+**Example of manual execution:**
+```bash
+cwltool workflow.cwl \
+--repo_url https://github.com/example.git \
+--server_url http://your-database-url/api/reports \
+--pipeline_id 1234 \
+--run_id run5678
+```
+
+Note: When using **cwltool**, ensure Docker is properly set up on your machine, as the tools rely on Docker containers for execution.

workflow/bandit-workflow.cwl

@@ -0,0 +1,38 @@
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  name:
+    type: string
+    default: bandit
+  pipeline_id:
+    type: string
+  repo_path:
+    type: Directory
+  run_id:
+    type: string
+  server_url:
+    type: string
+
+outputs:
+  bandit_report:
+    type: File
+    outputSource: bandit_step/bandit_report
+
+steps:
+  bandit_step:
+    in:
+      source_directory: repo_path
+    run: tools/bandit-tool.cwl
+    out:
+    - bandit_report
+  save_bandit_step:
+    in:
+      name: name
+      pipeline_id: pipeline_id
+      report: bandit_step/bandit_report
+      run_id: run_id
+      server_url: server_url
+    run: tools/save-tool.cwl
+    out: []

workflow/flake8-workflow.cwl

@@ -0,0 +1,38 @@
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  name:
+    type: string
+    default: flake8
+  pipeline_id:
+    type: string
+  repo_path:
+    type: Directory
+  run_id:
+    type: string
+  server_url:
+    type: string
+
+outputs:
+  flake8_report:
+    type: File
+    outputSource: flake8_step/flake8_report
+
+steps:
+  flake8_step:
+    in:
+      source_directory: repo_path
+    run: tools/flake8-tool.cwl
+    out:
+    - flake8_report
+  save_flake8_step:
+    in:
+      name: name
+      pipeline_id: pipeline_id
+      report: flake8_step/flake8_report
+      run_id: run_id
+      server_url: server_url
+    run: tools/save-tool.cwl
+    out: []

workflow/pylint-workflow.cwl

@@ -0,0 +1,38 @@
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  name:
+    type: string
+    default: pylint
+  pipeline_id:
+    type: string
+  repo_path:
+    type: Directory
+  run_id:
+    type: string
+  server_url:
+    type: string
+
+outputs:
+  pylint_report:
+    type: File
+    outputSource: pylint_step/pylint_report
+
+steps:
+  pylint_step:
+    in:
+      source_directory: repo_path
+    run: tools/pylint-tool.cwl
+    out:
+    - pylint_report
+  save_pylint_step:
+    in:
+      name: name
+      pipeline_id: pipeline_id
+      report: pylint_step/pylint_report
+      run_id: run_id
+      server_url: server_url
+    run: tools/save-tool.cwl
+    out: []

workflow/ruff-workflow.cwl

@@ -0,0 +1,38 @@
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  name:
+    type: string
+    default: ruff
+  pipeline_id:
+    type: string
+  repo_path:
+    type: Directory
+  run_id:
+    type: string
+  server_url:
+    type: string
+
+outputs:
+  ruff_report:
+    type: File
+    outputSource: ruff_step/ruff_report
+
+steps:
+  ruff_step:
+    in:
+      source_directory: repo_path
+    run: tools/ruff-tool.cwl
+    out:
+    - ruff_report
+  save_ruff_step:
+    in:
+      name: name
+      pipeline_id: pipeline_id
+      report: ruff_step/ruff_report
+      run_id: run_id
+      server_url: server_url
+    run: tools/save-tool.cwl
+    out: []

workflow/tools/bandit-tool.cwl

@@ -0,0 +1,32 @@
+#!/usr/bin/env cwltool
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: cytopia/bandit
+  InlineJavascriptRequirement: {}
+
+inputs:
+  source_directory:
+    type: Directory
+    inputBinding:
+      position: 1
+
+outputs:
+  bandit_report:
+    type: File
+    outputBinding:
+      glob: bandit_report.json
+
+baseCommand:
+- bandit
+arguments:
+- -x
+- .git
+- -f
+- json
+- -o
+- bandit_report.json
+- --exit-zero

workflow/tools/clone-tool.cwl

@@ -0,0 +1,29 @@
+#!/usr/bin/env cwltool
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: alpine/git
+  InlineJavascriptRequirement: {}
+
+inputs:
+  branch:
+    type: string
+    default: main
+  repo_url:
+    type: string
+
+outputs:
+  repo_directory:
+    type: Directory
+    outputBinding:
+      glob: $(inputs.repo_url.split('/').pop().replace('.git',''))
+
+baseCommand: git
+arguments:
+- clone
+- $(inputs.repo_url)
+- -b
+- $(inputs.branch)

workflow/tools/flake8-tool.cwl

@@ -0,0 +1,26 @@
+#!/usr/bin/env cwltool
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: eoepca/appquality-flake8-json:v0.1.0
+
+inputs:
+  source_directory:
+    type: Directory
+    inputBinding:
+      position: 1
+
+outputs:
+  flake8_report:
+    type: File
+    outputBinding:
+      glob: flake8_report.json
+
+baseCommand: flake8
+arguments:
+- --format=json
+- --output-file=flake8_report.json
+- --exit-zero

workflow/tools/pylint-tool.cwl

@@ -0,0 +1,28 @@
+#!/usr/bin/env cwltool
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: cytopia/pylint
+  InlineJavascriptRequirement: {}
+
+inputs:
+  source_directory:
+    type: Directory
+    inputBinding:
+      position: 1
+      valueFrom: $(inputs.source_directory.path + "/**/*.py")
+
+outputs:
+  pylint_report:
+    type: File
+    outputBinding:
+      glob: pylint_report.json
+
+baseCommand: pylint
+arguments:
+- --output-format=json
+- --output=pylint_report.json
+- --exit-zero

workflow/tools/ruff-tool.cwl

@@ -0,0 +1,33 @@
+#!/usr/bin/env cwltool
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: ghcr.io/astral-sh/ruff:alpine
+  InlineJavascriptRequirement: {}
+
+inputs:
+  source_directory:
+    type: Directory
+    inputBinding:
+      position: 1
+
+outputs:
+  ruff_report:
+    type: File
+    outputBinding:
+      glob: ruff_report.json
+
+baseCommand:
+- ruff
+- check
+arguments:
+- --exclude
+- .git
+- --output-format
+- json
+- -o
+- ruff_report.json
+- -en

workflow/tools/save-tool.cwl

@@ -0,0 +1,35 @@
+#!/usr/bin/env cwltool
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerPull: curlimages/curl
+  InlineJavascriptRequirement: {}
+
+inputs:
+  name:
+    type: string
+  pipeline_id:
+    type: string
+  report:
+    type: File
+  run_id:
+    type: string
+  server_url:
+    type: string
+
+outputs: []
+
+baseCommand: curl
+arguments:
+- prefix: -X
+  valueFrom: POST
+- prefix: -L
+  valueFrom: |-
+    $('http://' + inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name)
+- prefix: -H
+  valueFrom: Content-Type:application/json
+- prefix: -d
+  valueFrom: $('@' + inputs.report.path)