diff --git a/.gitignore b/.gitignore index eba74f4..8bc348e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -venv/ \ No newline at end of file +venv/ +__pycache__ \ No newline at end of file diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 0000000..131bdb1 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,39 @@ +# Workflow Module: Code Quality Analysis and Reporting + +This module is a Common Workflow Language (CWL) workflow designed to automate code quality analysis for Python repositories. It is used in a Django application and executed using [pycalrissian](https://terradue.github.io/pycalrissian/), which runs the workflow in a Kubernetes environment. The workflow analyzes a Git repository using **pylint** and **flake8**, generates reports, and sends them to a specified database via POST requests. + +## Inputs + +The workflow requires the following inputs: + +- **`repo_url`**: The URL to a Git repository that will be cloned and analyzed. +- **`server_url`**: The address of the database where the pylint and flake8 reports will be saved. +- **`pipeline_id`**: A unique identifier used for organizing report files in the database. +- **`run_id`**: A run-specific identifier to distinguish between different runs within the same pipeline. + +## Workflow Overview + +1. **Clone Repository**: The specified `repo_url` is cloned. +2. **Run pylint and flake8**: + - Two sub-workflows run pylint and flake8 on the cloned repository. + - Both generate code quality reports. +3. **Post Reports to Database**: Reports are sent to the `server_url` database using POST requests, with `pipeline_id` and `run_id` used to manage report storage. + +## Running the Workflow + +This workflow is normally executed within a Django app using **pycalrissian**, which handles the orchestration in a Kubernetes environment. Users do not typically run it directly. The app takes care of passing the inputs (`repo_url`, `server_url`, `pipeline_id`, `run_id`), executing the workflow, and processing the reports. + +### Manual Execution (Using cwltool) + +If manual execution is needed, you can run the workflow using **[cwltool](https://www.commonwl.org/user_guide/introduction/quick-start.html#installing-a-cwl-runner)** by providing inputs through the command line. However, you must modify the tool files to comment out or remove the `baseCommand` lines due to **cwltool** handling Docker ENTRYPOINT differently than Calrissian. + +**Example of manual execution:** +```bash +cwltool workflow.cwl \ +--repo_url https://github.com/example.git \ +--server_url http://your-database-url/api/reports \ +--pipeline_id 1234 \ +--run_id run5678 +``` + +Note: When using **cwltool**, ensure Docker is properly set up on your machine, as the tools rely on Docker containers for execution. diff --git a/workflow/bandit-workflow.cwl b/workflow/bandit-workflow.cwl new file mode 100644 index 0000000..0abf13a --- /dev/null +++ b/workflow/bandit-workflow.cwl @@ -0,0 +1,38 @@ +#!/usr/bin/env cwltool +cwlVersion: v1.0 +class: Workflow + +inputs: + name: + type: string + default: bandit + pipeline_id: + type: string + repo_path: + type: Directory + run_id: + type: string + server_url: + type: string + +outputs: + bandit_report: + type: File + outputSource: bandit_step/bandit_report + +steps: + bandit_step: + in: + source_directory: repo_path + run: tools/bandit-tool.cwl + out: + - bandit_report + save_bandit_step: + in: + name: name + pipeline_id: pipeline_id + report: bandit_step/bandit_report + run_id: run_id + server_url: server_url + run: tools/save-tool.cwl + out: [] diff --git a/workflow/flake8-workflow.cwl b/workflow/flake8-workflow.cwl new file mode 100644 index 0000000..a0dc3fb --- /dev/null +++ b/workflow/flake8-workflow.cwl @@ -0,0 +1,38 @@ +#!/usr/bin/env cwltool +cwlVersion: v1.0 +class: Workflow + +inputs: + name: + type: string + default: flake8 + pipeline_id: + type: string + repo_path: + type: Directory + run_id: + type: string + server_url: + type: string + +outputs: + flake8_report: + type: File + outputSource: flake8_step/flake8_report + +steps: + flake8_step: + in: + source_directory: repo_path + run: tools/flake8-tool.cwl + out: + - flake8_report + save_flake8_step: + in: + name: name + pipeline_id: pipeline_id + report: flake8_step/flake8_report + run_id: run_id + server_url: server_url + run: tools/save-tool.cwl + out: [] diff --git a/workflow/pylint-workflow.cwl b/workflow/pylint-workflow.cwl new file mode 100644 index 0000000..82a352b --- /dev/null +++ b/workflow/pylint-workflow.cwl @@ -0,0 +1,38 @@ +#!/usr/bin/env cwltool +cwlVersion: v1.0 +class: Workflow + +inputs: + name: + type: string + default: pylint + pipeline_id: + type: string + repo_path: + type: Directory + run_id: + type: string + server_url: + type: string + +outputs: + pylint_report: + type: File + outputSource: pylint_step/pylint_report + +steps: + pylint_step: + in: + source_directory: repo_path + run: tools/pylint-tool.cwl + out: + - pylint_report + save_pylint_step: + in: + name: name + pipeline_id: pipeline_id + report: pylint_step/pylint_report + run_id: run_id + server_url: server_url + run: tools/save-tool.cwl + out: [] diff --git a/workflow/ruff-workflow.cwl b/workflow/ruff-workflow.cwl new file mode 100644 index 0000000..592d07a --- /dev/null +++ b/workflow/ruff-workflow.cwl @@ -0,0 +1,38 @@ +#!/usr/bin/env cwltool +cwlVersion: v1.0 +class: Workflow + +inputs: + name: + type: string + default: ruff + pipeline_id: + type: string + repo_path: + type: Directory + run_id: + type: string + server_url: + type: string + +outputs: + ruff_report: + type: File + outputSource: ruff_step/ruff_report + +steps: + ruff_step: + in: + source_directory: repo_path + run: tools/ruff-tool.cwl + out: + - ruff_report + save_ruff_step: + in: + name: name + pipeline_id: pipeline_id + report: ruff_step/ruff_report + run_id: run_id + server_url: server_url + run: tools/save-tool.cwl + out: [] diff --git a/workflow/tools/bandit-tool.cwl b/workflow/tools/bandit-tool.cwl new file mode 100755 index 0000000..962f0f0 --- /dev/null +++ b/workflow/tools/bandit-tool.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwltool + +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: cytopia/bandit + InlineJavascriptRequirement: {} + +inputs: + source_directory: + type: Directory + inputBinding: + position: 1 + +outputs: + bandit_report: + type: File + outputBinding: + glob: bandit_report.json + +baseCommand: +- bandit +arguments: +- -x +- .git +- -f +- json +- -o +- bandit_report.json +- --exit-zero diff --git a/workflow/tools/clone-tool.cwl b/workflow/tools/clone-tool.cwl new file mode 100755 index 0000000..57cab9e --- /dev/null +++ b/workflow/tools/clone-tool.cwl @@ -0,0 +1,29 @@ +#!/usr/bin/env cwltool + +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: alpine/git + InlineJavascriptRequirement: {} + +inputs: + branch: + type: string + default: main + repo_url: + type: string + +outputs: + repo_directory: + type: Directory + outputBinding: + glob: $(inputs.repo_url.split('/').pop().replace('.git','')) + +baseCommand: git +arguments: +- clone +- $(inputs.repo_url) +- -b +- $(inputs.branch) diff --git a/workflow/tools/flake8-tool.cwl b/workflow/tools/flake8-tool.cwl new file mode 100755 index 0000000..6607e4f --- /dev/null +++ b/workflow/tools/flake8-tool.cwl @@ -0,0 +1,26 @@ +#!/usr/bin/env cwltool + +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: eoepca/appquality-flake8-json:v0.1.0 + +inputs: + source_directory: + type: Directory + inputBinding: + position: 1 + +outputs: + flake8_report: + type: File + outputBinding: + glob: flake8_report.json + +baseCommand: flake8 +arguments: +- --format=json +- --output-file=flake8_report.json +- --exit-zero diff --git a/workflow/tools/pylint-tool.cwl b/workflow/tools/pylint-tool.cwl new file mode 100755 index 0000000..fc7f3b2 --- /dev/null +++ b/workflow/tools/pylint-tool.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwltool + +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: cytopia/pylint + InlineJavascriptRequirement: {} + +inputs: + source_directory: + type: Directory + inputBinding: + position: 1 + valueFrom: $(inputs.source_directory.path + "/**/*.py") + +outputs: + pylint_report: + type: File + outputBinding: + glob: pylint_report.json + +baseCommand: pylint +arguments: +- --output-format=json +- --output=pylint_report.json +- --exit-zero diff --git a/workflow/tools/ruff-tool.cwl b/workflow/tools/ruff-tool.cwl new file mode 100755 index 0000000..4c2d6b7 --- /dev/null +++ b/workflow/tools/ruff-tool.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwltool + +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: ghcr.io/astral-sh/ruff:alpine + InlineJavascriptRequirement: {} + +inputs: + source_directory: + type: Directory + inputBinding: + position: 1 + +outputs: + ruff_report: + type: File + outputBinding: + glob: ruff_report.json + +baseCommand: +- ruff +- check +arguments: +- --exclude +- .git +- --output-format +- json +- -o +- ruff_report.json +- -en diff --git a/workflow/tools/save-tool.cwl b/workflow/tools/save-tool.cwl new file mode 100755 index 0000000..7cf7cf5 --- /dev/null +++ b/workflow/tools/save-tool.cwl @@ -0,0 +1,35 @@ +#!/usr/bin/env cwltool + +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerPull: curlimages/curl + InlineJavascriptRequirement: {} + +inputs: + name: + type: string + pipeline_id: + type: string + report: + type: File + run_id: + type: string + server_url: + type: string + +outputs: [] + +baseCommand: curl +arguments: +- prefix: -X + valueFrom: POST +- prefix: -L + valueFrom: |- + $('http://' + inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name) +- prefix: -H + valueFrom: Content-Type:application/json +- prefix: -d + valueFrom: $('@' + inputs.report.path) diff --git a/workflow/workflow.cwl b/workflow/workflow.cwl new file mode 100755 index 0000000..f802c37 --- /dev/null +++ b/workflow/workflow.cwl @@ -0,0 +1,77 @@ +#!/usr/bin/env cwltool +cwlVersion: v1.0 +class: Workflow + +requirements: + SubworkflowFeatureRequirement: {} + +inputs: + repo_url: + type: string + branch: + type: string + pipeline_id: + type: string + run_id: + type: string + server_url: + type: string + +outputs: + pylint_report: + type: File + outputSource: pylint_workflow/pylint_report + flake8_report: + type: File + outputSource: flake8_workflow/flake8_report + ruff_report: + type: File + outputSource: ruff_workflow/ruff_report + bandit_report: + type: File + outputSource: bandit_workflow/bandit_report + +steps: + clone_step: + in: + repo_url: repo_url + branch: branch + run: tools/clone-tool.cwl + out: + - repo_directory + pylint_workflow: + in: + repo_path: clone_step/repo_directory + pipeline_id: pipeline_id + run_id: run_id + server_url: server_url + run: pylint-workflow.cwl + out: + - pylint_report + flake8_workflow: + in: + repo_path: clone_step/repo_directory + pipeline_id: pipeline_id + run_id: run_id + server_url: server_url + run: flake8-workflow.cwl + out: + - flake8_report + ruff_workflow: + in: + repo_path: clone_step/repo_directory + pipeline_id: pipeline_id + run_id: run_id + server_url: server_url + run: ruff-workflow.cwl + out: + - ruff_report + bandit_workflow: + in: + repo_path: clone_step/repo_directory + pipeline_id: pipeline_id + run_id: run_id + server_url: server_url + run: bandit-workflow.cwl + out: + - bandit_report