Merge pull request #89 from matthew-t-smith/dev/mthomas

test: new GitHub Actions for front, back, Docker; increase PyWorkflow coverage to 83%

.github/workflows/main.yml → .github/workflows/test_backend.yml

@@ -1,40 +1,38 @@
-# This is a basic workflow to help you get started with Actions
+# This workflow tests starting the back-end server, unit and API tests
 
-name: Postman Tests
+name: Test back-end
 
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the master branch
 on:
   push:
   pull_request:
     branches: [ master ]
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
-  build:
-    # The type of runner that the job will run on
+  setup-back-end:
     runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+    # Checks-out your repository, for access in the workflow
     - uses: actions/checkout@v2
 
+    # Setup python
     - name: Set up Python 3.x
       uses: actions/setup-python@v1
       with:
         python-version: 3.x
 
+    # Install dependencies
     - name: Install dependencies
-
       run: |
         python3 -m pip install --upgrade pip
         pip3 install pipenv
         cd back-end
-        pipenv install
-        pipenv run echo "SECRET_KEY='TEMPORARY SECRET KEY'" > vp/.environment
+        pipenv install --dev
 
+    # Add .env and test data
+    - name: Add test/env data
+      run: |
+        cd back-end
+        pipenv run echo "SECRET_KEY='TEMPORARY SECRET KEY'" > vp/.environment
         echo ",key,A
               0,K0,A0
               1,K1,A1
@@ -43,17 +41,23 @@ jobs:
               4,K4,A4
               5,K5,A5" > /tmp/sample.csv
 
+    # Run unittests
+    - name: Run unittests
+      run: |
+        cd back-end/pyworkflow/pyworkflow
+        pipenv run coverage run -m unittest tests/*.py
+        pipenv run coverage report
 
-        cd pyworkflow/pyworkflow
-        pipenv run python3 -m unittest tests/*.py
-        cd ../../vp
+    # Start server in background for API tests
+    - name: Start server
+      run: |
+        cd back-end/vp
         pipenv run nohup python3 manage.py runserver &
 
-  newman:
-    needs: build
-    runs-on: ubuntu-latest
 
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+  postman-tests:
+    needs: setup-back-end
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
@@ -62,3 +66,4 @@ jobs:
       with:
         collection: Postman/Visual\ Programming-Tests.postman_collection.json
         environment: Postman/Local\ Testing.postman_environment.json
+

.github/workflows/test_docker.yml

@@ -0,0 +1,23 @@
+# This workflow tests building the Docker images and starting containers
+
+name: Test Docker build
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+    # Checks-out your repository, for access in the workflow
+    - uses: actions/checkout@v2
+
+    # Build containers
+    - name: Build docker container
+      run: docker-compose build
+
+    # Run containers
+    - name: Start Docker containers
+      run: docker-compose up -d

.github/workflows/test_frontend.yml

@@ -0,0 +1,24 @@
+# This workflow tests front-end
+
+name: Test front-end
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test-front-end:
+    runs-on: ubuntu-latest
+    steps:
+    # Checks-out your repository, for access in the workflow
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        cd front-end
+        npm ci
+
+    - name: Run unit tests
+      run: |
+        cd front-end
+        npm test

README.md

@@ -1,12 +1,11 @@
 # PyWorkflow
-|            |        |
+|            | Status |
 |------------|--------|
-| Docker     | TBD    |
-| Back-end   | ![Postman Tests](https://github.com/matthew-t-smith/visual-programming/workflows/Postman%20Tests/badge.svg) |
-| Front-end  | TBD |
+| Docker     | ![Test Docker build](https://github.com/matthew-t-smith/visual-programming/workflows/Test%20Docker%20build/badge.svg)    |
+| Back-end   | ![Test back-end](https://github.com/matthew-t-smith/visual-programming/workflows/Test%20back-end/badge.svg) |
+| Front-end  | ![Test front-end](https://github.com/matthew-t-smith/visual-programming/workflows/Test%20front-end/badge.svg) |
 | PyWorkflow | ![Code Coverage](./docs/media/pyworkflow_coverage.svg) |
-| CLI        | TBD |
-| Jest       | TBD |  
+| UI       | ![Code Coverage](./docs/media/ui_coverage.svg) |  
 
 PyWorkflow is a visual programming application for building data science
 pipelines and workflows. It is inspired by [KNIME](https://www.knime.com)

back-end/pyworkflow/pyworkflow/.coveragerc

@@ -1,4 +1,5 @@
 [run]
 omit=
 	*/.local/share/virtualenvs/*
-    ./tests/*
+    ./tests/*
+    ./nodes/custom_nodes/*

back-end/pyworkflow/pyworkflow/node.py

@@ -113,7 +113,7 @@ def to_json(self):
         }
 
     def __str__(self):
-        return "Test"
+        return self.name
 
 
 class FlowNode(Node):

back-end/pyworkflow/pyworkflow/node_factory.py

@@ -69,5 +69,5 @@ def custom_node(node_key, node_info):
 
         return instance
     except Exception as e:
-        print(str(e))
+        # print(str(e))
         return None

back-end/pyworkflow/pyworkflow/tests/sample_test_data.py

@@ -28,7 +28,13 @@
         "node_key": "JoinNode",
         "is_global": False,
         "options": {
-            "on": "key"
+            "on": "to_replace"
+        },
+        "option_replace": {
+            "on": {
+                "node_id": "7",
+                "is_global": False,
+            }
         }
     },
     "filter_node": {
@@ -51,14 +57,32 @@
             "on": "key"
         }
     },
+    "graph_node": {
+        "name": "Graph",
+        "node_id": "6",
+        "node_type": "visualization",
+        "node_key": "GraphNode",
+        "is_global": False,
+    },
     "string_input": {
         "name": "String Input",
-        "node_id": "6",
+        "node_id": "7",
         "node_type": "flow_control",
         "node_key": "StringNode",
         "is_global": False,
         "options": {
-            "default_value": "My value",
+            "default_value": "key",
+            "var_name": "local_flow_var"
+        }
+    },
+    "integer_input": {
+        "name": "Integer Input",
+        "node_id": "8",
+        "node_type": "flow_control",
+        "node_key": "IntegerNode",
+        "is_global": False,
+        "options": {
+            "default_value": 42,
             "var_name": "my_var"
         }
     },
@@ -69,8 +93,8 @@
         "node_key": "StringNode",
         "is_global": True,
         "options": {
-            "default_value": "My value",
-            "var_name": "my_var"
+            "default_value": ",",
+            "var_name": "global_flow_var"
         }
     },
 }
@@ -97,6 +121,13 @@
         "node_key": "foobar",
         "is_global": False,
     },
+    "bad_visualization_node": {
+        "name": "Foobar",
+        "node_id": "1",
+        "node_type": "visualization",
+        "node_key": "foobar",
+        "is_global": False,
+    },
     "bad_node_type": {
         "name": "Foobar",
         "node_id": "1",
@@ -112,6 +143,11 @@
         default='my value',
         docstring='my docstring'
     ),
+    "text_param": TextParameter(
+        'CSV Input',
+        default='my value',
+        docstring='my docstring'
+    ),
     "bool_param": BooleanParameter(
         'Drop NaN columns',
         default=True,
@@ -126,6 +162,12 @@
         default=42,
         docstring="CSV File"
     ),
+    "select_param": SelectParameter(
+        'Graph type',
+        options=["area", "bar", "line", "point"],
+        default='bar',
+        docstring='my docstring'
+    ),
 }
 
 BAD_PARAMETERS = {
@@ -149,6 +191,16 @@
         default=42,
         docstring="CSV File"
     ),
+    "bad_text_param": TextParameter(
+        'Bad Bool Param',
+        default=42,
+        docstring="CSV File"
+    ),
+    "bad_select_param": SelectParameter(
+        'Bad Bool Param',
+        default=42,
+        docstring="CSV File"
+    ),
 }
 
 DATA_FILES = {
@@ -162,5 +214,22 @@
     "sample2": (',key,B\n'
                 '0,K0,B0\n'
                 '1,K1,B1\n'
-                '2,K2,B2\n')
+                '2,K2,B2\n'),
+    "good_custom_node": ('from pyworkflow.node import Node, NodeException\n'
+                         'from pyworkflow.parameters import *\n'
+                         'class MyGoodCustomNode(Node):\n'
+                         '\tname="Custom Node"\n'
+                         '\tnum_in=1\n'
+                         '\tnum_out=1\n'
+                         '\tdef execute(self, predecessor_data, flow_vars):\n'
+                         '\t\tprint("Hello world")\n'),
+    "bad_custom_node": ('from pyworkflow.node import Node, NodeException\n'
+                        'from pyworkflow.parameters import *\n'
+                        'import torch\n'
+                        'class MyBadCustomNode(Node):\n'
+                        '\tname="Custom Node"\n'
+                        '\tnum_in=1\n'
+                        '\tnum_out=1\n'
+                        '\tdef execute(self, predecessor_data, flow_vars):\n'
+                        '\t\tprint("Hello world")\n'),
 }