Merge branch 'dev' of gitlab.com:faradaysec/cloud/faraday_agent_dispa…

…tcher

.gitlab-ci.yml

@@ -38,146 +38,18 @@ stages:
 services:
     - postgres:latest
 
+include:
+  - local: docs/.gitlab-ci.yml
 
-flake8:
-    image: python:3
-    stage: .pre
-    script:
-      - pip install flake8
-      - flake8 .
-
-black:
-    image: python:3
-    stage: .pre
-    script:
-      - pip install black
-      - black --line-length=119 --check .
-
-.unit_tests_base:
-    stage: testing
-    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
-    script:
-        - source faraday_venv/bin/activate
-        - openssl req -x509 -nodes -days 1095 -newkey rsa:2048 -keyout tests/data/ok.key -out tests/data/ok.crt -subj "/CN=localhost"
-        - openssl req -x509 -nodes -days 1095 -newkey rsa:2048 -keyout tests/data/other.key -out tests/data/wrong.crt -subj "/CN=localhost"
-        - mkdir run_from
-        - cd run_from && mkdir logs && pytest ../tests/unittests --capture=sys -v --cov=../faraday_agent_dispatcher --cov-config=../tests/unittests/.coveragerc --color=yes --disable-warnings --junitxml=report.xml
-        - cp report.xml ../
-    artifacts:
-      reports:
-        junit: report.xml
-      paths:
-        - dist/*
-        - run_from/logs/*
-      expire_in: 7 days
-    rules:
-      - if: '$JUST_INTEGRATION'
-        when: never
-      - if: '$CI_COMMIT_REF_NAME =~ /^.*ci-test.*$/'
-        when: never
-      - when: on_success
-
-.latest_unit_test_base:
-  extends: .unit_tests_base
-  before_script:
-      - pip install virtualenv
-      - apt-get update
-      - apt-get install openssl
-      - virtualenv faraday_venv
-      - source faraday_venv/bin/activate
-      - pip install .[test]
-
-.alpha_unit_test_base:
-  extends: .unit_tests_base
-  before_script:
-      - pip install virtualenv
-      - apt-get update
-      - apt-get install openssl
-      - virtualenv faraday_venv
-      - source faraday_venv/bin/activate
-      - pip install --pre .[test] #It does not override alphas
-  allow_failure: true
-  rules:
-    - if: '$CI_COMMIT_REF_NAME =~ /^(dev|master)$/'
-      when: on_success
-    - if: '$TEST_ALPHA'
-      when: on_success
-
-unit_test 3.7:
-    extends: .latest_unit_test_base
-    image: python:3.7
-
-unit_test 3.8:
-    extends: .latest_unit_test_base
-    image: python:3.8
-
-unit_test 3.9:
-  extends: .latest_unit_test_base
-  image: python:3.9
-
-.unit_test 3.10: # To reactivate in beta (May 3rd 2021)
-    extends: .latest_unit_test_base
-    image: python:3.10-rc
-    allow_failure: true
-
-alpha_unit_test 3.7:
-    extends: .alpha_unit_test_base
-    image: python:3.7
-
-alpha_unit_test 3.8:
-    extends: .alpha_unit_test_base
-    image: python:3.8
-
-alpha_unit_test 3.9:
-    extends: .alpha_unit_test_base
-    image: python:3.9
-
-.alpha_unit_test 3.10: # To reactivate in beta (May 3rd 2021)
-    extends: .alpha_unit_test_base
-    image: python:3.10-rc
-
-integration_faraday:
-    stage: post_testing
-    variables:
-        DISPATCHER_REF: $CI_COMMIT_REF_NAME
-    trigger:
-        project: faradaysec/devops
-        strategy: depend
-    rules:
-        - if: '$CI_COMMIT_REF_NAME =~ /^(dev|master)$/'
-          when: on_success
-        - if: $INTEGRATION
-          when: on_success
-
-
-publish_pypi:
-    image: python:3
-    stage: publish
-    script:
-      - apt-get update -qy
-      - apt-get install twine -y
-      - python setup.py sdist bdist_wheel
-      - twine upload -u $PYPI_USER -p $PYPI_PASS dist/* --verbose
-    rules:
-      - if: '$CI_COMMIT_TAG'
-        when: on_success
-
+  - local: .gitlab/ci/.pre-gitlab-ci.yml
 
-#Historical integration test with faraday trigger
-#integration_faraday:
-#    stage: post_testing
-#    script:
-#        - apt-get update -qy
-#        - apt-get install curl -y
-#        - "curl -X POST -F token=$INTEGRATION_TOKEN -F ref=master -F \"variables[DISPATCHER_REF]=$CI_COMMIT_REF_NAME\" -F \"variables[FARADAY_REF]=$FARADAY_REF\" https://gitlab.com/api/v4/projects/15311100/trigger/pipeline"
-#    only:
-#        variables:
-#            - $CI_COMMIT_REF_NAME =~ /^(dev|master)$/
-#            - $INTEGRATION
+  - local: .gitlab/ci/testing/.testing-gitlab-ci.yml
+  - local: .gitlab/ci/testing/.post-gitlab-ci.yml
 
-include:
-  - local: docs/.gitlab-ci.yml
   - local: .gitlab/ci/plugins-integration/.build-gitlab-ci.yml
   - local: .gitlab/ci/plugins-integration/.testing-gitlab-ci.yml
+
   - local: .gitlab/ci/build_ci/.build-gitlab-ci.yml
+
+  - local: .gitlab/ci/publish/.pypi-gitlab-ci.yml
   - local: .gitlab/ci/publish/.publish-dockerhub-gitlab-ci.yml

.gitlab/ci/.pre-gitlab-ci.yml

@@ -0,0 +1,14 @@
+
+flake8:
+    image: python:3
+    stage: .pre
+    script:
+      - pip install flake8
+      - flake8 .
+
+black:
+    image: python:3
+    stage: .pre
+    script:
+      - pip install black
+      - black --line-length=119 --check .

.gitlab/ci/publish/.pypi-gitlab-ci.yml

@@ -0,0 +1,12 @@
+
+publish_pypi:
+    image: python:3
+    stage: publish
+    script:
+      - apt-get update -qy
+      - apt-get install twine -y
+      - python setup.py sdist bdist_wheel
+      - twine upload -u $PYPI_USER -p $PYPI_PASS dist/* --verbose
+    rules:
+      - if: '$CI_COMMIT_TAG'
+        when: on_success

.gitlab/ci/testing/.post-gitlab-ci.yml

@@ -0,0 +1,13 @@
+
+integration_faraday:
+    stage: post_testing
+    variables:
+        DISPATCHER_REF: $CI_COMMIT_REF_NAME
+    trigger:
+        project: faradaysec/devops
+        strategy: depend
+    rules:
+        - if: '$CI_COMMIT_REF_NAME =~ /^(dev|master)$/'
+          when: on_success
+        - if: $INTEGRATION
+          when: on_success

.gitlab/ci/testing/.testing-gitlab-ci.yml

@@ -0,0 +1,83 @@
+.venv_ready:
+  before_script:
+    - pip install virtualenv
+    - apt-get update
+    - apt-get install openssl
+    - virtualenv faraday_venv
+    - source faraday_venv/bin/activate
+
+
+.unit_tests_base:
+    stage: testing
+    coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
+    script:
+        - source faraday_venv/bin/activate
+        - openssl req -x509 -nodes -days 1095 -newkey rsa:2048 -keyout tests/data/ok.key -out tests/data/ok.crt -subj "/CN=localhost"
+        - openssl req -x509 -nodes -days 1095 -newkey rsa:2048 -keyout tests/data/other.key -out tests/data/wrong.crt -subj "/CN=localhost"
+        - mkdir run_from
+        - cd run_from && mkdir logs && pytest ../tests/unittests --capture=sys -v --cov=../faraday_agent_dispatcher --cov-config=../tests/unittests/.coveragerc --color=yes --disable-warnings --junitxml=report.xml
+        - cp report.xml ../
+    artifacts:
+      reports:
+        junit: report.xml
+      paths:
+        - dist/*
+        - run_from/logs/*
+      expire_in: 7 days
+    rules:
+      - if: '$JUST_INTEGRATION'
+        when: never
+      - if: '$CI_COMMIT_REF_NAME =~ /^.*ci-test.*$/'
+        when: never
+      - when: on_success
+
+.latest_unit_test_base:
+  extends: .unit_tests_base
+  before_script:
+    - !reference [.venv_ready, before_script]
+    - pip install .[test]
+
+.alpha_unit_test_base:
+  extends: .unit_tests_base
+  before_script:
+    - !reference [.venv_ready, before_script]
+    - pip install --pre .[test] #It does not override alphas
+  allow_failure: true
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /^(dev|master)$/'
+      when: on_success
+    - if: '$TEST_ALPHA'
+      when: on_success
+
+unit_test 3.7:
+    extends: .latest_unit_test_base
+    image: python:3.7
+
+unit_test 3.8:
+    extends: .latest_unit_test_base
+    image: python:3.8
+
+unit_test 3.9:
+  extends: .latest_unit_test_base
+  image: python:3.9
+
+.unit_test 3.10: # To reactivate in beta (May 3rd 2021)
+    extends: .latest_unit_test_base
+    image: python:3.10-rc
+    allow_failure: true
+
+alpha_unit_test 3.7:
+    extends: .alpha_unit_test_base
+    image: python:3.7
+
+alpha_unit_test 3.8:
+    extends: .alpha_unit_test_base
+    image: python:3.8
+
+alpha_unit_test 3.9:
+    extends: .alpha_unit_test_base
+    image: python:3.9
+
+.alpha_unit_test 3.10: # To reactivate in beta (May 3rd 2021)
+    extends: .alpha_unit_test_base
+    image: python:3.10-rc

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.4.0
     hooks:
     -   id: trailing-whitespace
         args: [ --markdown-linebreak-ext=md ]
@@ -13,12 +13,12 @@ repos:
     #-   id: requirements-txt-fixer
     #-   id: double-quote-string-fixer
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.9.1
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-typing-imports==1.9.0]
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 21.4b1
     hooks:
     -   id: black
         language_version: python3

CHANGELOG/1.5.1/date.md

@@ -0,0 +1 @@
+May 6th, 2021

CHANGELOG/1.5.1/faraday_version.md

@@ -0,0 +1 @@
+[Faraday][faraday] versions: 3.14.3

RELEASE.md

@@ -1,3 +1,8 @@
+1.5.1 [May 6th, 2021]:
+---
+ * FIX Burp executor parse the IP
+ * [Faraday][faraday] versions: 3.14.3
+
 1.5.0 [Mar 30th, 2021]:
 ---
  * ADD having at least a executor is mandatory, if not it will not save the configuration

faraday_agent_dispatcher/__init__.py

@@ -18,6 +18,6 @@
 
 """Top-level package for faraday_agent_dispatcher."""
 
-__author__ = """Eric Horvat"""
-__email__ = "erich@infobytesec.com"
-__version__ = "1.5.0"
+__author__ = """Faraday Development Team"""
+__email__ = "devel@infobytesec.com"
+__version__ = "1.5.1"

faraday_agent_dispatcher/static/executors/dev/heroku_discovery_agent.py

@@ -55,7 +55,7 @@
 
 
 def main():
-    """ heroku cli user must be logged to run this agent"""
+    """heroku cli user must be logged to run this agent"""
 
     try:
         subprocess.run(["heroku", "auth:whoami"], stdout=subprocess.DEVNULL, check=True)

faraday_agent_dispatcher/static/executors/official/burp.py

@@ -5,17 +5,17 @@
 import requests
 import time
 import socket
-from pathlib import Path
 import xml.etree.cElementTree as ET
 from urllib.parse import urlparse
 from faraday_plugins.plugins.repo.burp.plugin import BurpPlugin
 
 
-def get_ip(host_name):
+def get_ip(url):
+    url_data = urlparse(url)
     try:
-        ip = socket.gethostbyname(host_name)
+        ip = socket.gethostbyname(url_data.netloc)
     except socket.error:
-        ip = "0.0.0.0"
+        ip = url_data.netloc
     return ip
 
 
@@ -122,10 +122,8 @@ def main():
             file=sys.stderr,
         )
         sys.exit()
-
-    with tempfile.TemporaryDirectory() as tempdirname:
-        tmpdir = Path(tempdirname)
-        name_result = tmpdir / "output.xml"
+    print(f"Scanning {url_target}", file=sys.stderr)
+    with tempfile.TemporaryFile() as tmp_file:
         issue_def = f"{api_host}/{api_key}" f"/v0.1/knowledge_base/issue_definitions"
         rg_issue_definitions = requests.get(issue_def)
         json_issue_definitions = rg_issue_definitions.json()
@@ -138,7 +136,7 @@ def main():
         rp_scan = requests.post(f"{api_host}/{api_key}/v0.1/scan", json=json_scan)
         get_location = rp_scan.headers["Location"]
         scan_status = ""
-        while scan_status != "succeeded":
+        while scan_status not in ("succeeded", "failed"):
             try:
                 rg_issues = requests.get(f"{api_host}/{api_key}" f"/v0.1/scan/{get_location}")
             except ConnectionError:
@@ -149,12 +147,14 @@ def main():
             scan_status = issues["scan_status"]
             # Before checking back, wait 15 seconds.
             time.sleep(5)
+            print(f"Waiting for results [{scan_status}]", file=sys.stderr)
 
-        generate_xml(issues, name_result, json_issue_definitions)
+        print("Scan finished", file=sys.stderr)
+        generate_xml(issues, tmp_file, json_issue_definitions)
         plugin = BurpPlugin()
-        with open(name_result, "r") as f:
-            plugin.parseOutputString(f.read())
-            print(plugin.get_json())
+        tmp_file.seek(0)
+        plugin.parseOutputString(tmp_file.read())
+        print(plugin.get_json())
 
 
 if __name__ == "__main__":