Use pipenv to maintain/lock dependencies

MANIFEST.in

@@ -9,5 +9,4 @@ include LICENSE
 include AUTHORS
 include requirements.txt
 include requirements-dev.txt
-include requirements-test.txt
 include install.sh

Pipfile

@@ -0,0 +1,47 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+Flask-Testing = "*"
+flake8 = "*"
+mock = "*"
+pathlib = "*"
+pre-commit = "*"
+pytest = "*"
+selenium = "*"
+
+[packages]
+Flask = "*"
+Flask-Cors = "*"
+SQLAlchemy = "*"
+SQLAlchemy-Utils = "*"
+bitstring = "*"
+boltons = "*"
+celery = "*"
+click = "*"
+configparser = "*"
+elasticsearch = "<7.0.0,>=6.0.0"
+elasticsearch-curator = "*"
+future = "*"
+mmbot = {git = "https://github.com/egaus/MaliciousMacroBot.git"}
+paramiko = "*"
+pefile = "*"
+pehash = {git = "https://github.com/knowmalware/pehash.git"}
+psycopg2-binary = "*"
+pyClamd = "*"
+pyimpfuzzy = "*"
+pymongo = "*"
+python-magic = "*"
+rarfile = "*"
+reportlab = "*"
+requests = "*"
+setuptools = "*"
+six = "*"
+sqlalchemy-datatables = "==1.2.0"
+ssdeep = "*"
+stix2 = "*"
+tika = "*"
+tqdm = "*"
+yara-python = {git = "https://github.com/VirusTotal/yara-python/"}

docker_utils/Dockerfile

@@ -71,6 +71,8 @@ RUN apk add --no-cache \
   # install ms dependencies
   && cd /opt/multiscanner \
   && pip3 install --upgrade pip \
+  # mmbot is difficult
+  && sed -i '/mmbot/d' ./requirements.txt \
   && pip3 install -r requirements.txt \
   && pip3 install gevent \
   # clean up

requirements-dev.txt

@@ -1,3 +1,35 @@
--r requirements-test.txt
-flake8
-pre-commit
+-i https://pypi.org/simple
+aspy.yaml==1.2.0
+atomicwrites==1.3.0
+attrs==19.1.0
+cfgv==1.6.0
+click==6.7
+entrypoints==0.3
+flake8==3.7.7
+flask-testing==0.7.1
+flask==1.0.2
+identify==1.4.1
+importlib-metadata==0.9
+itsdangerous==1.1.0
+jinja2==2.10.1
+markupsafe==1.1.1
+mccabe==0.6.1
+mock==2.0.0
+more-itertools==7.0.0 ; python_version > '2.7'
+nodeenv==1.3.3
+pathlib==1.0.1
+pbr==5.1.3
+pluggy==0.9.0
+pre-commit==1.15.2
+py==1.8.0
+pycodestyle==2.5.0
+pyflakes==2.1.1
+pytest==4.4.1
+pyyaml==5.1
+selenium==3.141.0
+six==1.12.0
+toml==0.10.0
+urllib3==1.24.2 ; python_version >= '3.4'
+virtualenv==16.4.3
+werkzeug==0.15.2
+zipp==0.3.3

requirements.txt

@@ -1,38 +1,65 @@
-#Required
-future
-six
-configparser
+-i https://pypi.org/simple
+amqp==2.4.2
+antlr4-python3-runtime==4.7.2 ; python_version >= '3'
+asn1crypto==0.24.0
+bcrypt==3.1.6
+billiard==3.6.0.0
+bitstring==3.1.5
+boltons==19.1.0
+boto3==1.9.132
+botocore==1.12.132
+celery==4.3.0
+certifi==2019.3.9
+cffi==1.12.2
+chardet==3.0.4
 click==6.7
-#Required by modules
-bitstring
-boltons
-git+https://github.com/knowmalware/pehash.git#egg=pehash
-paramiko
-pefile
-pyclamd
-pyimpfuzzy
-python-magic
-requests
-ssdeep
-tika
-yara-python
-#Required for STIX2 content
-stix2
-#Required for PDF
-reportlab
-#Required by API
-flask
-psycopg2-binary
-rarfile
-sqlalchemy
-sqlalchemy-utils
+configparser==3.7.4
+cryptography==2.6.1
+docutils==0.14
+elasticsearch-curator==5.6.0
+elasticsearch==6.3.1
+flask-cors==3.0.7
+flask==1.0.2
+future==0.17.1
+git+https://github.com/VirusTotal/yara-python/@63ac2417a918692be6d5bd659fbfd39564396ec3#egg=yara-python
+git+https://github.com/egaus/MaliciousMacroBot.git@f5f54780695e4394b3afcd729c7f8c7f332ea7e5#egg=mmbot
+git+https://github.com/knowmalware/pehash.git@d1092b48bc6c06bb6b6a7f60ffaee469d8842ad7#egg=pehash
+idna==2.8
+itsdangerous==1.1.0
+jinja2==2.10.1
+jmespath==0.9.4
+kombu==4.5.0
+markupsafe==1.1.1
+paramiko==2.4.2
+pefile==2019.4.14
+pillow==6.0.0
+psycopg2-binary==2.8.2
+pyasn1==0.4.5
+pyclamd==0.4.0
+pycparser==2.19
+pyimpfuzzy==0.5
+pymongo==3.7.2
+pynacl==1.3.0
+python-dateutil==2.8.0 ; python_version >= '2.7'
+python-magic==0.4.15
+pytz==2019.1
+pyyaml==5.1
+rarfile==3.0
+reportlab==3.5.19
+requests-aws4auth==0.9
+requests==2.21.0
+s3transfer==0.2.0
+simplejson==3.16.0
+six==1.12.0
 sqlalchemy-datatables==1.2.0
-#Required by storage modules
-elasticsearch>=6.0.0,<7.0.0
-elasticsearch-curator
-pymongo
-#Required for distributed
-celery
-flask_cors
-#Required by utils
-tqdm
+sqlalchemy-utils==0.33.11
+sqlalchemy==1.3.3
+ssdeep==3.3
+stix2-patterns==1.1.0
+stix2==1.1.2
+tika==1.19
+tqdm==4.31.1
+urllib3==1.24.2 ; python_version >= '3.4'
+vine==1.3.0
+voluptuous==0.11.5
+werkzeug==0.15.2

setup.py

@@ -24,7 +24,9 @@ def get_requirements(filename, dep_links_only=False):
     requirements = []
     with open(filename) as f:
         for l in f.readlines():
-            if dep_links_only is False and REQ_REGEX.match(l):
+            if '-i' in l:
+                continue
+            elif dep_links_only is False and REQ_REGEX.match(l):
                 requirements.append(l.strip())
             elif dep_links_only is True and DEPS_LINKS_REGEX.match(l):
                 requirements.append(l.strip())
@@ -60,8 +62,7 @@ def get_long_description():
         ],
     },
     extras_require={
-        'dev': get_requirements('requirements-test.txt') + get_requirements('requirements-dev.txt'),
-        'test': get_requirements('requirements-test.txt'),
+        'dev': get_requirements('requirements-dev.txt'),
     },
     dependency_links=get_requirements('requirements.txt', dep_links_only=True),
     classifiers=[

tox.ini

@@ -2,7 +2,9 @@
 envlist = py27, py35, py36, py37
 
 [testenv]
-deps = -rrequirements-dev.txt
+deps =
+    -rrequirements.txt
+    -rrequirements-dev.txt
 commands =
   multiscanner init
   pytest