Merge pull request #117 from superphy/develop

PyPy3.6 Support

.circleci/config.yml

@@ -6,14 +6,7 @@ version: 2
 jobs:
     build:
         docker:
-            # specify the version you desire here
-            # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
-            - image: superphy/tox-base:latest
-
-            # Specify service dependencies here if necessary
-            # CircleCI maintains a library of pre-built images
-            # documented at https://circleci.com/docs/2.0/circleci-images/
-            # - image: circleci/postgres:9.4
+            - image: superphy/tox-base:d698931a52ddeef64dcf5918b32790bcef75af61
 
         working_directory: ~/repo
 
@@ -26,13 +19,6 @@ jobs:
                     git submodule init
                     git submodule update --remote
 
-            # Download and cache dependencies
-            - restore_cache:
-                  keys:
-                      - v1-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements_dev.txt" }}
-                      # fallback to using the latest cache if no exact match is found
-                      - v1-dependencies-
-
             - run:
                   name: install dependencies
                   command: |
@@ -45,26 +31,23 @@ jobs:
                   name: install lemongraph
                   command: |
                       . venv/bin/activate
-                      apt-get update -y && apt-get install -y libffi-dev zlib1g-dev python-dev python-cffi
+                      apt-get update -y && apt-get install -y libffi-dev zlib1g-dev python-dev python-cffi libatlas-base-dev
+                      pip install --upgrade cffi
                       cd lemongraph && python setup.py install
 
-
-            - save_cache:
-                  paths:
-                      - ./venv
-                  key: v1-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements_dev.txt" }}
+            - run:
+                  name: update pip
+                  command: |
+                      pip install --upgrade pip
+                      pypy3 -m ensurepip
+                      pypy3 -m pip install --upgrade pip
 
             # run tests!
-            # this example uses Django's built-in test-runner
-            # other common Python testing frameworks include pytest and nose
-            # https://pytest.org
-            # https://nose.readthedocs.io
             - run:
                   name: run tests
                   command: |
-                      . venv/bin/activate
-                      pip install tox
                       tox -v
+
             - run:
                   name: codecov
                   command: |

.gitmodules

@@ -3,5 +3,5 @@
 	url = https://github.com/superphy/diffpool.git
 [submodule "lemongraph"]
 	path = lemongraph
-	url = https://github.com/NationalSecurityAgency/lemongraph.git
-	ignore = dirty
+	url = https://github.com/superphy/lemongraph.git
+	ignore = dirty

Snakefile

@@ -8,13 +8,12 @@ import dill
 import pandas as pd
 import numpy as np
 from contextlib import contextmanager
-from pyinstrument import Profiler
 
+from prairiedog.profiler import Profiler
 from prairiedog.kmers import Kmers
 from prairiedog.networkx_graph import NetworkXGraph
 from prairiedog.graph_ref import GraphRef
 from prairiedog.subgraph_ref import SubgraphRef
-from prairiedog.dgl_graph import DGLGraph
 from prairiedog.lemon_graph import LGGraph, DB_PATH
 
 configfile: "config.yaml"
@@ -68,11 +67,7 @@ rule pangenome:
             print("Using LemonGraph as graph backend")
             sg = SubgraphRef(LGGraph())
         else:
-            print("Using DGL as graph backend")
-            sg = SubgraphRef(DGLGraph(
-                n_labels=len(input),
-                n_nodes=4**K + 20 # We have some odd contigs that use N
-            ))
+            raise Exception("No graph backend found")
 
         # Setup pyinstrument profiler
         if config['pyinstrument'] is True:

prairiedog/lemon_graph.py

@@ -31,7 +31,7 @@ def __init__(self, db_path: str = None, delete_on_exit=False, nosync=True,
         else:
             os.makedirs(prairiedog.config.OUTPUT_DIRECTORY, exist_ok=True)
             self.db_path = DB_PATH
-        log.debug("Creating LemonGraph with backing file {}".format(
+        log.info("Initializing LemonGraph object with backing file {}".format(
             self.db_path))
         self.g = LemonGraph.Graph(path=self.db_path, nosync=nosync,
                                   noreadahead=noreadahead, readonly=readonly)

prairiedog/profiler.py

@@ -0,0 +1,26 @@
+import platform
+import logging
+
+log = logging.getLogger("prairiedog")
+
+
+if platform.python_implementation() == 'PyPy':
+    class Profiler:
+        """
+        PyPy is missing some functions required for pyinstrument
+        """
+        def __init__(self):
+            log.warning("Initialized empty Profiler since platform is"
+                        " {}".format(platform.python_implementation()))
+
+        def start(self):
+            pass
+
+        def stop(self):
+            pass
+
+        def output_text(self, unicode, color):
+            return ""
+else:
+    import pyinstrument
+    Profiler = pyinstrument.Profiler

prairiedog/test_profiler.py

@@ -0,0 +1,9 @@
+from prairiedog.profiler import Profiler
+
+
+def test_profiler():
+    profiler = Profiler()
+    profiler.start()
+    profiler.stop()
+    print(profiler.output_text(unicode=True, color=True))
+    assert True

requirements.txt

@@ -2,11 +2,11 @@ coloredlogs==10.0
 decorator==4.4.0
 humanfriendly==4.18
 networkx==2.3
+cython
+dateutils
 numpy
 pandas
 click>=6.0
 dill
-dgl
-torch
 psutil
 pyinstrument

requirements_dev.txt

@@ -1,4 +1,4 @@
-pip==18.1
+pip==19.1.1
 bumpversion==0.5.3
 wheel==0.32.1
 watchdog==0.9.0
@@ -8,8 +8,8 @@ coverage==4.5.1
 Sphinx==1.8.1
 twine==1.12.1
 
-pytest==3.8.2
-pytest-runner==4.2
+pytest==4.6.3
+pytest-runner==5.1
 
 pytest-cov
 codecov

tests/test_subgraph_ref.py

@@ -1,18 +1,20 @@
 import pytest
 import logging
 
-from pyinstrument import Profiler
-
+from prairiedog.profiler import Profiler
 from prairiedog.subgraph_ref import SubgraphRef
 from prairiedog.graph_ref import GraphRef
 
 log = logging.getLogger("prairiedog")
 
 
 def _do_subgraph(g, km):
+    log.info("Setting up SubgraphRef...")
     sgr = SubgraphRef(g)
+    log.info("Setting up GraphRef....")
     gr = GraphRef()
 
+    log.info("Updating graph...")
     sgr.update_graph(km, gr)
 
     assert True
@@ -23,11 +25,14 @@ def test_subgraph_creation(g, km_short):
 
 
 def test_subgraph_creation_profile(g, km_short):
+    log.info("Initializing and starting profiler...")
     profiler = Profiler()
     profiler.start()
 
     _do_subgraph(g, km_short)
 
+    log.info("Stopping profiler...")
     profiler.stop()
 
+    log.info("Trying to print profiler test...")
     print(profiler.output_text(unicode=True, color=True))

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35, py36, py37, flake8,
+envlist = py35, py36, py37, pypy36, flake8,
 
 [flake8]
 exclude = tests/*
@@ -22,12 +22,13 @@ deps =
 ; requirements.txt with the pinned versions and uncomment the following line:
      -r{toxinidir}/requirements.txt
 commands =
+    pip -V
     pip install -U pip
     pip install git+https://github.com/pytries/datrie.git
     pip install snakemake
     bash -ec '[ "$CI" = true ] && \
         echo "In CI, will not install profiler for benchmarks" || \
         pip install memory_profiler'
     bash -ec 'cd lemongraph && python setup.py install'
-    py.test -v -s --basetemp={envtmpdir} \
+    pytest -v -s --basetemp={envtmpdir} \
         --cov=prairiedog --cov-report=xml --cov-report=html