diff --git a/.circleci/config.yml b/.circleci/config.yml index 475c29e..a531829 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,14 +6,7 @@ version: 2 jobs: build: docker: - # specify the version you desire here - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: superphy/tox-base:latest - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 + - image: superphy/tox-base:d698931a52ddeef64dcf5918b32790bcef75af61 working_directory: ~/repo @@ -26,13 +19,6 @@ jobs: git submodule init git submodule update --remote - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements_dev.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - run: name: install dependencies command: | @@ -45,26 +31,23 @@ jobs: name: install lemongraph command: | . venv/bin/activate - apt-get update -y && apt-get install -y libffi-dev zlib1g-dev python-dev python-cffi + apt-get update -y && apt-get install -y libffi-dev zlib1g-dev python-dev python-cffi libatlas-base-dev + pip install --upgrade cffi cd lemongraph && python setup.py install - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements_dev.txt" }} + - run: + name: update pip + command: | + pip install --upgrade pip + pypy3 -m ensurepip + pypy3 -m pip install --upgrade pip # run tests! - # this example uses Django's built-in test-runner - # other common Python testing frameworks include pytest and nose - # https://pytest.org - # https://nose.readthedocs.io - run: name: run tests command: | - . venv/bin/activate - pip install tox tox -v + - run: name: codecov command: | diff --git a/.gitmodules b/.gitmodules index 9ffa1f1..70f622c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,5 +3,5 @@ url = https://github.com/superphy/diffpool.git [submodule "lemongraph"] path = lemongraph - url = https://github.com/NationalSecurityAgency/lemongraph.git - ignore = dirty \ No newline at end of file + url = https://github.com/superphy/lemongraph.git + ignore = dirty diff --git a/Snakefile b/Snakefile index 91c6829..3f52638 100644 --- a/Snakefile +++ b/Snakefile @@ -8,13 +8,12 @@ import dill import pandas as pd import numpy as np from contextlib import contextmanager -from pyinstrument import Profiler +from prairiedog.profiler import Profiler from prairiedog.kmers import Kmers from prairiedog.networkx_graph import NetworkXGraph from prairiedog.graph_ref import GraphRef from prairiedog.subgraph_ref import SubgraphRef -from prairiedog.dgl_graph import DGLGraph from prairiedog.lemon_graph import LGGraph, DB_PATH configfile: "config.yaml" @@ -68,11 +67,7 @@ rule pangenome: print("Using LemonGraph as graph backend") sg = SubgraphRef(LGGraph()) else: - print("Using DGL as graph backend") - sg = SubgraphRef(DGLGraph( - n_labels=len(input), - n_nodes=4**K + 20 # We have some odd contigs that use N - )) + raise Exception("No graph backend found") # Setup pyinstrument profiler if config['pyinstrument'] is True: diff --git a/lemongraph b/lemongraph index d464d1f..5d4e7be 160000 --- a/lemongraph +++ b/lemongraph @@ -1 +1 @@ -Subproject commit d464d1f3b4797a534ae2522223c36393b18bd127 +Subproject commit 5d4e7be86b2b6b373ed1ebd9c2cbd0f7ed75cd9a diff --git a/prairiedog/dgl_graph.py b/prairiedog/dgl_graph.py deleted file mode 100644 index fc21695..0000000 --- a/prairiedog/dgl_graph.py +++ /dev/null @@ -1,67 +0,0 @@ -import logging -import typing -import pickle - -import dgl -import torch as th - -import prairiedog.graph - -log = logging.getLogger("prairiedog") - - -class DGLGraph(prairiedog.graph.Graph): - def __init__(self, n_labels: int, n_nodes): - self.g = dgl.DGLGraph(multigraph=True) - self.g.set_n_initializer(dgl.init.zero_initializer) - self.labels = th.nn.functional.one_hot( - th.arange(0, n_labels) - ) - # We pre-initialize all nodes for performance reasons - self.g.add_nodes(n_nodes) - - def upsert_node(self, node: int, labels: dict = None): - pass - - def add_edge(self, node_a: int, node_b: int, labels: dict = None): - if labels is not None: - encoded_labels = {} - for k, v in labels.items(): - encoded_labels[k] = self.labels[v] - self.g.add_edge(node_a, node_b, encoded_labels) - else: - self.g.add_edge(node_a, node_b) - - def clear(self): - self.g.clear() - - @property - def nodes(self) -> set: - return set(self.g.nodes) - - @property - def edges(self) -> set: - return set(self.g.edges) - - def get_labels(self, node: str) -> dict: - return self.g.nodes[node].data - - def save(self, f: str): - pickle.dump( - self.g, - open('f', 'wb') - ) - - @property - def edgelist(self) -> typing.Generator: - return self.g.edges - - def set_graph_labels(self, labels: dict): - for k, v in labels: - self.g.ndata[k] = v - - def __len__(self): - return len(self.g) - - def filter(self): - pass diff --git a/prairiedog/lemon_graph.py b/prairiedog/lemon_graph.py index 7a344f3..6845684 100644 --- a/prairiedog/lemon_graph.py +++ b/prairiedog/lemon_graph.py @@ -31,7 +31,7 @@ def __init__(self, db_path: str = None, delete_on_exit=False, nosync=True, else: os.makedirs(prairiedog.config.OUTPUT_DIRECTORY, exist_ok=True) self.db_path = DB_PATH - log.debug("Creating LemonGraph with backing file {}".format( + log.info("Initializing LemonGraph object with backing file {}".format( self.db_path)) self.g = LemonGraph.Graph(path=self.db_path, nosync=nosync, noreadahead=noreadahead, readonly=readonly) diff --git a/prairiedog/profiler.py b/prairiedog/profiler.py new file mode 100644 index 0000000..83804bf --- /dev/null +++ b/prairiedog/profiler.py @@ -0,0 +1,26 @@ +import platform +import logging + +log = logging.getLogger("prairiedog") + + +if platform.python_implementation() == 'PyPy': + class Profiler: + """ + PyPy is missing some functions required for pyinstrument + """ + def __init__(self): + log.warning("Initialized empty Profiler since platform is" + " {}".format(platform.python_implementation())) + + def start(self): + pass + + def stop(self): + pass + + def output_text(self, unicode, color): + return "" +else: + import pyinstrument + Profiler = pyinstrument.Profiler diff --git a/prairiedog/test_profiler.py b/prairiedog/test_profiler.py new file mode 100644 index 0000000..b1e4c37 --- /dev/null +++ b/prairiedog/test_profiler.py @@ -0,0 +1,9 @@ +from prairiedog.profiler import Profiler + + +def test_profiler(): + profiler = Profiler() + profiler.start() + profiler.stop() + print(profiler.output_text(unicode=True, color=True)) + assert True diff --git a/requirements.txt b/requirements.txt index 6ef5a58..10ba15a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,11 +2,11 @@ coloredlogs==10.0 decorator==4.4.0 humanfriendly==4.18 networkx==2.3 +cython +dateutils numpy pandas click>=6.0 dill -dgl -torch psutil pyinstrument diff --git a/requirements_dev.txt b/requirements_dev.txt index 7999cd6..604b911 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ -pip==18.1 +pip==19.1.1 bumpversion==0.5.3 wheel==0.32.1 watchdog==0.9.0 @@ -8,8 +8,8 @@ coverage==4.5.1 Sphinx==1.8.1 twine==1.12.1 -pytest==3.8.2 -pytest-runner==4.2 +pytest==4.6.3 +pytest-runner==5.1 pytest-cov codecov diff --git a/tests/test_subgraph_ref.py b/tests/test_subgraph_ref.py index 27d154e..1b57093 100644 --- a/tests/test_subgraph_ref.py +++ b/tests/test_subgraph_ref.py @@ -1,8 +1,7 @@ import pytest import logging -from pyinstrument import Profiler - +from prairiedog.profiler import Profiler from prairiedog.subgraph_ref import SubgraphRef from prairiedog.graph_ref import GraphRef @@ -10,9 +9,12 @@ def _do_subgraph(g, km): + log.info("Setting up SubgraphRef...") sgr = SubgraphRef(g) + log.info("Setting up GraphRef....") gr = GraphRef() + log.info("Updating graph...") sgr.update_graph(km, gr) assert True @@ -23,11 +25,14 @@ def test_subgraph_creation(g, km_short): def test_subgraph_creation_profile(g, km_short): + log.info("Initializing and starting profiler...") profiler = Profiler() profiler.start() _do_subgraph(g, km_short) + log.info("Stopping profiler...") profiler.stop() + log.info("Trying to print profiler test...") print(profiler.output_text(unicode=True, color=True)) diff --git a/tox.ini b/tox.ini index d1aa3e7..5f3a1ba 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35, py36, py37, flake8, +envlist = py35, py36, py37, pypy36, flake8, [flake8] exclude = tests/* @@ -22,6 +22,7 @@ deps = ; requirements.txt with the pinned versions and uncomment the following line: -r{toxinidir}/requirements.txt commands = + pip -V pip install -U pip pip install git+https://github.com/pytries/datrie.git pip install snakemake @@ -29,5 +30,5 @@ commands = echo "In CI, will not install profiler for benchmarks" || \ pip install memory_profiler' bash -ec 'cd lemongraph && python setup.py install' - py.test -v -s --basetemp={envtmpdir} \ + pytest -v -s --basetemp={envtmpdir} \ --cov=prairiedog --cov-report=xml --cov-report=html