Merge branch 'develop'

.dockerfiles/docker-entrypoint.sh

@@ -6,6 +6,8 @@ set -e
 if [ -d /code ]; then
     echo "*** $0 --- Uninstalling wildbook-ia"
     pip uninstall -y wildbook-ia
+    echo "*** $0 --- Uninstalling sentry_sdk (in development)"
+    pip uninstall -y sentry_sdk
     echo "*** $0 --- Installing development version of wildbook-ia at /code"
     pushd /code && pip install -e ".[tests,postgres]" && popd
 fi

.dockerfiles/init-db.sh

@@ -3,7 +3,7 @@
 set -e
 
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    CREATE USER $DB_USER WITH PASSWORD '$DB_PASSWORD';
+    CREATE USER $DB_USER WITH SUPERUSER PASSWORD '$DB_PASSWORD';
     CREATE DATABASE $DB_NAME;
     GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER;
 EOSQL

.github/workflows/nightly.yml

@@ -5,6 +5,9 @@ on:
   push:
     paths:
       - '.github/workflows/nightly.yml'
+      - 'devops/**'
+      - 'Dockerfile'
+      - '.dockerfiles/*'
   pull_request:
     paths:
       - '.github/workflows/nightly.yml'
@@ -19,6 +22,8 @@ jobs:
     name: DevOps nightly image build
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 2
+      fail-fast: false
       matrix:
         images:
           - wildbook-ia

.github/workflows/python-publish.yml

@@ -9,6 +9,8 @@ jobs:
     name: Build on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      max-parallel: 2
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: [3.7]

.github/workflows/testing.yml

@@ -9,6 +9,8 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 3
+      fail-fast: false
       matrix:
         # For speed, we choose one version and that should be the lowest common denominator
         python-version: [3.6, 3.7, 3.8]
@@ -43,14 +45,34 @@ jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
+      max-parallel: 6
+      fail-fast: false
       matrix:
         os: [ubuntu-latest] # Disable "macos-latest" for now
         # For speed, we choose one version and that should be the lowest common denominator
         python-version: [3.6, 3.7, 3.8]
+        postgres-uri: ['', 'postgresql://postgres:wbia@localhost:5432/postgres']
+
+    services:
+      db:
+        image: postgres:10
+        env:
+          POSTGRES_PASSWORD: wbia
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
 
     steps:
       # Checkout and env setup
       - uses: actions/checkout@v2
+      - name: Install pgloader
+        if: matrix.postgres-uri
+        run: sudo apt-get install pgloader
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
@@ -65,12 +87,14 @@ jobs:
 
       # Install and test
       - name: Install the project
-        run: pip install -e .[tests]
+        run: pip install -e .[tests,postgres]
       - name: Test with pytest
         run: |
           mkdir -p data/work
           python -m wbia --set-workdir data/work --preload-exit
-          pytest --slow
+          pytest --slow --web-tests --with-postgres-uri=$POSTGRES_URI
+        env:
+          POSTGRES_URI: ${{ matrix.postgres-uri }}
 
   on-failure:
     # This is not in the 'test' job itself because it would otherwise notify once per matrix combination.

.gitignore

@@ -36,7 +36,7 @@ testdb*/
 tmp.txt
 vsone.*.cPkl
 vsone.*.json
-wbia/DEPCACHE*/
+wbia/dtool/DEPCACHE*/
 
 # Translations
 *.mo
@@ -67,3 +67,4 @@ branch.sync.sh
 
 # setuptools_scm version discovery writes the version to <pkg>/_version.py
 **/_version.py
+map.png

Dockerfile

@@ -67,6 +67,10 @@ RUN set -x \
        libxext6 \
        #: opencv2 dependency
        libgl1 \
+       #: required to stop prompting by pgloader
+       libssl1.0.0 \
+       #: sqlite->postgres dependency
+       pgloader \
        #: dev debug dependency
        #: python3-dev required to build 'annoy'
        python3-dev \
@@ -118,7 +122,12 @@ EXPOSE 5000
 # Move to the workdir
 WORKDIR /data
 
+# Set the "workdir"
+RUN python3 -m wbia --set-workdir /data --preload-exit
+
 COPY .dockerfiles/docker-entrypoint.sh /docker-entrypoint.sh
 
+ENV WBIA_DB_DIR="/data/db"
+
 ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["python3", "-m", "wbia.dev", "--dbdir", "/data/db", "--logdir", "/data/logs/", "--web", "--port", "5000", "--web-deterministic-ports", "--containerized", "--cpudark", "--production"]
+CMD ["python3", "-m", "wbia.dev", "--dbdir", "$WBIA_DB_DIR", "--logdir", "/data/logs/", "--web", "--port", "5000", "--web-deterministic-ports", "--containerized", "--cpudark", "--production"]

README.rst

@@ -12,7 +12,7 @@ use in computer vision algorithms. It aims to compute who an animal is, what
 species an animal is, and where an animal is with the ultimate goal being to
 ask important why biological questions.
 
-This project is the Machine Learning (ML) / computer vision component of the WildBook project: See https://github.com/WildbookOrg/.  This project is an actively maintained fork of the popular IBEIS (Image Based Ecological Information System) software suite for wildlife conservation.  The original IBEIS project is maintained by Jon Crall (@Erotemic) at https://github.com/Erotemic/ibeis.  The IBEIS toolkit originally was a wrapper around HotSpotter, which original binaries can be downloaded from: http://cs.rpi.edu/hotspotter/
+This project is the Machine Learning (ML) / computer vision component of the WildBook project: See https://github.com/WildMeOrg/.  This project is an actively maintained fork of the popular IBEIS (Image Based Ecological Information System) software suite for wildlife conservation.  The original IBEIS project is maintained by Jon Crall (@Erotemic) at https://github.com/Erotemic/ibeis.  The IBEIS toolkit originally was a wrapper around HotSpotter, which original binaries can be downloaded from: http://cs.rpi.edu/hotspotter/
 
 Currently the system is build around and SQLite database, a web GUI,
 and matplotlib visualizations. Algorithms employed are: convolutional neural network
@@ -53,7 +53,7 @@ We highly recommend using a Python virtual environment: https://docs.python-guid
 Documentation
 ~~~~~~~~~~~~~
 
-The documentation is built and available online at `wildbookorg.github.io/wildbook-ia/ <http://wildbookorg.github.io/wildbook-ia/>`_. However, if you need to build a local copy of the source, the following instructions can be used.
+The documentation is built and available online at `wildmeorg.github.io/wildbook-ia/ <http://wildmeorg.github.io/wildbook-ia/>`_. However, if you need to build a local copy of the source, the following instructions can be used.
 
 .. code:: bash
 
@@ -87,76 +87,76 @@ This project depends on an array of other repositories for functionality.
 
 First Party Toolkits (Required)
 
-* https://github.com/WildbookOrg/wbia-utool
+* https://github.com/WildMeOrg/wbia-utool
 
-* https://github.com/WildbookOrg/wbia-vtool
+* https://github.com/WildMeOrg/wbia-vtool
 
 First Party Dependencies for Third Party Libraries (Required)
 
-* https://github.com/WildbookOrg/wbia-tpl-pyhesaff
+* https://github.com/WildMeOrg/wbia-tpl-pyhesaff
 
-* https://github.com/WildbookOrg/wbia-tpl-pyflann
+* https://github.com/WildMeOrg/wbia-tpl-pyflann
 
-* https://github.com/WildbookOrg/wbia-tpl-pydarknet
+* https://github.com/WildMeOrg/wbia-tpl-pydarknet
 
-* https://github.com/WildbookOrg/wbia-tpl-pyrf
+* https://github.com/WildMeOrg/wbia-tpl-pyrf
 
 First Party Plug-ins (Optional)
 
-* https://github.com/WildbookOrg/wbia-plugin-cnn
+* https://github.com/WildMeOrg/wbia-plugin-cnn
 
-* https://github.com/WildbookOrg/wbia-plugin-flukematch
+* https://github.com/WildMeOrg/wbia-plugin-flukematch
 
-* https://github.com/WildbookOrg/wbia-plugin-deepsense
+* https://github.com/WildMeOrg/wbia-plugin-deepsense
 
-* https://github.com/WildbookOrg/wbia-plugin-finfindr
+* https://github.com/WildMeOrg/wbia-plugin-finfindr
 
-* https://github.com/WildbookOrg/wbia-plugin-curvrank
+* https://github.com/WildMeOrg/wbia-plugin-curvrank
 
-    + https://github.com/WildbookOrg/wbia-tpl-curvrank
+    + https://github.com/WildMeOrg/wbia-tpl-curvrank
 
-* https://github.com/WildbookOrg/wbia-plugin-kaggle7
+* https://github.com/WildMeOrg/wbia-plugin-kaggle7
 
-    + https://github.com/WildbookOrg/wbia-tpl-kaggle7
+    + https://github.com/WildMeOrg/wbia-tpl-kaggle7
 
-* https://github.com/WildbookOrg/wbia-plugin-2d-orientation
+* https://github.com/WildMeOrg/wbia-plugin-2d-orientation
 
-    + https://github.com/WildbookOrg/wbia-tpl-2d-orientation
+    + https://github.com/WildMeOrg/wbia-tpl-2d-orientation
 
-* https://github.com/WildbookOrg/wbia-plugin-lca
+* https://github.com/WildMeOrg/wbia-plugin-lca
 
-    + https://github.com/WildbookOrg/wbia-tpl-lca
+    + https://github.com/WildMeOrg/wbia-tpl-lca
 
 Deprecated Toolkits (Deprecated)
-* https://github.com/WildbookOrg/wbia-deprecate-ubelt
+* https://github.com/WildMeOrg/wbia-deprecate-ubelt
 
-* https://github.com/WildbookOrg/wbia-deprecate-dtool
+* https://github.com/WildMeOrg/wbia-deprecate-dtool
 
-* https://github.com/WildbookOrg/wbia-deprecate-guitool
+* https://github.com/WildMeOrg/wbia-deprecate-guitool
 
-* https://github.com/WildbookOrg/wbia-deprecate-plottool
+* https://github.com/WildMeOrg/wbia-deprecate-plottool
 
-* https://github.com/WildbookOrg/wbia-deprecate-detecttools
+* https://github.com/WildMeOrg/wbia-deprecate-detecttools
 
-* https://github.com/WildbookOrg/wbia-deprecate-plugin-humpbacktl
+* https://github.com/WildMeOrg/wbia-deprecate-plugin-humpbacktl
 
-* https://github.com/WildbookOrg/wbia-deprecate-tpl-lightnet
+* https://github.com/WildMeOrg/wbia-deprecate-tpl-lightnet
 
-* https://github.com/WildbookOrg/wbia-deprecate-tpl-brambox
+* https://github.com/WildMeOrg/wbia-deprecate-tpl-brambox
 
 Plug-in Templates (Reference)
 
-* https://github.com/WildbookOrg/wbia-plugin-template
+* https://github.com/WildMeOrg/wbia-plugin-template
 
-* https://github.com/WildbookOrg/wbia-plugin-id-example
+* https://github.com/WildMeOrg/wbia-plugin-id-example
 
 Miscellaneous (Reference)
 
-* https://github.com/WildbookOrg/wbia-pypkg-build
+* https://github.com/WildMeOrg/wbia-pypkg-build
 
-* https://github.com/WildbookOrg/wbia-project-website
+* https://github.com/WildMeOrg/wbia-project-website
 
-* https://github.com/WildbookOrg/wbia-aws-codedeploy
+* https://github.com/WildMeOrg/wbia-aws-codedeploy
 
 Citation
 --------
@@ -259,8 +259,8 @@ To run doctests with `+REQUIRES(--web-tests)` do:
 
     pytest --web-tests
 
-.. |Build| image:: https://img.shields.io/github/workflow/status/WildbookOrg/wildbook-ia/Build%20and%20upload%20to%20PyPI/master
-    :target: https://github.com/WildbookOrg/wildbook-ia/actions?query=branch%3Amaster+workflow%3A%22Build+and+upload+to+PyPI%22
+.. |Build| image:: https://img.shields.io/github/workflow/status/WildMeOrg/wildbook-ia/Build%20and%20upload%20to%20PyPI/master
+    :target: https://github.com/WildMeOrg/wildbook-ia/actions?query=branch%3Amaster+workflow%3A%22Build+and+upload+to+PyPI%22
     :alt: Build and upload to PyPI (master)
 
 .. |Pypi| image:: https://img.shields.io/pypi/v/wildbook-ia.svg