Updates for Flann and Prometheus monitoring

.github/workflows/docker-publish.yaml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         images:
-          - wbia-base wbia-provision wbia wildbook-ia
+          - wbia-base wbia-provision wbia
 
     steps:
       - uses: actions/checkout@v2

devops/build.sh

@@ -16,21 +16,25 @@ cd ${ROOT_LOC}
 while [ $# -ge 1 ]; do
     if [ "$1" == "wbia-base" ]; then
         docker build \
+            --compress \
             -t wildme/wbia-base:latest \
             base
     elif [ "$1" == "wbia-provision" ]; then
         docker build \
+            --compress \
             -t wildme/wbia-provision:latest \
             provision
     elif [ "$1" == "wbia" ] || [ "$1" == "wildbook-ia" ]; then
         docker build \
+            --compress \
             -t wildme/wbia:latest \
             -t wildme/wildbook-ia:latest \
             --no-cache \
             .
     elif [ "$1" == "wbia-develop" ]; then
         cd ../
         docker build \
+            --compress \
             -t wildme/wbia:develop \
             devops/develop
         cd devops/

requirements/runtime.txt

@@ -120,7 +120,7 @@ visdom
 wbia-lightnet
 wbia-pydarknet
 wbia-pyflann
-wbia-pyhesaff==3.3.0
+wbia-pyhesaff
 
 wbia-utool
 wbia-vtool

wbia/control/IBEISControl.py

@@ -396,6 +396,7 @@ def __init__(
         if not ut.get_argflag('--noclean') and not self.readonly:
             # self._init_burned_in_species()
             self._clean_species()
+            self._clean_flann()
         self.job_manager = None
 
         # Hack for changing the way chips compute

wbia/other/ibsfuncs.py

@@ -6615,6 +6615,27 @@ def _clean_species(ibs):
     return species_mapping_dict
 
 
+@register_ibs_method
+def _clean_flann(ibs):
+    if ut.VERBOSE:
+        logger.info('[_clean_flann] Cleaning...')
+
+    if ibs.readonly:
+        # SUPER HACK
+        return
+
+    flann_cachedir = ibs.get_flann_cachedir()
+    flann_locks = ut.glob('%s/*.lock' % (flann_cachedir,))
+
+    flann_delete = []
+    for flann_lock in flann_locks:
+        flann_base = flann_lock.replace('.lock', '*')
+        flann_delete += ut.glob(flann_base)
+
+    for path in flann_delete:
+        ut.delete(path)
+
+
 @register_ibs_method
 def get_annot_encounter_text(ibs, aids):
     """Encounter identifier for annotations"""

wbia/web/apis_query.py

@@ -19,7 +19,7 @@
 import requests
 from datetime import datetime
 
-(print, rrr, profile) = ut.inject2(__name__)
+# (print, rrr, profile) = ut.inject2(__name__)
 logger = logging.getLogger('wbia')
 
 CLASS_INJECT_KEY, register_ibs_method = controller_inject.make_ibs_register_decorator(
@@ -32,6 +32,9 @@
 GRAPH_CLIENT_PEEK = 100
 
 
+RENDER_STATUS = None
+
+
 @register_ibs_method
 @register_api('/api/query/annot/rowid/', methods=['GET'])
 def get_recognition_query_aids(ibs, is_known, species=None):
@@ -278,8 +281,10 @@ def ensure_review_image(
 
         if hasattr(qreq_, 'render_single_result'):
             image = qreq_.render_single_result(cm, aid, **render_config)
-        else:
+        elif hasattr(cm, 'render_single_annotmatch'):
             image = cm.render_single_annotmatch(qreq_, aid, **render_config)
+        else:
+            image = ibs.get_annot_chips(aid)
         # image = vt.crop_out_imgfill(image, fillval=(255, 255, 255), thresh=64)
         cv2.imwrite(match_thumb_filepath, image)
     return image, match_thumb_filepath
@@ -837,11 +842,22 @@ def query_chips_graph_complete(ibs, aid_list, query_config_dict={}, k=5, **kwarg
 def log_render_status(ibs, *args):
     import os
 
+    global RENDER_STATUS
+
+    if RENDER_STATUS is None:
+        RENDER_STATUS = ibs._init_render_status()
+    assert RENDER_STATUS is not None
+
     json_log_path = ibs.get_logdir_local()
     json_log_filename = 'render.log'
     json_log_filepath = os.path.join(json_log_path, json_log_filename)
     logger.info('Logging renders added to: %r' % (json_log_filepath,))
 
+    status = '%s' % (args[-1],)
+    if status not in RENDER_STATUS:
+        RENDER_STATUS[status] = 0
+    RENDER_STATUS[status] += 1
+
     try:
         with open(json_log_filepath, 'a') as json_log_file:
             line = ','.join(['%s' % (arg,) for arg in args])
@@ -851,6 +867,32 @@ def log_render_status(ibs, *args):
         logger.info('WRITE RENDER.LOG FAILED')
 
 
+@register_ibs_method
+def _init_render_status(ibs):
+    import os
+
+    json_log_path = ibs.get_logdir_local()
+    json_log_filename = 'render.log'
+    json_log_filepath = os.path.join(json_log_path, json_log_filename)
+
+    status_dict = {}
+    try:
+        with open(json_log_filepath, 'r') as json_log_file:
+            for line in json_log_file.readlines():
+                try:
+                    line = line.strip().split(',')
+                    status = line[-1]
+                    if status not in status_dict:
+                        status_dict[status] = 0
+                    status_dict[status] += 1
+                except Exception:
+                    pass
+    except Exception:
+        pass
+
+    return status_dict
+
+
 @register_ibs_method
 @register_api('/api/query/graph/', methods=['GET', 'POST'])
 def query_chips_graph(
@@ -1055,6 +1097,7 @@ def convert_to_uuid(nid):
                         ut.printex(ex, iswarning=True)
                     log_render_status(
                         ibs,
+                        ut.timestamp(),
                         cm.qaid,
                         daid,
                         quuid,
@@ -1083,6 +1126,7 @@ def convert_to_uuid(nid):
                         ut.printex(ex, iswarning=True)
                     log_render_status(
                         ibs,
+                        ut.timestamp(),
                         cm.qaid,
                         daid,
                         quuid,
@@ -1111,6 +1155,7 @@ def convert_to_uuid(nid):
                         ut.printex(ex, iswarning=True)
                     log_render_status(
                         ibs,
+                        ut.timestamp(),
                         cm.qaid,
                         daid,
                         quuid,

wbia/web/prometheus.py

@@ -2,6 +2,7 @@
 import logging
 from prometheus_client import Info, Gauge, Counter, Enum, Histogram  # NOQA
 from wbia.control import controller_inject
+from wbia.web.apis_query import RENDER_STATUS  # NOQA
 import wbia.constants as const
 import utool as ut
 
@@ -58,10 +59,15 @@
         'Number of species in WBIA database',
         ['name'],
     ),
+    'renders': Gauge(
+        'wbia_assets_renders',
+        'Number of rendered images in WBIA database',
+        ['name', 'status'],
+    ),
     'engine': Gauge(
         'wbia_engine_jobs',
         'Job engine status',
-        ['status', 'name', 'endpoint'],
+        ['name', 'status', 'endpoint'],
     ),
     'process': Gauge(
         'wbia_engine_dead_process',
@@ -159,6 +165,11 @@ def prometheus_update(ibs, *args, **kwargs):
 
             global PROMETHEUS_COUNTER
 
+            global RENDER_STATUS
+
+            if RENDER_STATUS is None:
+                RENDER_STATUS = ibs._init_render_status()
+
             PROMETHEUS_COUNTER = PROMETHEUS_COUNTER + 1  # NOQA
             # logger.info('PROMETHEUS LIMIT %d / %d' % (PROMETHEUS_COUNTER, PROMETHEUS_LIMIT, ))
 
@@ -363,6 +374,15 @@ def prometheus_update(ibs, *args, **kwargs):
                 except Exception:
                     pass
 
+                try:
+                    for status in RENDER_STATUS:
+                        number = RENDER_STATUS[status]
+                        PROMETHEUS_DATA['renders'].labels(
+                            status=status, name=container_name
+                        ).set(number)
+                except Exception:
+                    pass
+
                 try:
                     # logger.info(ut.repr3(status_dict))
                     process_status_dict = ibs.get_process_alive_status()