From 54e1b62185f406098ffb5695d952ec43543f0eb4 Mon Sep 17 00:00:00 2001 From: Jason Parham Date: Tue, 7 Dec 2021 12:50:29 -0800 Subject: [PATCH] Updates for Flann and Prometheus monitoring --- .github/workflows/docker-publish.yaml | 2 +- devops/build.sh | 4 +++ requirements/runtime.txt | 2 +- wbia/control/IBEISControl.py | 1 + wbia/other/ibsfuncs.py | 21 ++++++++++++ wbia/web/apis_query.py | 49 +++++++++++++++++++++++++-- wbia/web/prometheus.py | 22 +++++++++++- 7 files changed, 96 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 5611acc28e..378fdf25e1 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -17,7 +17,7 @@ jobs: strategy: matrix: images: - - wbia-base wbia-provision wbia wildbook-ia + - wbia-base wbia-provision wbia steps: - uses: actions/checkout@v2 diff --git a/devops/build.sh b/devops/build.sh index 0e0d604c7c..4b89864e2f 100755 --- a/devops/build.sh +++ b/devops/build.sh @@ -16,14 +16,17 @@ cd ${ROOT_LOC} while [ $# -ge 1 ]; do if [ "$1" == "wbia-base" ]; then docker build \ + --compress \ -t wildme/wbia-base:latest \ base elif [ "$1" == "wbia-provision" ]; then docker build \ + --compress \ -t wildme/wbia-provision:latest \ provision elif [ "$1" == "wbia" ] || [ "$1" == "wildbook-ia" ]; then docker build \ + --compress \ -t wildme/wbia:latest \ -t wildme/wildbook-ia:latest \ --no-cache \ @@ -31,6 +34,7 @@ while [ $# -ge 1 ]; do elif [ "$1" == "wbia-develop" ]; then cd ../ docker build \ + --compress \ -t wildme/wbia:develop \ devops/develop cd devops/ diff --git a/requirements/runtime.txt b/requirements/runtime.txt index f393575e38..728a1dd5c6 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -120,7 +120,7 @@ visdom wbia-lightnet wbia-pydarknet wbia-pyflann -wbia-pyhesaff==3.3.0 +wbia-pyhesaff wbia-utool wbia-vtool diff --git a/wbia/control/IBEISControl.py b/wbia/control/IBEISControl.py index 6baa52b60e..85fbf535df 100644 --- a/wbia/control/IBEISControl.py +++ b/wbia/control/IBEISControl.py @@ -396,6 +396,7 @@ def __init__( if not ut.get_argflag('--noclean') and not self.readonly: # self._init_burned_in_species() self._clean_species() + self._clean_flann() self.job_manager = None # Hack for changing the way chips compute diff --git a/wbia/other/ibsfuncs.py b/wbia/other/ibsfuncs.py index d52d685c2a..fe7246f1cf 100644 --- a/wbia/other/ibsfuncs.py +++ b/wbia/other/ibsfuncs.py @@ -6615,6 +6615,27 @@ def _clean_species(ibs): return species_mapping_dict +@register_ibs_method +def _clean_flann(ibs): + if ut.VERBOSE: + logger.info('[_clean_flann] Cleaning...') + + if ibs.readonly: + # SUPER HACK + return + + flann_cachedir = ibs.get_flann_cachedir() + flann_locks = ut.glob('%s/*.lock' % (flann_cachedir,)) + + flann_delete = [] + for flann_lock in flann_locks: + flann_base = flann_lock.replace('.lock', '*') + flann_delete += ut.glob(flann_base) + + for path in flann_delete: + ut.delete(path) + + @register_ibs_method def get_annot_encounter_text(ibs, aids): """Encounter identifier for annotations""" diff --git a/wbia/web/apis_query.py b/wbia/web/apis_query.py index b58c417a76..409ae6e57b 100644 --- a/wbia/web/apis_query.py +++ b/wbia/web/apis_query.py @@ -19,7 +19,7 @@ import requests from datetime import datetime -(print, rrr, profile) = ut.inject2(__name__) +# (print, rrr, profile) = ut.inject2(__name__) logger = logging.getLogger('wbia') CLASS_INJECT_KEY, register_ibs_method = controller_inject.make_ibs_register_decorator( @@ -32,6 +32,9 @@ GRAPH_CLIENT_PEEK = 100 +RENDER_STATUS = None + + @register_ibs_method @register_api('/api/query/annot/rowid/', methods=['GET']) def get_recognition_query_aids(ibs, is_known, species=None): @@ -278,8 +281,10 @@ def ensure_review_image( if hasattr(qreq_, 'render_single_result'): image = qreq_.render_single_result(cm, aid, **render_config) - else: + elif hasattr(cm, 'render_single_annotmatch'): image = cm.render_single_annotmatch(qreq_, aid, **render_config) + else: + image = ibs.get_annot_chips(aid) # image = vt.crop_out_imgfill(image, fillval=(255, 255, 255), thresh=64) cv2.imwrite(match_thumb_filepath, image) return image, match_thumb_filepath @@ -837,11 +842,22 @@ def query_chips_graph_complete(ibs, aid_list, query_config_dict={}, k=5, **kwarg def log_render_status(ibs, *args): import os + global RENDER_STATUS + + if RENDER_STATUS is None: + RENDER_STATUS = ibs._init_render_status() + assert RENDER_STATUS is not None + json_log_path = ibs.get_logdir_local() json_log_filename = 'render.log' json_log_filepath = os.path.join(json_log_path, json_log_filename) logger.info('Logging renders added to: %r' % (json_log_filepath,)) + status = '%s' % (args[-1],) + if status not in RENDER_STATUS: + RENDER_STATUS[status] = 0 + RENDER_STATUS[status] += 1 + try: with open(json_log_filepath, 'a') as json_log_file: line = ','.join(['%s' % (arg,) for arg in args]) @@ -851,6 +867,32 @@ def log_render_status(ibs, *args): logger.info('WRITE RENDER.LOG FAILED') +@register_ibs_method +def _init_render_status(ibs): + import os + + json_log_path = ibs.get_logdir_local() + json_log_filename = 'render.log' + json_log_filepath = os.path.join(json_log_path, json_log_filename) + + status_dict = {} + try: + with open(json_log_filepath, 'r') as json_log_file: + for line in json_log_file.readlines(): + try: + line = line.strip().split(',') + status = line[-1] + if status not in status_dict: + status_dict[status] = 0 + status_dict[status] += 1 + except Exception: + pass + except Exception: + pass + + return status_dict + + @register_ibs_method @register_api('/api/query/graph/', methods=['GET', 'POST']) def query_chips_graph( @@ -1055,6 +1097,7 @@ def convert_to_uuid(nid): ut.printex(ex, iswarning=True) log_render_status( ibs, + ut.timestamp(), cm.qaid, daid, quuid, @@ -1083,6 +1126,7 @@ def convert_to_uuid(nid): ut.printex(ex, iswarning=True) log_render_status( ibs, + ut.timestamp(), cm.qaid, daid, quuid, @@ -1111,6 +1155,7 @@ def convert_to_uuid(nid): ut.printex(ex, iswarning=True) log_render_status( ibs, + ut.timestamp(), cm.qaid, daid, quuid, diff --git a/wbia/web/prometheus.py b/wbia/web/prometheus.py index e489b6eb2b..e2d6286920 100644 --- a/wbia/web/prometheus.py +++ b/wbia/web/prometheus.py @@ -2,6 +2,7 @@ import logging from prometheus_client import Info, Gauge, Counter, Enum, Histogram # NOQA from wbia.control import controller_inject +from wbia.web.apis_query import RENDER_STATUS # NOQA import wbia.constants as const import utool as ut @@ -58,10 +59,15 @@ 'Number of species in WBIA database', ['name'], ), + 'renders': Gauge( + 'wbia_assets_renders', + 'Number of rendered images in WBIA database', + ['name', 'status'], + ), 'engine': Gauge( 'wbia_engine_jobs', 'Job engine status', - ['status', 'name', 'endpoint'], + ['name', 'status', 'endpoint'], ), 'process': Gauge( 'wbia_engine_dead_process', @@ -159,6 +165,11 @@ def prometheus_update(ibs, *args, **kwargs): global PROMETHEUS_COUNTER + global RENDER_STATUS + + if RENDER_STATUS is None: + RENDER_STATUS = ibs._init_render_status() + PROMETHEUS_COUNTER = PROMETHEUS_COUNTER + 1 # NOQA # logger.info('PROMETHEUS LIMIT %d / %d' % (PROMETHEUS_COUNTER, PROMETHEUS_LIMIT, )) @@ -363,6 +374,15 @@ def prometheus_update(ibs, *args, **kwargs): except Exception: pass + try: + for status in RENDER_STATUS: + number = RENDER_STATUS[status] + PROMETHEUS_DATA['renders'].labels( + status=status, name=container_name + ).set(number) + except Exception: + pass + try: # logger.info(ut.repr3(status_dict)) process_status_dict = ibs.get_process_alive_status()