From d58f3f8255149c29c95e71177b43ea7edab455ba Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Fri, 28 Jun 2024 23:32:23 -0500 Subject: [PATCH] add code to download cog --- cdrhook/server.py | 137 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/cdrhook/server.py b/cdrhook/server.py index 5bc008f..cac8c3f 100644 --- a/cdrhook/server.py +++ b/cdrhook/server.py @@ -145,6 +145,128 @@ def check_uncharted_event(event_id): send_message(message, f'{config["prefix"]}download') +def process_cog(cog_id): + """ + Check to see if we have all the information for the cog. If we do then + write it to a temporary file and fire the download event. + """ + headers = {'Authorization': f'Bearer {config["cdr_token"]}'} + + # create the result + result = { + "system": "ncsa", + "system_version": "0.0.0", + "cog_id": cog_id, + "line_feature_results": [ ], + "point_feature_results": [ ], + "polygon_feature_results": [ ], + "cog_area_extractions": [ ], + "cog_metadata_extractions": [ ] + } + + # get the system information for map area (area_extraction) + r = requests.get(f"{cdr_url}/v1/features/{cog_id}/system_versions?type=area_extraction", headers=headers) + r.raise_for_status() + data = r.json() + logging.debug("Got system versions for area_extraction : %s", data) + + # check if there is a map area + system = None + for version in data: + if version[0] == "uncharted": + system = version + if system: + logging.debug(f"MapArea found from {system[0]} version {system[1]}") + else: + logging.debug("No map area found") + + # download map area + map_area = [ ] + if system: + r = requests.get(f"{cdr_url}/v1/features/{cog_id}/area_extractions?system_version={system[0]}__{system[1]}", headers=headers) + r.raise_for_status() + for item in r.json(): + if item["system"] != system[0] or item["system_version"] != system[1]: + continue + if item["category"] == "map_area": + map_area.append(item) + result["cog_area_extractions"].extend(map_area) + + # get the system information for legends + r = requests.get(f"{cdr_url}/v1/features/{cog_id}/system_versions?type=legend_item", headers=headers) + r.raise_for_status() + data = r.json() + logging.debug("Got system versions legend_item : %s", data) + + # check if there is a legend + system = None + for version in data: + if version[0] == "polymer": + system = version + if not system: + for version in data: + if version[0] == "uncharted": + system = version + if system: + logging.debug(f"Legend found from {system[0]} version {system[1]}") + else: + logging.debug("No legend found") + + # download legend + polygon_legend_area = [ ] + line_point_legend_area = [ ] + if system: + r = requests.get(f"{cdr_url}/v1/features/{cog_id}/legend_items?system_version={system[0]}__{system[1]}", headers=headers) + r.raise_for_status() + # legend does not have any filtering, so we do it locally as well as extract the polygons/point_line + for item in r.json(): + if item["system"] != system[0] or item["system_version"] != system[1]: + continue + if item["category"] == "line_point_legend_area": + line_point_legend_area.append(item) + elif item["category"] == "polygon": + polygon_legend_area.append(item) + result["cog_area_extractions"].extend(polygon_legend_area) + result["cog_area_extractions"].extend(line_point_legend_area) + + # write the cog_area to disk + folder = os.path.join(cog_id[0:2], cog_id[2:4]) + filepart = os.path.join(folder, cog_id) + filename = os.path.join("/data", f"{filepart}.cog_area.json") + os.makedirs(os.path.dirname(filename) , exist_ok=True) + with open(filename, "w") as outputfile: + json.dump(result, outputfile) + + # get the basic information + r = requests.get(f"{cdr_url}/v1/maps/cog/{cog_id}", headers=headers) + r.raise_for_status() + cog_info = r.json() + + # send the download event + firemodels = [ ] + for k, v in config["models"].items(): + goodmodel = True + if "map_area" in v and not map_area: + logging.debug("Skipping %s because of map_area", k) + goodmodel = False + if "polygon_legend_area" in v and not polygon_legend_area: + logging.debug("Skipping %s because of polygon_legend_area", k) + goodmodel = False + if "line_point_legend_area" in v and not line_point_legend_area: + logging.debug("Skipping %s because of line_point_legend_area", k) + goodmodel = False + if goodmodel: + firemodels.append(k) + + message = { + "cog_id": cog_id, + "cog_url": cog_info["cog_url"], + "map_area": f'{config["callback_url"]}/download/{filepart}.cog_area.json', + "models": firemodels + } + logging.info("Firing download event for %s '%s'", cog_id, json.dumps(message)) + send_message(message, f'{config["prefix"]}download') + # ---------------------------------------------------------------------- # Process incoming requests # ---------------------------------------------------------------------- @@ -175,6 +297,17 @@ def hook(): return {"ok": "success"} +@auth.login_required +def cog(id): + """ + Process the cog + """ + logging.info(f"Received process cog for {id}") + send_message({"event": "ncsacog", "cog_id": id}, f'{config["prefix"]}cdrhook') + + return {"ok": "success"} + + @auth.login_required def download(filename): """ @@ -199,6 +332,9 @@ def cdrhook_callback(channel, method, properties, body): logging.error("No event in message") elif data.get("event") == "ping": logging.debug("ping/pong") + elif data.get("event") == "ncsacog": + cog_id = data.get("cog_id", "").strip() + process_cog(cog_id) elif data.get("event") == "map.process": logging.debug("ignoring map.process") elif data.get("event") == "feature.process": @@ -315,6 +451,7 @@ def create_app(): path = urllib.parse.urlparse(config["callback_url"]).path app.route(os.path.join(path, "hook"), methods=['POST'])(hook) app.route(os.path.join(path, "download", ""), methods=['GET'])(download) + app.route(os.path.join(path, "cog", ""), methods=['POST'])(cog) # start daemon thread for rabbitmq thread = threading.Thread(target=cdrhook_listener, args=(config,))