From b5a6f38b0908abe3940f4d62229d6f32882d4682 Mon Sep 17 00:00:00 2001 From: Shankari Date: Tue, 22 Mar 2022 15:39:27 -0700 Subject: [PATCH] Create a script that exports user travel diaries as geojson This reuses the same method as the the code to generate the diary for a particular day for display in the UI. The related For an example of working with the data, please see https://github.com/e-mission/e-mission-eval-private-data/blob/master/OpenPATH_geojson_example.ipynb @PatGendre in case this helps --- ...extract_geojson_for_time_range_and_user.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 bin/debug/extract_geojson_for_time_range_and_user.py diff --git a/bin/debug/extract_geojson_for_time_range_and_user.py b/bin/debug/extract_geojson_for_time_range_and_user.py new file mode 100644 index 000000000..2b9ec2ce3 --- /dev/null +++ b/bin/debug/extract_geojson_for_time_range_and_user.py @@ -0,0 +1,90 @@ +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +# Exports all data for the particular group of users for the particular day range +# Exports them as geojson using the `emission.analysis.plotting.geojson.geojson_feature_converter` method +# Note that you probably want to edit `conf/analysis/debug.conf.json.sample` to +# turn off `output.conversion.validityAssertions` to avoid running into asserts +# while generating the geojson files +from future import standard_library +standard_library.install_aliases() +from builtins import * +import sys +import logging +logging.basicConfig(level=logging.DEBUG) +import gzip + +import uuid +import datetime as pydt +import json +import bson.json_util as bju +import arrow +import argparse + +import emission.core.wrapper.user as ecwu +import emission.storage.timeseries.abstract_timeseries as esta +import emission.storage.timeseries.timequery as estt +import emission.storage.decorations.user_queries as esdu +import emission.storage.timeseries.cache_series as estcs +# only needed to read the motion_activity +# https://github.com/e-mission/e-mission-docs/issues/356#issuecomment-520630934 +import emission.net.usercache.abstract_usercache as enua +import emission.analysis.plotting.geojson.geojson_feature_converter as gfc + +def export_geojson(user_id, start_day_str, end_day_str, timezone, file_name): + logging.info("Extracting geojson for user %s day %s -> %s and saving to file %s" % + (user_id, start_day_str, end_day_str, file_name)) + + # day_dt = pydt.datetime.strptime(day_str, "%Y-%m-%d").date() + start_day_ts = arrow.get(start_day_str).replace(tzinfo=timezone).timestamp + end_day_ts = arrow.get(end_day_str).replace(tzinfo=timezone).timestamp + logging.debug("start_day_ts = %s (%s), end_day_ts = %s (%s)" % + (start_day_ts, arrow.get(start_day_ts).to(timezone), + end_day_ts, arrow.get(end_day_ts).to(timezone))) + + ts = esta.TimeSeries.get_time_series(user_id) + loc_time_query = estt.TimeQuery("data.ts", start_day_ts, end_day_ts) + user_gj = gfc.get_geojson_for_ts(user_id, start_day_ts, end_day_ts) + + geojson_filename = "%s_%s.gz" % (file_name, user_id) + with gzip.open(geojson_filename, "wt") as gcfd: + json.dump(user_gj, + gcfd, default=bju.default, allow_nan=False, indent=4) + +def export_geojson_for_users(user_id_list, args): + for curr_uuid in user_id_list: + if curr_uuid != '': + logging.info("=" * 50) + export_geojson(user_id=curr_uuid, start_day_str=args.start_day, + end_day_str= args.end_day, timezone=args.timezone, + file_name=args.file_prefix) + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + parser = argparse.ArgumentParser(prog="extract_geojson_for_day_range_and_user") + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-e", "--user_email", nargs="+") + group.add_argument("-u", "--user_uuid", nargs="+") + group.add_argument("-a", "--all", action="store_true") + group.add_argument("-f", "--file") + + parser.add_argument("--timezone", default="UTC") + parser.add_argument("start_day", help="start day in utc - e.g. 'YYYY-MM-DD'" ) + parser.add_argument("end_day", help="start day in utc - e.g. 'YYYY-MM-DD'" ) + parser.add_argument("file_prefix", help="prefix for the filenames generated - e.g /tmp/dump_ will generate files /tmp/dump_.gz, /tmp/dump_.gz..." ) + + args = parser.parse_args() + + if args.user_uuid: + uuid_list = [uuid.UUID(uuid_str) for uuid_str in args.user_uuid] + elif args.user_email: + uuid_list = [ecwu.User.fromEmail(uuid_str).uuid for uuid_str in args.user_email] + elif args.all: + uuid_list = esdu.get_all_uuids() + elif args.file: + with open(args.file) as fd: + uuid_entries = json.load(fd, object_hook=bju.object_hook) + uuid_list = [ue["uuid"] for ue in uuid_entries] + export_geojson_for_users(uuid_list, args)