Merge pull request #66 from achasmita/master

Added trajectory table to data page

README.md

@@ -66,6 +66,7 @@ These are all the permissions that you can specify:
 - `data_uuids`: User can view the UUIDs data in the Data page.
 - `data_trips`: User can view the trips data in the Data page.
 - `data_demographics`: User can view the demographics data in the Data page.
+- `data_trajectories`: User can view the trajectories data in the Data page.
 - `data_trips_columns_exclude`: It used to specify a list of column names that should be excluded from the trips data
 that is displayed on the Data page. It includes valid columns from the **Stage_analysis_timeseries** collection. Valid
 columns are specified in the following sections.
@@ -74,6 +75,8 @@ that is displayed on the Data page. It includes valid columns from the **Stage_u
 specified in the following sections.
 - `data_demographics_columns_exclude`: It used to specify a list of column names that should be excluded from the demographics data
 that is displayed on the Data page. 
+- `data_trajectories_columns_exclude`: It used to specify a list of column names that should be excluded from the trajectories data
+that is displayed on the Data page.
 
 ### Token Page
 - `token_generate`: User can generate new tokens in the Token page.

app_sidebar_collapsible.py

@@ -169,12 +169,14 @@ def update_store_demographics():
 
 demographics_data = update_store_demographics()
 
+
 app.layout = html.Div(
     [
         dcc.Location(id='url', refresh=False),
         dcc.Store(id='store-trips', data={}),
         dcc.Store(id='store-uuids', data={}),
         dcc.Store(id='store-demographics', data= demographics_data),
+        dcc.Store(id ='store-trajectories', data = {}),   
         html.Div(id='page-content', children=home_page),
     ]
 )

pages/data.py

@@ -12,7 +12,7 @@
 
 from utils import permissions as perm_utils
 from utils import db_utils
-
+from utils.db_utils import query_trajectories
 register_page(__name__, path="/data")
 
 intro = """## Data"""
@@ -23,7 +23,8 @@
         dcc.Tabs(id="tabs-datatable", value='tab-uuids-datatable', children=[
             dcc.Tab(label='UUIDs', value='tab-uuids-datatable'),
             dcc.Tab(label='Trips', value='tab-trips-datatable'),
-            dcc.Tab(label='Demographics', value='tab-demographics-datatable' )
+            dcc.Tab(label='Demographics', value='tab-demographics-datatable'),
+            dcc.Tab(label='Trajectories', value='tab-trajectories-datatable'),
         ]),
         html.Div(id='tabs-content'),
     ]
@@ -37,15 +38,27 @@ def clean_location_data(df):
         df['data.end_loc.coordinates'] = df['data.end_loc.coordinates'].apply(lambda x: f'({x[0]}, {x[1]})')
     return df
 
+def update_store_trajectories():
+    global store_trajectories
+    df = query_trajectories()
+    records = df.to_dict("records")   
+    store = {
+        "data": records,
+        "length": len(records),
+    }
+    store_trajectories = store
+    return store
+
 
 @callback(
     Output('tabs-content', 'children'),
     Input('tabs-datatable', 'value'),
     Input('store-uuids', 'data'),
     Input('store-trips', 'data'),
     Input('store-demographics', 'data'),
+    Input('store-trajectories', 'data'),
 )
-def render_content(tab, store_uuids, store_trips, store_demographics):
+def render_content(tab, store_uuids, store_trips, store_demographics, store_trajectories):
     data, columns, has_perm = None, [], False
     if tab == 'tab-uuids-datatable':
         data = store_uuids["data"]
@@ -63,6 +76,14 @@ def render_content(tab, store_uuids, store_trips, store_demographics):
         data = store_demographics["data"]
         columns = list(data[0].keys())
         has_perm = perm_utils.has_permission('data_demographics')
+    elif tab == 'tab-trajectories-datatable':
+        # Currently store_trajectories data is loaded only when the respective tab is selected
+        #Here we query for trajectory data once "Trajectories" tab is selected
+        if store_trajectories == {}:
+            store_trajectories = update_store_trajectories()
+        data = store_trajectories["data"]
+        columns = list(data[0].keys())
+        has_perm = perm_utils.has_permission('data_trajectories')
 
     df = pd.DataFrame(data)
     if df.empty or not has_perm:

utils/constants.py

@@ -50,6 +50,14 @@
     '_id',
 ]
 
+
+BINARY_TRAJECTORIES_COLS = [
+    'user_id',
+    '_id',
+    'data.section',
+    'data.loc.coordinates'
+]
+
 EXCLUDED_DEMOGRAPHICS_COLS = [
     'data.xmlResponse', 
     'data.name',

utils/db_utils.py

@@ -118,6 +118,24 @@ def query_demographics():
     return df
 
 
+def query_trajectories():
+    ts = esta.TimeSeries.get_aggregate_time_series()
+
+    entries = ts.find_entries(
+        key_list=["analysis/recreated_location"],
+    )
+    df = pd.json_normalize(list(entries))
+    if not df.empty:
+        for col in constants.BINARY_TRAJECTORIES_COLS:
+            if col in df.columns:
+                df[col] = df[col].apply(str)  
+    columns_to_drop = [col for col in df.columns if col.startswith("metadata")]
+    df.drop(columns= columns_to_drop, inplace=True) 
+    modified_columns = perm_utils.get_trajectories_columns(df.columns)  
+    df.columns = modified_columns 
+    return df
+
+
 def add_user_stats(user_data):
     for user in user_data:
         user_uuid = UUID(user['user_id'])

utils/permissions.py

@@ -33,6 +33,8 @@
 # When all the current studies are completed we can remove the below changes.
 if 'data_demographics_columns_exclude' not in permissions:
     permissions['data_demographics_columns_exclude'] = []
+if 'data_trajectories_columns_exclude' not in permissions:
+    permissions['data_trajectories_columns_exclude'] = []
 
 def has_permission(perm):
     return False if permissions.get(perm) is False else True
@@ -96,5 +98,10 @@ def get_demographic_columns(columns):
         columns.discard(column)
     return columns
 
+def get_trajectories_columns(columns):
+    for column in permissions.get("data_trajectories_columns_exclude", []):
+        columns.discard(column)
+    return columns
+
 def get_token_prefix():
     return permissions['token_prefix'] + '_' if permissions.get('token_prefix') else ''