sync.py

"""
Sync traffic lights from Hamburg's server into our proxy server.

This script runs nightly and performs the following steps:
1. Download the latest traffic lights from Hamburg's server
2. Filter out traffic lights that won't send any data
3. Push the traffic lights to our proxy server
"""

import json
import os
import re

import requests

# Format: https://tld.iot.hamburg.de/v1.1/
FROST_HAMBURG_URL = os.environ['FROST_HAMBURG_URL']
# Format: http://localhost:8080/FROST-Server/v1.1/
FROST_PROXY_URL = os.environ['FROST_PROXY_URL']

if not FROST_HAMBURG_URL or not FROST_PROXY_URL:
    raise Exception('Missing environment variables')
if not FROST_HAMBURG_URL.endswith('/'):
    raise Exception('FROST_HAMBURG_URL must end with a slash')
if not FROST_PROXY_URL.endswith('/'):
    raise Exception('FROST_PROXY_URL must end with a slash')

# Skip this step when there is a things file 
# (will be clean in the Container build)
# to accelerate development
if os.path.exists('things.json'):
    with open('things.json') as f:
        things = json.loads(f.read())
else:
    # Step 1: Fetch all things
    elements_per_page = 100
    page = 0
    things = []
    while True:
        page_url = FROST_HAMBURG_URL + "Things?" + \
            "$filter=Datastreams/properties/serviceName eq 'HH_STA_traffic_lights' " + \
            "and (Datastreams/properties/layerName eq 'primary_signal' " + \
            "  or Datastreams/properties/layerName eq 'signal_program' " + \
            "  or Datastreams/properties/layerName eq 'cycle_second' " + \
            "  or Datastreams/properties/layerName eq 'detector_car' " + \
            "  or Datastreams/properties/layerName eq 'detector_bike') " + \
            "&$expand=Datastreams($expand=ObservedProperty,Sensor),Locations" + \
            "&$skip=" + str(page * elements_per_page)

        print(f'Downloading page {page} from {page_url}')
        response = requests.get(page_url)
        response.raise_for_status()
        response_json = response.json()
        if 'value' not in response_json:
            raise Exception('Missing value in response')
        if len(response_json['value']) == 0:
            print('Finished downloading all traffic lights')
            break
        for traffic_light in response_json['value']:
            things.append(traffic_light)
        page += 1
    with open('things.json', 'w') as f:
        f.write(json.dumps(things))

frost_connections = set([thing['name'] for thing in things])
print(f'Found {len(frost_connections)} connections in FROST server.')

# Step 2: Filter out traffic lights that aren't provided by the MAP portal
# https://daten-hamburg.de/tlf_public/

maps = []
# Load the exclude list
index_page = requests.get('https://daten-hamburg.de/tlf_public/')
index_page.raise_for_status()
index_page = index_page.text
xml_links = re.findall(r'<a href="(.+\.xml)">', index_page)
for i, xml_link in enumerate(xml_links):
    # Check if we already have the file
    if os.path.exists('maps/' + xml_link):
        with open('maps/' + xml_link) as f:
            maps.append(f.read())
        continue

    print(f'Downloading xml file {i + 1}/{len(xml_links)}: {xml_link}')
    
    # Download the xml file
    xml = requests.get('https://daten-hamburg.de/tlf_public/' + xml_link)
    xml.raise_for_status()
    xml = xml.text
    # Write to dir, create folder if necessary
    if not os.path.exists('maps'):
        os.makedirs('maps')
    with open('maps/' + xml_link, 'w') as f:
        f.write(xml)
    maps.append(xml)
        
map_connections = set()
for xml in maps:
    # Find all strings in <name></name> tags
    names = re.findall(r'<name>(.+)</name>', xml)
    assert len(names) > 0, 'No names found in xml file'
    assert len(names) == 1, 'Multiple names found in xml file'
    # Parse the node id from the name (works for HH... and MAP_ITS... files)
    node_ids = re.findall(r'\d+', names[0])
    node_id = node_ids[1]
    # Find all <connectionID></connectionID> numbers
    connection_ids = re.findall(r'<connectionID>(\d+)</connectionID>', xml)
    map_connections.update([f"{node_id}_{cid}" for cid in connection_ids])

print(f'Found {len(map_connections)} connections in MAP portal.')

# Step 3: Push the traffic lights to our proxy server
things_to_keep = [thing for thing in things if thing['name'] in map_connections]
print(f'Keeping {len(things_to_keep)} traffic lights of {len(things)}')
for thing in things_to_keep:
    # Go through the thing recursively and remove all fields containing an @
    # since these fields are autogenerated by the FROST server
    def remove_fields(obj):
        if isinstance(obj, dict):
            for key in list(obj.keys()):
                if '@' in key and '@iot.id' not in key:
                    del obj[key]
                else:
                    remove_fields(obj[key])
        elif isinstance(obj, list):
            for item in obj:
                remove_fields(item)
    remove_fields(thing)
    response = requests.post(FROST_PROXY_URL + 'Things', data=json.dumps(thing))