OSM_areas.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COLLECT DISTRICT COORDINATE DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T07:52:28.497487Z",
     "iopub.status.busy": "2022-01-27T07:52:28.497265Z",
     "iopub.status.idle": "2022-01-27T07:52:28.849495Z",
     "shell.execute_reply": "2022-01-27T07:52:28.849091Z",
     "shell.execute_reply.started": "2022-01-27T07:52:28.497425Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import time\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from collections import OrderedDict\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T07:52:28.853792Z",
     "iopub.status.busy": "2022-01-27T07:52:28.852701Z",
     "iopub.status.idle": "2022-01-27T07:52:28.857022Z",
     "shell.execute_reply": "2022-01-27T07:52:28.856701Z",
     "shell.execute_reply.started": "2022-01-27T07:52:28.853769Z"
    }
   },
   "outputs": [],
   "source": [
    "user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T07:52:28.860139Z",
     "iopub.status.busy": "2022-01-27T07:52:28.859057Z",
     "iopub.status.idle": "2022-01-27T07:52:28.863328Z",
     "shell.execute_reply": "2022-01-27T07:52:28.862960Z",
     "shell.execute_reply.started": "2022-01-27T07:52:28.860115Z"
    }
   },
   "outputs": [],
   "source": [
    "overpass_url = \"http://overpass-api.de/api/interpreter\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T07:52:28.866562Z",
     "iopub.status.busy": "2022-01-27T07:52:28.865459Z",
     "iopub.status.idle": "2022-01-27T07:52:28.889046Z",
     "shell.execute_reply": "2022-01-27T07:52:28.888624Z",
     "shell.execute_reply.started": "2022-01-27T07:52:28.866540Z"
    }
   },
   "outputs": [],
   "source": [
    "def write_roman(num):\n",
    "    # convert number to roman numbers\n",
    "    # for district queries\n",
    "    roman = OrderedDict()\n",
    "    roman[1000] = \"M\"\n",
    "    roman[900] = \"CM\"\n",
    "    roman[500] = \"D\"\n",
    "    roman[400] = \"CD\"\n",
    "    roman[100] = \"C\"\n",
    "    roman[90] = \"XC\"\n",
    "    roman[50] = \"L\"\n",
    "    roman[40] = \"XL\"\n",
    "    roman[10] = \"X\"\n",
    "    roman[9] = \"IX\"\n",
    "    roman[5] = \"V\"\n",
    "    roman[4] = \"IV\"\n",
    "    roman[1] = \"I\"\n",
    "\n",
    "    def roman_num(num):\n",
    "        for r in roman.keys():\n",
    "            x, y = divmod(num, r)\n",
    "            yield roman[r] * x\n",
    "            num -= (r * x)\n",
    "            if num <= 0:\n",
    "                break\n",
    "\n",
    "    return \"\".join([a for a in roman_num(num)])\n",
    "\n",
    "def json_loader(file):\n",
    "    # loads a json 'file' into a dictionary object\n",
    "    with open(file, 'r') as fp:\n",
    "        loaded_json = json.load(open(file))\n",
    "    return loaded_json\n",
    "\n",
    "def get_overpass_query_from_listofIDs(type_, list_of_ids):\n",
    "    # queries a list of object ids with type_ (node, way, rel)\n",
    "    return f\"\"\"\n",
    "    [out:json];\n",
    "    \"\"\" + \"(\"+\";\".join([f\"{type_}({id_})\" for id_ in list_of_ids])+\";)\" + \"\"\";\n",
    "    out center;\n",
    "    \"\"\"\n",
    "\n",
    "def get_border_of_district(ker, nodes_resp, nodes_resp_byways):\n",
    "    # get first ways coordinates in correct order\n",
    "    firstway = list(nodes_resp_byways[ker].keys())[0]\n",
    "    sorted_df = sort_points(nodes_resp_byways[ker][firstway])\n",
    "    \n",
    "    # we should not look for continuation in ways done\n",
    "    ways_done = [firstway]\n",
    "    id_to_find = sorted_df.iloc[-1]['id']\n",
    "    \n",
    "    # find the next way to continue\n",
    "    while len(ways_done) != len(nodes_resp_byways[ker].keys()):\n",
    "        next_way = find_next_way(ker, nodes_resp_byways, ways_done, id_to_find)\n",
    "        if next_way is None:\n",
    "                next_way = find_next_way_alt(ker, nodes_resp, nodes_resp_byways, ways_done, id_to_find, sorted_df)\n",
    "        sorted_df_new = sort_points(nodes_resp_byways[ker][next_way], id_to_find)\n",
    "        sorted_df = sorted_df.append(sorted_df_new, sort=False)\n",
    "        ways_done.append(next_way)\n",
    "        id_to_find = sorted_df.iloc[-1]['id']\n",
    "    \n",
    "    sorted_df['name'] = f'{ker}. kerulet'\n",
    "    return json.loads(sorted_df.reset_index(drop=True).drop(['tags', 'type', 'dist'], axis = 1, errors='ignore').to_json(orient='records'))\n",
    "\n",
    "lat = 0\n",
    "lon = 0\n",
    "\n",
    "def get_dist(row, lat = lat, lon = lon):\n",
    "    # get distance of coordinates\n",
    "    return (((row['lat'] - lat) ** 2) + ((row['lon'] - lon) ** 2)) ** 0.5\n",
    "\n",
    "def sort_points(df, id_to_start=None):\n",
    "    # sort coordinates by closeness\n",
    "    if id_to_start:\n",
    "        elso_df = pd.DataFrame(df)\n",
    "        elso_df['ordering'] = np.where(elso_df['id'] == id_to_start, 0, 1)\n",
    "        elso_df = elso_df.sort_values('ordering').reset_index(drop=True)\n",
    "        elso_df = elso_df.drop('ordering', axis = 1)\n",
    "        sorted_df = pd.DataFrame(elso_df.loc[0]).T\n",
    "    else:\n",
    "        elso_df = pd.DataFrame(df).sort_values(by = ['lon', 'lat']).reset_index(drop = True)\n",
    "        sorted_df = pd.DataFrame(elso_df.loc[0]).T\n",
    "    lat = elso_df.loc[0, 'lat']\n",
    "    lon = elso_df.loc[0, 'lon']\n",
    "    elso_df.drop(0, inplace = True)\n",
    "    \n",
    "    while len(elso_df) > 0:\n",
    "        elso_df['dist'] = elso_df.apply(lambda x: get_dist(x, lat, lon), axis = 1)\n",
    "        elso_df = elso_df.sort_values(by = 'dist')\n",
    "        sorted_df = sorted_df.append(pd.DataFrame(elso_df.loc[elso_df.index[0]]).T, sort = True)\n",
    "        lat = sorted_df.iloc[-1]['lat']\n",
    "        lon = sorted_df.iloc[-1]['lon']\n",
    "        elso_df.drop(elso_df.index[0], inplace = True)\n",
    "    \n",
    "    return sorted_df\n",
    "\n",
    "def find_next_way(ker, nodes_resp_byways, ways_done, id_to_find):\n",
    "    # find the way which is not done yet having the node \n",
    "    for way in [f for f in nodes_resp_byways[ker].keys() if not f in ways_done]:\n",
    "        for node in nodes_resp_byways[ker][way]:\n",
    "            if node['id'] == id_to_find:\n",
    "                return way\n",
    "        \n",
    "def find_next_way_alt(ker, nodes_resp, nodes_resp_byways, ways_done, id_to_find, sorted_df):\n",
    "    # find potential next points\n",
    "    available_points = [f for f in nodes_resp[ker] if not f['id'] in sorted_df['id'].tolist()]\n",
    "    \n",
    "    # calculate the closest\n",
    "    temp_df = pd.DataFrame(available_points)\n",
    "    temp_df['dist'] = temp_df.apply(lambda x: get_dist(x, sorted_df.iloc[-1]['lat'], sorted_df.iloc[-1]['lon']), axis = 1)\n",
    "    next_id = temp_df.sort_values('dist').reset_index(drop=True).loc[0, 'id']\n",
    "    \n",
    "    # return way with the closest id to continue\n",
    "    return [f for f in get_wayids_for_nodeid(next_id, ker, nodes_resp_byways) if not f in ways_done][0]\n",
    "\n",
    "def json_to_file(my_json, filename):\n",
    "    with open(filename, 'w') as fp:\n",
    "        json.dump(my_json, fp)\n",
    "        \n",
    "def get_wayids_for_nodeid(nodeid, ker, nodes_resp_byways):\n",
    "    # get ways a node is part of\n",
    "    return [f for f in nodes_resp_byways[ker].keys() if nodeid in [g['id'] for g in nodes_resp_byways[ker][f]]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DISTRICTS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GET WAYS BORDERING DISTRICTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T07:52:28.891980Z",
     "iopub.status.busy": "2022-01-27T07:52:28.891553Z",
     "iopub.status.idle": "2022-01-27T08:04:48.450244Z",
     "shell.execute_reply": "2022-01-27T08:04:48.449622Z",
     "shell.execute_reply.started": "2022-01-27T07:52:28.891953Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: I. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: II. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading: III. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading: IV. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "Downloading: V. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: VI. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "Downloading: VII. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading: VIII. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: IX. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Try 4\n",
      "Success!\n",
      "Downloading: X. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XI. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Try 4\n",
      "Success!\n",
      "Downloading: XII. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XIII. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Try 4\n",
      "Success!\n",
      "Downloading: XIV. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XV. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Try 4\n",
      "Success!\n",
      "Downloading: XVI. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XVII. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "Downloading: XVIII. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XIX. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "Downloading: XX. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XXI. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Try 4\n",
      "Success!\n",
      "Downloading: XXII. kerület\n",
      "Try 1\n",
      "Success!\n",
      "Downloading: XXIII. kerület\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n"
     ]
    }
   ],
   "source": [
    "responses = {ker: None for ker in [write_roman(f) for f in range(1,24)]}\n",
    "\n",
    "for ker in [write_roman(f) for f in range(1,24)]:\n",
    "    if responses[ker]:\n",
    "        continue\n",
    "    \n",
    "    print(f\"Downloading: {ker}. kerület\")\n",
    "    overpass_query = f\"\"\"\n",
    "    [out:json];\n",
    "    area[\"ISO3166-1\"=\"HU\"][admin_level=2];\n",
    "    (\n",
    "     rel[\"name\"=\"{ker}. kerület\"](area);\n",
    "    );\n",
    "    out center;\n",
    "    \"\"\"\n",
    "    retry = True\n",
    "    try_num = 1\n",
    "    while retry:\n",
    "        print(f'Try {try_num}')\n",
    "        response = requests.get(overpass_url, params={'data': overpass_query}, headers={'User-Agent': user_agent})\n",
    "        if response.status_code == 200:\n",
    "            responses[ker] = response.json()\n",
    "            retry = False\n",
    "            print('Success!')\n",
    "        else:\n",
    "            time.sleep(try_num)\n",
    "            try_num += 1\n",
    "            continue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GET WAYS AND POINTS CONSTRUCTING THE DISTRICT BORDERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T08:05:07.622875Z",
     "iopub.status.busy": "2022-01-27T08:05:07.622223Z",
     "iopub.status.idle": "2022-01-27T08:05:07.626891Z",
     "shell.execute_reply": "2022-01-27T08:05:07.626390Z",
     "shell.execute_reply.started": "2022-01-27T08:05:07.622844Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# distill necessary information\n",
    "outer_ways = {ker: \n",
    "              [i['ref'] for i in responses[ker]['elements'][0]['members'] if i['role'] == 'outer']\n",
    "              for ker in responses.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T08:05:08.372125Z",
     "iopub.status.busy": "2022-01-27T08:05:08.371518Z",
     "iopub.status.idle": "2022-01-27T08:05:08.374945Z",
     "shell.execute_reply": "2022-01-27T08:05:08.374437Z",
     "shell.execute_reply.started": "2022-01-27T08:05:08.372098Z"
    }
   },
   "outputs": [],
   "source": [
    "ways_resp = {ker: None for ker in outer_ways.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T08:05:25.622710Z",
     "iopub.status.busy": "2022-01-27T08:05:25.622389Z",
     "iopub.status.idle": "2022-01-27T08:09:03.065706Z",
     "shell.execute_reply": "2022-01-27T08:09:03.065084Z",
     "shell.execute_reply.started": "2022-01-27T08:05:25.622680Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading ways for I. district...\n",
      "Downloading ways for II. district...\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading ways for III. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for IV. district...\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading ways for V. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for VI. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for VII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for VIII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for IX. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for X. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XI. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XIII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XIV. district...\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading ways for XV. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XVI. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XVII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XVIII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XIX. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XX. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XXI. district...\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "Downloading ways for XXII. district...\n",
      "Try 1\n",
      "Success!\n",
      "Downloading ways for XXIII. district...\n",
      "Try 1\n",
      "Success!\n"
     ]
    }
   ],
   "source": [
    "for ker in outer_ways.keys():\n",
    "    print(f\"Downloading ways for {ker}. district...\")\n",
    "    if ways_resp[ker]:\n",
    "        continue\n",
    "    \n",
    "    overpass_query = f\"\"\"\n",
    "        [out:json];\n",
    "         \"\"\"+ \"(\"+\";\".join([f\"way({wayid})\" for wayid in outer_ways[ker]])+\";)\" + \"\"\";\n",
    "        out center;\n",
    "        \"\"\"\n",
    "    retry = True\n",
    "    try_num = 1\n",
    "    while retry:\n",
    "        print(f'Try {try_num}')\n",
    "        response = requests.get(overpass_url, params={'data': overpass_query})\n",
    "        if response.status_code == 200:\n",
    "            response_json = response.json()\n",
    "            retry = False\n",
    "            print('Success!')\n",
    "        else:\n",
    "            time.sleep(try_num)\n",
    "            try_num += 1\n",
    "            continue\n",
    "            \n",
    "    ways_resp[ker] = response_json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GET NODES BY DISTRICT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:02:43.009104Z",
     "iopub.status.busy": "2022-01-27T11:02:43.008823Z",
     "iopub.status.idle": "2022-01-27T11:02:43.015326Z",
     "shell.execute_reply": "2022-01-27T11:02:43.014935Z",
     "shell.execute_reply.started": "2022-01-27T11:02:43.009079Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# node ids to query\n",
    "nodes_border = {ker: {i['id']: i['nodes'] for i in ways_resp[ker]['elements']} for ker in outer_ways.keys()}\n",
    "nodes_byker = {ker: [item for sublist in [list(set_) for k, set_ in nodes_border[ker].items()] for item in sublist] for ker in outer_ways.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:02:43.756398Z",
     "iopub.status.busy": "2022-01-27T11:02:43.756108Z",
     "iopub.status.idle": "2022-01-27T11:02:43.760479Z",
     "shell.execute_reply": "2022-01-27T11:02:43.760055Z",
     "shell.execute_reply.started": "2022-01-27T11:02:43.756378Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodes_resp = {ker: [] for ker in nodes_border.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:02:55.539745Z",
     "iopub.status.busy": "2022-01-27T11:02:55.539307Z",
     "iopub.status.idle": "2022-01-27T11:18:33.824239Z",
     "shell.execute_reply": "2022-01-27T11:18:33.823552Z",
     "shell.execute_reply.started": "2022-01-27T11:02:55.539715Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading nodes for I. district...\n",
      "The task to be done in multiple segments...\n",
      "-------------\n",
      "Segment 1 of 2:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 2 of 2:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Success!\n",
      "District I ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for II. district...\n",
      "The task to be done in multiple segments...\n",
      "-------------\n",
      "Segment 1 of 3:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 2 of 3:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 3 of 3:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Segment successful!\n",
      "-------------\n",
      "Success!\n",
      "District II ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for III. district...\n",
      "The task to be done in multiple segments...\n",
      "-------------\n",
      "Segment 1 of 2:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 2 of 2:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Segment successful!\n",
      "-------------\n",
      "Success!\n",
      "District III ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for IV. district...\n",
      "Try 0\n",
      "Success!\n",
      "District IV ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for V. district...\n",
      "Try 0\n",
      "Success!\n",
      "District V ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for VI. district...\n",
      "Try 0\n",
      "Success!\n",
      "District VI ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for VII. district...\n",
      "Try 0\n",
      "Success!\n",
      "District VII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for VIII. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "District VIII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for IX. district...\n",
      "Try 0\n",
      "Success!\n",
      "District IX ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for X. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "District X ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XI. district...\n",
      "The task to be done in multiple segments...\n",
      "-------------\n",
      "Segment 1 of 4:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 2 of 4:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 3 of 4:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 4 of 4:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Segment successful!\n",
      "-------------\n",
      "Success!\n",
      "District XI ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XII. district...\n",
      "The task to be done in multiple segments...\n",
      "-------------\n",
      "Segment 1 of 4:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 2 of 4:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 3 of 4:\n",
      "Try 0\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 4 of 4:\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Segment successful!\n",
      "-------------\n",
      "Success!\n",
      "District XII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XIII. district...\n",
      "Try 0\n",
      "Success!\n",
      "District XIII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XIV. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "District XIV ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XV. district...\n",
      "Try 0\n",
      "Success!\n",
      "District XV ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XVI. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "District XVI ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XVII. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Success!\n",
      "District XVII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XVIII. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Success!\n",
      "District XVIII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XIX. district...\n",
      "Try 0\n",
      "Success!\n",
      "District XIX ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XX. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Try 3\n",
      "Success!\n",
      "District XX ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XXI. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Success!\n",
      "District XXI ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XXII. district...\n",
      "The task to be done in multiple segments...\n",
      "-------------\n",
      "Segment 1 of 2:\n",
      "Try 0\n",
      "Try 1\n",
      "Segment successful!\n",
      "-------------\n",
      "Segment 2 of 2:\n",
      "Try 0\n",
      "Try 1\n",
      "Segment successful!\n",
      "-------------\n",
      "Success!\n",
      "District XXII ready\n",
      "-------------\n",
      "-------------\n",
      "Downloading nodes for XXIII. district...\n",
      "Try 0\n",
      "Try 1\n",
      "Try 2\n",
      "Success!\n",
      "District XXIII ready\n",
      "-------------\n",
      "-------------\n"
     ]
    }
   ],
   "source": [
    "for ker in nodes_resp.keys():\n",
    "    print(f\"Downloading nodes for {ker}. district...\")\n",
    "    if nodes_resp[ker]:\n",
    "        print(\"continuing, as it is ready.\")\n",
    "        continue\n",
    "    \n",
    "    if len(nodes_byker[ker]) > 300:\n",
    "        print(\"The task to be done in multiple segments...\")\n",
    "        print(\"-------------\")\n",
    "        overpass_queries = [get_overpass_query_from_listofIDs('node', nodes_byker[ker][i*300:(i+1)*300]) \n",
    "                            for i in range(int(\n",
    "                                len(nodes_byker[ker])/300))] + [get_overpass_query_from_listofIDs(\n",
    "            'node', nodes_byker[ker][int(len(nodes_byker[ker])/300)*300:])]\n",
    "        responses = []\n",
    "        for n, overpass_query in enumerate(overpass_queries):\n",
    "            print(f\"Segment {n+1} of {len(overpass_queries)}:\")\n",
    "            success = False\n",
    "            for i in range(4):\n",
    "                print(f'Try {i}')\n",
    "                response = requests.get(overpass_url, params={'data': overpass_query})\n",
    "                if response.status_code == 200:\n",
    "                    responses.append(response)\n",
    "                    success = True\n",
    "                    print(\"Segment successful!\")\n",
    "                    print(\"-------------\")\n",
    "                    break\n",
    "                else:\n",
    "                    time.sleep(2)\n",
    "                    continue\n",
    "            if not success:\n",
    "                raise ValueError(f\"Could not obtain all nodes for ker {ker}\")\n",
    "        \n",
    "        response_jsons = [response.json() for response in responses]\n",
    "        for resp in response_jsons:    \n",
    "            nodes_resp[ker].append(resp)\n",
    "        print('Success!')\n",
    "    else:\n",
    "        for i in range(4):\n",
    "            print(f\"Try {i}\")\n",
    "            overpass_query = get_overpass_query_from_listofIDs('node', nodes_byker[ker])\n",
    "            response = requests.get(overpass_url, params={'data': overpass_query})\n",
    "            time.sleep(1)\n",
    "            try:\n",
    "                response_json = response.json()\n",
    "                nodes_resp[ker] = [response_json]\n",
    "                print('Success!')\n",
    "                break\n",
    "            except:\n",
    "                continue\n",
    "    print(f\"District {ker} ready\")\n",
    "    print(\"-------------\")\n",
    "    print(\"-------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MAP NODES TO WAYS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:18:33.825717Z",
     "iopub.status.busy": "2022-01-27T11:18:33.825560Z",
     "iopub.status.idle": "2022-01-27T11:18:33.829090Z",
     "shell.execute_reply": "2022-01-27T11:18:33.828693Z",
     "shell.execute_reply.started": "2022-01-27T11:18:33.825699Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodes_resp = {ker: [item for sublist in [i['elements'] for i in nodes_resp[ker]] for item in sublist] for ker in nodes_resp.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:18:33.829947Z",
     "iopub.status.busy": "2022-01-27T11:18:33.829822Z",
     "iopub.status.idle": "2022-01-27T11:18:33.947619Z",
     "shell.execute_reply": "2022-01-27T11:18:33.947199Z",
     "shell.execute_reply.started": "2022-01-27T11:18:33.829930Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "nodes_resp_byways = {ker: {way: [f for f in nodes_resp[ker] if f['id'] in nodes_border[ker][way]] for way in nodes_border[ker].keys()} for ker in nodes_resp.keys()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SORT NODES USING THE ALGORITHM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:18:34.365317Z",
     "iopub.status.busy": "2022-01-27T11:18:34.365182Z",
     "iopub.status.idle": "2022-01-27T11:18:55.100535Z",
     "shell.execute_reply": "2022-01-27T11:18:55.100109Z",
     "shell.execute_reply.started": "2022-01-27T11:18:34.365300Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:1: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02011dfcab814f5982c08373ffaa46cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/23 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sorted_nodes = {ker: get_border_of_district(ker, nodes_resp, nodes_resp_byways) for ker in tqdm(nodes_resp.keys())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T11:18:55.101543Z",
     "iopub.status.busy": "2022-01-27T11:18:55.101400Z",
     "iopub.status.idle": "2022-01-27T11:18:55.104766Z",
     "shell.execute_reply": "2022-01-27T11:18:55.104425Z",
     "shell.execute_reply.started": "2022-01-27T11:18:55.101521Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create one big json list\n",
    "all_coords = []\n",
    "for ker in sorted_nodes.keys():\n",
    "    all_coords += sorted_nodes[ker]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### WRITE TO FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-27T13:31:14.222256Z",
     "iopub.status.busy": "2022-01-27T13:31:14.221953Z",
     "iopub.status.idle": "2022-01-27T13:31:14.283614Z",
     "shell.execute_reply": "2022-01-27T13:31:14.283298Z",
     "shell.execute_reply.started": "2022-01-27T13:31:14.222222Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('data/district_coords.txt', 'w') as fp:\n",
    "    json.dump(all_coords, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "🌍 Base Python 3.7 Environment",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": false,
   "toc_window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "02011dfcab814f5982c08373ffaa46cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_7abbbb04f1e74a26801d21a7864fdc66",
        "IPY_MODEL_92e554b216d84995ac68df9b7693c0bd",
        "IPY_MODEL_e7e70b93ae674ba7acd8224519ac3f23"
       ],
       "layout": "IPY_MODEL_faebb9203aed4e5c82b21ca4874a2cd6"
      }
     },
     "3ea87bcb3c8e42c4b39a045fd1883108": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "536b6219e6b04a7db8b83ff946477c8c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7abbbb04f1e74a26801d21a7864fdc66": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ed794aa889a84c28bfc7555960ee284d",
       "style": "IPY_MODEL_e3808a19de334f19954d64071997f718",
       "value": "100%"
      }
     },
     "8a6431e432fa4ee6b701bf2a0cd05c8d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "92e554b216d84995ac68df9b7693c0bd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_536b6219e6b04a7db8b83ff946477c8c",
       "max": 23,
       "style": "IPY_MODEL_8a6431e432fa4ee6b701bf2a0cd05c8d",
       "value": 23
      }
     },
     "e3808a19de334f19954d64071997f718": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e7e70b93ae674ba7acd8224519ac3f23": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ee72da8a50634aafaeaaef7314d7250e",
       "style": "IPY_MODEL_3ea87bcb3c8e42c4b39a045fd1883108",
       "value": " 23/23 [00:20&lt;00:00,  1.65it/s]"
      }
     },
     "ed794aa889a84c28bfc7555960ee284d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ee72da8a50634aafaeaaef7314d7250e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "faebb9203aed4e5c82b21ca4874a2cd6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}