From 6184824cf794d6696a2ac99c2701b8e96df00e9a Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Mon, 4 Nov 2024 15:16:14 +0000
Subject: [PATCH 01/33] 2024 Nov Build

---
 parameters.json |  2 +-
 route_ids.csv   | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/parameters.json b/parameters.json
index 78e18dc..44d7e6e 100644
--- a/parameters.json
+++ b/parameters.json
@@ -8,7 +8,7 @@
   "disag_threshold": [100],
   "min_distance_meters": [500],
   "region_buffer_distance_meters": [5000],
-  "region": ["Edinburgh and Lothians"],
+  "region": ["Tayside, Central and Fife"],
   "generate_CN_start": [true],
   "coherent_sources": ["OS"],
   "coherent_percentile": [0.94, 0.9]
diff --git a/route_ids.csv b/route_ids.csv
index 45c8ec4..f88f077 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,56 @@
 nrow,plan,purpose,region,date,id
+9425,ebike,school,Edinburgh and Lothians,2024-11-01,10897
+9425,balanced,school,Edinburgh and Lothians,2024-11-01,10896
+9425,quietest,school,Edinburgh and Lothians,2024-11-01,10895
+9425,fastest,school,Edinburgh and Lothians,2024-11-01,10894
+51181,ebike,utility,"Tayside, Central and Fife",2024-11-01,10893
+51181,quietest,utility,"Tayside, Central and Fife",2024-11-01,10892
+51181,fastest,utility,"Tayside, Central and Fife",2024-11-01,10891
+51181,balanced,utility,"Tayside, Central and Fife",2024-11-01,10890
+65662,ebike,commute,"Tayside, Central and Fife",2024-11-01,10889
+65662,balanced,commute,"Tayside, Central and Fife",2024-11-01,10888
+65662,quietest,commute,"Tayside, Central and Fife",2024-11-01,10887
+65662,fastest,commute,"Tayside, Central and Fife",2024-11-01,10886
+11451,ebike,school,"Tayside, Central and Fife",2024-11-01,10885
+11451,balanced,school,"Tayside, Central and Fife",2024-11-01,10884
+11451,quietest,school,"Tayside, Central and Fife",2024-11-01,10883
+11451,fastest,school,"Tayside, Central and Fife",2024-11-01,10882
+21026,ebike,utility,Aberdeen and North East,2024-11-01,10855
+21026,quietest,utility,Aberdeen and North East,2024-11-01,10854
+21026,fastest,utility,Aberdeen and North East,2024-11-01,10853
+21026,balanced,utility,Aberdeen and North East,2024-11-01,10852
+32967,ebike,commute,Aberdeen and North East,2024-11-01,10851
+32967,balanced,commute,Aberdeen and North East,2024-11-01,10850
+32967,quietest,commute,Aberdeen and North East,2024-11-01,10849
+32967,fastest,commute,Aberdeen and North East,2024-11-01,10848
+4525,ebike,school,Aberdeen and North East,2024-11-01,10847
+4525,balanced,school,Aberdeen and North East,2024-11-01,10846
+4525,quietest,school,Aberdeen and North East,2024-11-01,10845
+4525,fastest,school,Aberdeen and North East,2024-11-01,10844
+9536,ebike,utility,Scotland South,2024-11-01,10843
+9536,quietest,utility,Scotland South,2024-11-01,10842
+9536,fastest,utility,Scotland South,2024-11-01,10841
+9536,balanced,utility,Scotland South,2024-11-01,10840
+6943,ebike,commute,Scotland South,2024-11-01,10839
+6943,balanced,commute,Scotland South,2024-11-01,10838
+6943,quietest,commute,Scotland South,2024-11-01,10837
+6943,fastest,commute,Scotland South,2024-11-01,10836
+1607,ebike,school,Scotland South,2024-11-01,10835
+1607,balanced,school,Scotland South,2024-11-01,10834
+1607,quietest,school,Scotland South,2024-11-01,10833
+1607,fastest,school,Scotland South,2024-11-01,10832
+14963,ebike,utility,Highlands and Islands,2024-11-01,10831
+14963,quietest,utility,Highlands and Islands,2024-11-01,10830
+14963,fastest,utility,Highlands and Islands,2024-11-01,10829
+14963,balanced,utility,Highlands and Islands,2024-11-01,10828
+7976,ebike,commute,Highlands and Islands,2024-11-01,10827
+7976,balanced,commute,Highlands and Islands,2024-11-01,10826
+7976,quietest,commute,Highlands and Islands,2024-11-01,10825
+7976,fastest,commute,Highlands and Islands,2024-11-01,10824
+3228,ebike,school,Highlands and Islands,2024-11-01,10823
+3228,balanced,school,Highlands and Islands,2024-11-01,10822
+3228,quietest,school,Highlands and Islands,2024-11-01,10821
+3228,fastest,school,Highlands and Islands,2024-11-01,10820
 100,ebike,utility,Edinburgh and Lothians,2024-09-30,10779
 100,quietest,utility,Edinburgh and Lothians,2024-09-30,10778
 100,fastest,utility,Edinburgh and Lothians,2024-09-30,10777

From 52dc0acc6525b9fb6aa7642adf365a0b3d309d28 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Tue, 5 Nov 2024 17:27:39 +0000
Subject: [PATCH 02/33] add route ids

---
 parameters.json |  2 +-
 route_ids.csv   | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/parameters.json b/parameters.json
index 44d7e6e..78e18dc 100644
--- a/parameters.json
+++ b/parameters.json
@@ -8,7 +8,7 @@
   "disag_threshold": [100],
   "min_distance_meters": [500],
   "region_buffer_distance_meters": [5000],
-  "region": ["Tayside, Central and Fife"],
+  "region": ["Edinburgh and Lothians"],
   "generate_CN_start": [true],
   "coherent_sources": ["OS"],
   "coherent_percentile": [0.94, 0.9]
diff --git a/route_ids.csv b/route_ids.csv
index f88f077..a23200e 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,14 @@
 nrow,plan,purpose,region,date,id
+111504,balanced,utility,Edinburgh and Lothians,2024-11-01,11217
+90716,ebike,commute,Edinburgh and Lothians,2024-11-01,11216
+218877,fastest,commute,Glasgow and Strathclyde,2024-11-01,11214
+29032,ebike,school,Glasgow and Strathclyde,2024-11-01,11213
+29032,balanced,school,Glasgow and Strathclyde,2024-11-01,11212
+29032,quietest,school,Glasgow and Strathclyde,2024-11-01,11211
+29032,fastest,school,Glasgow and Strathclyde,2024-11-01,11202
+90716,balanced,commute,Edinburgh and Lothians,2024-11-01,11116
+90716,quietest,commute,Edinburgh and Lothians,2024-11-01,11105
+90716,fastest,commute,Edinburgh and Lothians,2024-11-01,11104
 9425,ebike,school,Edinburgh and Lothians,2024-11-01,10897
 9425,balanced,school,Edinburgh and Lothians,2024-11-01,10896
 9425,quietest,school,Edinburgh and Lothians,2024-11-01,10895

From 6661ad2304682af44fc3f9d5173b355d3c720198 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Tue, 5 Nov 2024 22:43:33 +0000
Subject: [PATCH 03/33] Manually update route_ids, speed things up

---
 route_ids.csv | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/route_ids.csv b/route_ids.csv
index a23200e..76a87e9 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,7 @@
 nrow,plan,purpose,region,date,id
+111504,ebike,utility,Edinburgh and Lothians,2024-11-01,11221
+111504,quietest,utility,Edinburgh and Lothians,2024-11-01,11220
+111504,fastest,utility,Edinburgh and Lothians,2024-11-01,11218
 111504,balanced,utility,Edinburgh and Lothians,2024-11-01,11217
 90716,ebike,commute,Edinburgh and Lothians,2024-11-01,11216
 218877,fastest,commute,Glasgow and Strathclyde,2024-11-01,11214

From 17b2c4372f42fb77526c6637aba399020a6ccd5d Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Wed, 6 Nov 2024 09:32:18 +0000
Subject: [PATCH 04/33] add route id from zw

---
 parameters.json | 2 +-
 route_ids.csv   | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/parameters.json b/parameters.json
index 78e18dc..a276eed 100644
--- a/parameters.json
+++ b/parameters.json
@@ -8,7 +8,7 @@
   "disag_threshold": [100],
   "min_distance_meters": [500],
   "region_buffer_distance_meters": [5000],
-  "region": ["Edinburgh and Lothians"],
+  "region": ["Glasgow and Strathclyde"],
   "generate_CN_start": [true],
   "coherent_sources": ["OS"],
   "coherent_percentile": [0.94, 0.9]
diff --git a/route_ids.csv b/route_ids.csv
index 76a87e9..85dbf38 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,7 +1,11 @@
 nrow,plan,purpose,region,date,id
 111504,ebike,utility,Edinburgh and Lothians,2024-11-01,11221
 111504,quietest,utility,Edinburgh and Lothians,2024-11-01,11220
+<<<<<<< HEAD
 111504,fastest,utility,Edinburgh and Lothians,2024-11-01,11218
+=======
+111504,fastest,utility,Edinburgh and Lothians,2024-11-01,11219
+>>>>>>> 0f4fae1 (add more route id)
 111504,balanced,utility,Edinburgh and Lothians,2024-11-01,11217
 90716,ebike,commute,Edinburgh and Lothians,2024-11-01,11216
 218877,fastest,commute,Glasgow and Strathclyde,2024-11-01,11214

From ad4f0c64725ffe101af00104a90f12ada561c81b Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Wed, 6 Nov 2024 11:46:38 +0000
Subject: [PATCH 05/33] fix typo in route_ids.csv

---
 R/simplify_network.R                   | 11 ++++------
 code/tests/simplified_network_test.qmd | 29 ++++++++++++++++++++++++++
 route_ids.csv                          |  4 ----
 3 files changed, 33 insertions(+), 11 deletions(-)
 create mode 100644 code/tests/simplified_network_test.qmd

diff --git a/R/simplify_network.R b/R/simplify_network.R
index 2f76c9a..4864af3 100644
--- a/R/simplify_network.R
+++ b/R/simplify_network.R
@@ -8,12 +8,9 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
   region_snake_case = snakecase::to_snake_case(parameters$region[[1]])
   base_name = paste0("OS_Scotland_Network_", region_snake_case, ".geojson")
   rnet_x_f = file.path("inputdata", base_name)
-  rnet_x = sf::read_sf(rnet_x_f)
-  # rnet_x = geojsonsf::geojson_sf(rnet_x_f) # bit faster
+  rnet_x = sf::read_sf(rnet_x_f) |> sf::st_transform(crs = "EPSG:27700")
 
-  # Transform the spatial data to a different coordinate reference system (EPSG:27700)
-  # TODO: uncomment:
-  rnet_x = rnet_x[region_boundary, ] # TODO: is this needed? Can remove if not
+  rnet_x = rnet_x[region_boundary |> sf::st_transform(crs = "EPSG:27700") , ] # TODO: is this needed? Can remove if not
   rnet_xp = sf::st_transform(rnet_x, "EPSG:27700")
   rnet_yp = sf::st_transform(rnet_y, "EPSG:27700")
 
@@ -88,11 +85,11 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
   rnet_merged_all_projected_buffer = sf::st_as_sf(rnet_merged_all_geos_buffer)
 
   # Transform the coordinate reference system of 'rnet_merged_all' to WGS 84 (EPSG:4326).
-  rnet_merged_all_buffer = sf::st_transform(rnet_merged_all_projected_buffer, "EPSG:4326")
+  # rnet_merged_all_buffer = sf::st_transform(rnet_merged_all_projected_buffer, "EPSG:4326")
 
   # Subsetting another dataset 'rnet_y' based on the spatial relation with 'rnet_merged_all_buffer'.
   # It selects features from 'rnet_y' that are within the boundaries of 'rnet_merged_all_buffer'.
-  rnet_y_subset = rnet_y[rnet_merged_all_buffer, , op = sf::st_within]
+  rnet_y_subset = rnet_y[rnet_merged_all_projected_buffer, , op = sf::st_within]
 
   # Filter 'rnet_y' to exclude geometries within 'within_join'
   rnet_y_rest = rnet_y[!rnet_y$geometry %in% rnet_y_subset$geometry, ]
diff --git a/code/tests/simplified_network_test.qmd b/code/tests/simplified_network_test.qmd
new file mode 100644
index 0000000..60ae766
--- /dev/null
+++ b/code/tests/simplified_network_test.qmd
@@ -0,0 +1,29 @@
+---
+format: gfm
+---
+
+# create a exmaple network
+
+```{r}
+library(tidyverse)
+library(targets)
+library(tidygraph)
+library(osmextract)
+library(foreach)
+library(iterators)
+library(parallel)
+library(doParallel)
+tar_source()
+
+zone = zonebuilder::zb_zone("Edinburgh", n_circles = 3) |> sf::st_transform(crs = "EPSG:27700")
+rnet_edingburgh = sf::st_read("outputdata/2024-11-01/edinburgh_and_lothians/combined_network_tile.geojson") |> sf::st_transform(crs = "EPSG:27700")
+rnet_edingburgh_zone = rnet_edingburgh[sf::st_union(zone), , op = sf::st_intersects]
+parameters$region[[1]] = "Edinburgh and Lothians"
+
+os_file_path = "inputdata/open_roads_scotland.gpkg"
+sf::st_geometry(os_scotland) = "geometry"
+
+system.time({
+rnet_simple = simplify_network(rnet_edingburgh_zone, parameters, region_boundary = zone)
+})
+```
\ No newline at end of file
diff --git a/route_ids.csv b/route_ids.csv
index 85dbf38..76a87e9 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,11 +1,7 @@
 nrow,plan,purpose,region,date,id
 111504,ebike,utility,Edinburgh and Lothians,2024-11-01,11221
 111504,quietest,utility,Edinburgh and Lothians,2024-11-01,11220
-<<<<<<< HEAD
 111504,fastest,utility,Edinburgh and Lothians,2024-11-01,11218
-=======
-111504,fastest,utility,Edinburgh and Lothians,2024-11-01,11219
->>>>>>> 0f4fae1 (add more route id)
 111504,balanced,utility,Edinburgh and Lothians,2024-11-01,11217
 90716,ebike,commute,Edinburgh and Lothians,2024-11-01,11216
 218877,fastest,commute,Glasgow and Strathclyde,2024-11-01,11214

From e5d67f21e836c15ba79ccc86d6e524373a9425c4 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Wed, 6 Nov 2024 13:04:15 +0000
Subject: [PATCH 06/33] pre processing os by adding osm orcp

---
 code/tests/simplified_network_test.qmd | 76 ++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/code/tests/simplified_network_test.qmd b/code/tests/simplified_network_test.qmd
index 60ae766..ee0bb70 100644
--- a/code/tests/simplified_network_test.qmd
+++ b/code/tests/simplified_network_test.qmd
@@ -13,8 +13,84 @@ library(foreach)
 library(iterators)
 library(parallel)
 library(doParallel)
+library(mapview)
 tar_source()
+```
 
+
+```{r}
+# get off road from osm to os, simple combine
+zone = zonebuilder::zb_zone("Edinburgh", n_circles = 3) |> sf::st_transform(crs = "EPSG:27700")
+
+os_file_path = "inputdata/open_roads_scotland.gpkg"
+os_scotland = sf::read_sf(os_file_path)
+sf::st_geometry(os_scotland) = "geometry"
+
+os_scotland = os_scotland[sf::st_union(zone), , op = sf::st_within]
+
+osm_file_path = "inputdata/connectivity_fixed_osm.gpkg"
+osm_scotland = sf::read_sf(osm_file_path)
+sf::st_geometry(osm_scotland) = "geometry"
+
+osm_scotland = osm_scotland[sf::st_union(zone), , op = sf::st_within]
+
+os_scotland = os_scotland |> sf::st_transform(crs = "EPSG:27700") |> dplyr::select(geometry)
+osm_scotland = osm_scotland |> sf::st_transform(crs = "EPSG:27700")
+
+rnet_merged_all_geos = geos::as_geos_geometry(os_scotland)
+
+rnet_merged_all_geos_buffer = geos::geos_buffer(rnet_merged_all_geos, distance = 30, params = geos::geos_buffer_params(quad_segs = 4))
+
+rnet_merged_all_projected_buffer = sf::st_as_sf(rnet_merged_all_geos_buffer)
+
+osm_scotland_subset = osm_scotland[sf::st_union(rnet_merged_all_projected_buffer), , op = sf::st_within]
+
+osm_scotland_rest = osm_scotland[!osm_scotland$geometry %in% osm_scotland_subset$geometry, ]
+
+os_scotland = dplyr::bind_rows(osm_scotland_rest, os_scotland)
+```
+
+```{r}
+parameters = jsonlite::read_json("parameters.json", simplifyVector = T)
+lads = sf::read_sf("inputdata/boundaries/la_regions_2023.geojson")
+# To test for a single local authority:
+# lads = lads |> filter(LAD23NM %in% c("City of Edinburgh", "Clackmannanshire"))
+date_folder = parameters$date_routing
+la_names = lads$LAD23NM
+output_folder = file.path("outputdata", date_folder)
+la_names_lowercase = snakecase::to_snake_case(la_names)
+```
+
+```{r}
+la_name = la_names[1]
+for (la_name in la_names) {
+  message("Processing la_name: ", la_name)
+  la_name_lowercase = snakecase::to_snake_case(la_name)
+  lads_la_name = lads |> filter(LAD23NM == la_name)
+  os_scotland_la_name = os_scotland[sf::st_union(lads_la_name), , op = sf::st_intersects]
+  # save os_scotland_la_name
+  os_scotland_la_name_f = file.path("inputdata", paste0("os_", la_name_lowercase, "_osm_orcp.geojson"))
+  
+  sf::st_write(os_scotland_la_name, os_scotland_la_name_f, delete_dsn = T)
+}
+```
+
+```{r}
+region_snake_case = snakecase::to_snake_case(region_names[1])
+base_name = paste0("OS_Scotland_Network_", region_snake_case, ".geojson")
+rnet_x_f = file.path("inputdata", base_name)
+rnet_x = sf::read_sf(rnet_x_f) |> sf::st_transform(crs = "EPSG:27700")
+
+osm_file_path = "inputdata/connectivity_fixed_osm.gpkg"
+osm_scotland = sf::read_sf(osm_file_path)
+sf::st_geometry(osm_scotland) = "geometry"
+
+
+
+```
+
+
+```{r}
 zone = zonebuilder::zb_zone("Edinburgh", n_circles = 3) |> sf::st_transform(crs = "EPSG:27700")
 rnet_edingburgh = sf::st_read("outputdata/2024-11-01/edinburgh_and_lothians/combined_network_tile.geojson") |> sf::st_transform(crs = "EPSG:27700")
 rnet_edingburgh_zone = rnet_edingburgh[sf::st_union(zone), , op = sf::st_intersects]

From 06480596348f73cc13cd60a66f1e257589ded230 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 11:32:49 +0000
Subject: [PATCH 07/33] Update route_ids for Glasgow

---
 route_ids.csv | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/route_ids.csv b/route_ids.csv
index 76a87e9..f7e913a 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,7 @@
 nrow,plan,purpose,region,date,id
+218877,ebike,commute,Glasgow and Strathclyde,2024-11-01,11312
+218877,balanced,commute,Glasgow and Strathclyde,2024-11-01,11303
+218877,quietest,commute,Glasgow and Strathclyde,2024-11-01,11302
 111504,ebike,utility,Edinburgh and Lothians,2024-11-01,11221
 111504,quietest,utility,Edinburgh and Lothians,2024-11-01,11220
 111504,fastest,utility,Edinburgh and Lothians,2024-11-01,11218

From a0c6876afec0e56668f9dc8a702fe5fc233f8559 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 11:34:45 +0000
Subject: [PATCH 08/33] Update gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index acacedc..38aec13 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,4 @@ npt_net_qgis
 scottraffic/
 *.geojson
 osm_os/
+nptscot.github.io/

From 61ac56b2ad192ea861f6fd2e730df5ef33a3ff7c Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 14:38:21 +0000
Subject: [PATCH 09/33] Update route_ids.csv

---
 route_ids.csv | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/route_ids.csv b/route_ids.csv
index f7e913a..e915cd8 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,6 @@
 nrow,plan,purpose,region,date,id
+93472,fastest,utility,Glasgow and Strathclyde,2024-11-01,11352
+93472,balanced,utility,Glasgow and Strathclyde,2024-11-01,11349
 218877,ebike,commute,Glasgow and Strathclyde,2024-11-01,11312
 218877,balanced,commute,Glasgow and Strathclyde,2024-11-01,11303
 218877,quietest,commute,Glasgow and Strathclyde,2024-11-01,11302

From d6fac91c5b18598638e55279171475ba2cacca71 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 16:54:47 +0000
Subject: [PATCH 10/33] More utility trips

---
 route_ids.csv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/route_ids.csv b/route_ids.csv
index e915cd8..ca74350 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,5 @@
 nrow,plan,purpose,region,date,id
+93472,quietest,utility,Glasgow and Strathclyde,2024-11-01,11353
 93472,fastest,utility,Glasgow and Strathclyde,2024-11-01,11352
 93472,balanced,utility,Glasgow and Strathclyde,2024-11-01,11349
 218877,ebike,commute,Glasgow and Strathclyde,2024-11-01,11312

From e8c224f0dd76880c717d6b9fb96cc68dfe1211d4 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 18:47:34 +0000
Subject: [PATCH 11/33] Updates with final routes

---
 parameters.json | 2 +-
 route_ids.csv   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/parameters.json b/parameters.json
index a276eed..5eae84b 100644
--- a/parameters.json
+++ b/parameters.json
@@ -8,7 +8,7 @@
   "disag_threshold": [100],
   "min_distance_meters": [500],
   "region_buffer_distance_meters": [5000],
-  "region": ["Glasgow and Strathclyde"],
+  "region": ["Highlands and Islands"],
   "generate_CN_start": [true],
   "coherent_sources": ["OS"],
   "coherent_percentile": [0.94, 0.9]
diff --git a/route_ids.csv b/route_ids.csv
index ca74350..517b1ed 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,5 @@
 nrow,plan,purpose,region,date,id
+93472,ebike,utility,Glasgow and Strathclyde,2024-11-01,11354
 93472,quietest,utility,Glasgow and Strathclyde,2024-11-01,11353
 93472,fastest,utility,Glasgow and Strathclyde,2024-11-01,11352
 93472,balanced,utility,Glasgow and Strathclyde,2024-11-01,11349

From 131b8a27733f0d25c221290ae3e5ed1003ba5979 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 18:55:09 +0000
Subject: [PATCH 12/33] Debugging error

---
 R/simplify_network.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/R/simplify_network.R b/R/simplify_network.R
index 4864af3..0bca272 100644
--- a/R/simplify_network.R
+++ b/R/simplify_network.R
@@ -82,6 +82,7 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
   rnet_merged_all_geos_buffer = geos::geos_buffer(rnet_merged_all_geos, distance = 30, params = geos::geos_buffer_params(quad_segs = 4))
 
   # Converting the buffered GEOS geometry back to an sf object.
+  browser()
   rnet_merged_all_projected_buffer = sf::st_as_sf(rnet_merged_all_geos_buffer)
 
   # Transform the coordinate reference system of 'rnet_merged_all' to WGS 84 (EPSG:4326).

From ca20e19c00d4fb13713873127bf3432b78e2e323 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 20:05:49 +0000
Subject: [PATCH 13/33] Debugging simplify network, speed things up

---
 R/simplify_network.R | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/R/simplify_network.R b/R/simplify_network.R
index 0bca272..990f3df 100644
--- a/R/simplify_network.R
+++ b/R/simplify_network.R
@@ -66,17 +66,8 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
   rnet_merged_all = rnet_merged_all |>
     dplyr::filter_at(columns_to_check, any_vars(!is.na(.)))
 
-  # Selecting only the geometry column from the 'rnet_merged_all' dataset.
-  rnet_merged_all_only_geometry = rnet_merged_all |> dplyr::select(geometry)
-
-  # Merging all geometries into a single geometry using st_union from the sf package.
-  rnet_merged_all_union = sf::st_union(rnet_merged_all_only_geometry)
-
-  # Transforming the merged geometry to a specific coordinate reference system (CRS), EPSG:27700.
-  rnet_merged_all_projected = sf::st_transform(rnet_merged_all_union, "EPSG:27700")
-
   # Converting the projected geometry into a GEOS geometry. GEOS is a library used for spatial operations.
-  rnet_merged_all_geos = geos::as_geos_geometry(rnet_merged_all_projected)
+  rnet_merged_all_geos = geos::as_geos_geometry(rnet_merged_all)
 
   # Creating a buffer around the GEOS geometry. This expands the geometry by a specified distance (in meters).
   rnet_merged_all_geos_buffer = geos::geos_buffer(rnet_merged_all_geos, distance = 30, params = geos::geos_buffer_params(quad_segs = 4))
@@ -90,13 +81,14 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
 
   # Subsetting another dataset 'rnet_y' based on the spatial relation with 'rnet_merged_all_buffer'.
   # It selects features from 'rnet_y' that are within the boundaries of 'rnet_merged_all_buffer'.
-  rnet_y_subset = rnet_y[rnet_merged_all_projected_buffer, , op = sf::st_within]
+  rnet_y_subset = sf::st_intersection(rnet_yp, rnet_merged_all)
 
   # Filter 'rnet_y' to exclude geometries within 'within_join'
-  rnet_y_rest = rnet_y[!rnet_y$geometry %in% rnet_y_subset$geometry, ]
+  rnet_y_rest = rnet_yp[!rnet_yp$geometry %in% rnet_y_subset$geometry, ]
 
   # Transform the CRS of the 'rnet_merged_all' object to WGS 84 (EPSG:4326)
   rnet_merged_all = sf::st_transform(rnet_merged_all, "EPSG:4326")
+  rnet_y_rest = sf::st_transform(rnet_y_rest, "EPSG:4326")
 
   # Combine 'rnet_y_rest' and 'rnet_merged_all' into a single dataset
   simplified_network = dplyr::bind_rows(rnet_y_rest, rnet_merged_all)

From f4cb88828b3dca894b420e52d898b2a1089ba8c7 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 20:06:53 +0000
Subject: [PATCH 14/33] Tidy up

---
 R/simplify_network.R | 4 ----
 _targets.R           | 1 +
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/R/simplify_network.R b/R/simplify_network.R
index 990f3df..907a96b 100644
--- a/R/simplify_network.R
+++ b/R/simplify_network.R
@@ -73,12 +73,8 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
   rnet_merged_all_geos_buffer = geos::geos_buffer(rnet_merged_all_geos, distance = 30, params = geos::geos_buffer_params(quad_segs = 4))
 
   # Converting the buffered GEOS geometry back to an sf object.
-  browser()
   rnet_merged_all_projected_buffer = sf::st_as_sf(rnet_merged_all_geos_buffer)
 
-  # Transform the coordinate reference system of 'rnet_merged_all' to WGS 84 (EPSG:4326).
-  # rnet_merged_all_buffer = sf::st_transform(rnet_merged_all_projected_buffer, "EPSG:4326")
-
   # Subsetting another dataset 'rnet_y' based on the spatial relation with 'rnet_merged_all_buffer'.
   # It selects features from 'rnet_y' that are within the boundaries of 'rnet_merged_all_buffer'.
   rnet_y_subset = sf::st_intersection(rnet_yp, rnet_merged_all)
diff --git a/_targets.R b/_targets.R
index 95473cb..e087db4 100644
--- a/_targets.R
+++ b/_targets.R
@@ -1243,6 +1243,7 @@ list(
     # TODO (nice to have): replace with global setting (needs testing):
     use_sf_s2_status = sf::sf_use_s2()
     sf::sf_use_s2(FALSE)
+    # Debugging the function:
     rnet_simple = simplify_network(combined_network_tile, parameters, region_boundary)
     make_geojson_zones(rnet_simple, paste0(region_folder, "/simplified_network.geojson"))
     # Restore previous status

From b199c9ea93e975577add539171cf2c6b53cd8900 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 20:09:18 +0000
Subject: [PATCH 15/33] Full build for Glasgow again

---
 parameters.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parameters.json b/parameters.json
index 5eae84b..a276eed 100644
--- a/parameters.json
+++ b/parameters.json
@@ -8,7 +8,7 @@
   "disag_threshold": [100],
   "min_distance_meters": [500],
   "region_buffer_distance_meters": [5000],
-  "region": ["Highlands and Islands"],
+  "region": ["Glasgow and Strathclyde"],
   "generate_CN_start": [true],
   "coherent_sources": ["OS"],
   "coherent_percentile": [0.94, 0.9]

From 3f2a167d7c65c46e668d70635a5c4294cd220452 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 7 Nov 2024 20:38:29 +0000
Subject: [PATCH 16/33] Tweak simplify_network bit

---
 R/simplify_network.R | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/R/simplify_network.R b/R/simplify_network.R
index 907a96b..7af893e 100644
--- a/R/simplify_network.R
+++ b/R/simplify_network.R
@@ -77,7 +77,11 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
 
   # Subsetting another dataset 'rnet_y' based on the spatial relation with 'rnet_merged_all_buffer'.
   # It selects features from 'rnet_y' that are within the boundaries of 'rnet_merged_all_buffer'.
-  rnet_y_subset = sf::st_intersection(rnet_yp, rnet_merged_all)
+  # rnet_y_subset = sf::st_intersection(rnet_yp, rnet_merged_all_projected_buffer)
+  browser()
+  rnet_yp_points = sf::st_point_on_surface(rnet_yp)
+  rnet_y_points_subset = rnet_y_points[rnet_merged_all_projected_buffer, ]
+  rnet_y_subset = rnet_yp[rnet_y_points_subset, ]
 
   # Filter 'rnet_y' to exclude geometries within 'within_join'
   rnet_y_rest = rnet_yp[!rnet_yp$geometry %in% rnet_y_subset$geometry, ]

From ce1e9788bdca00081e661031c6350346fc670cbb Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Fri, 8 Nov 2024 09:28:50 +0000
Subject: [PATCH 17/33] Fix simplified network

---
 R/simplify_network.R | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/R/simplify_network.R b/R/simplify_network.R
index 7af893e..d131680 100644
--- a/R/simplify_network.R
+++ b/R/simplify_network.R
@@ -78,10 +78,10 @@ simplify_network = function(rnet_y, parameters, region_boundary) {
   # Subsetting another dataset 'rnet_y' based on the spatial relation with 'rnet_merged_all_buffer'.
   # It selects features from 'rnet_y' that are within the boundaries of 'rnet_merged_all_buffer'.
   # rnet_y_subset = sf::st_intersection(rnet_yp, rnet_merged_all_projected_buffer)
-  browser()
+  # browser()
   rnet_yp_points = sf::st_point_on_surface(rnet_yp)
-  rnet_y_points_subset = rnet_y_points[rnet_merged_all_projected_buffer, ]
-  rnet_y_subset = rnet_yp[rnet_y_points_subset, ]
+  rnet_yp_points_subset = rnet_yp_points[rnet_merged_all_projected_buffer, ]
+  rnet_y_subset = rnet_yp[rnet_yp_points_subset, ]
 
   # Filter 'rnet_y' to exclude geometries within 'within_join'
   rnet_y_rest = rnet_yp[!rnet_yp$geometry %in% rnet_y_subset$geometry, ]

From b27e54a0ac38e72b55d0ef589a449459df36ffe3 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Tue, 19 Nov 2024 16:06:47 +0000
Subject: [PATCH 18/33] fix UNCLOSED_STRING error in the rs_commute_quietest

---
 _targets.R | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/_targets.R b/_targets.R
index e087db4..baca7bc 100644
--- a/_targets.R
+++ b/_targets.R
@@ -726,7 +726,6 @@ list(
     os_pois_subset[region_boundary_buffered, ] |>
       sf::st_transform("EPSG:27700")
   }),
-
   tar_target(grid, {
     grid = readRDS("./inputdata/grid_scot.Rds")
     grid = sf::st_transform(grid, "EPSG:4326")
@@ -776,14 +775,13 @@ list(
   # Combined utility trip purposes --------------------------------------------
 
   tar_target(od_utility_combined, {
-
     od_utility_combined = rbind(od_shopping, od_visiting, od_leisure) |>
       dplyr::slice_max(n = parameters$max_to_route, order_by = all, with_ties = FALSE)
     sum(od_utility_combined$bicycle) / sum(od_utility_combined$all)
 
     # Get % cycling for commuting per zone
     # pcycle_regional = sum(commute_stats$comm_orig_bicycle, na.rm = TRUE) /
-      # sum(commute_stats$comm_orig_all, na.rm = TRUE)
+    # sum(commute_stats$comm_orig_all, na.rm = TRUE)
     pcycle_national = 0.016
 
     commute_stats_minimal = commute_stats |>
@@ -792,7 +790,7 @@ list(
       dplyr::transmute(
         DataZone,
         multiplier = (comm_orig_bicycle / comm_orig_all) /
-         pcycle_national
+          pcycle_national
       ) |>
       # 0 to 0.1:
       dplyr::mutate(multiplier = case_when(
@@ -978,7 +976,7 @@ list(
 
   tar_target(utility_stats_baseline, {
     stats = sf::st_drop_geometry(od_utility_combined)
-     stats = stats[, c(
+    stats = stats[, c(
       "startDZ", "endDZ", "purpose", "all", "car",
       "foot", "bicycle", "public_transport", "taxi"
     )]
@@ -1280,7 +1278,6 @@ list(
     responce = system(command_all, intern = TRUE)
     responce
   }),
-
   tar_target(pmtiles_buildings, {
     check = length(zones_dasymetric_tile)
 
@@ -1441,7 +1438,7 @@ list(
 
     message("Saving outputs for ", parameters$date_routing)
 
-    saveRDS(od_commute_subset, file.path(region_folder, "od_commute_subset.Rds"))  
+    saveRDS(od_commute_subset, file.path(region_folder, "od_commute_subset.Rds"))
     saveRDS(zones_stats, file.path(region_folder, "zones_stats.Rds"))
     saveRDS(school_stats, file.path(region_folder, "school_stats.Rds"))
 
@@ -1453,7 +1450,6 @@ list(
     sf::write_sf(combined_network, file.path(region_folder, "combined_network.gpkg"), delete_dsn = TRUE)
     as.character(Sys.Date())
   }),
-
   tar_target(metadata, {
     # TODO: generate build summary
     # metadata_all = tar_meta()

From 7f94edfdfc6796a5b6ce8eeef703ce7e9bd0f034 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <74598734+wangzhao0217@users.noreply.github.com>
Date: Wed, 27 Nov 2024 14:34:35 +0000
Subject: [PATCH 19/33] update traffic results and save CN as gpkg (#554)

* update traffic results and save CN as gpkg

* add httr and set set_config(timeout(seconds = 600))

* fix error

* 2 -> 600

* set to 11
---
 R/corenet_build_OS.R | 17 +++++++++++------
 R/pkgs.R             |  3 ++-
 _targets.R           |  2 ++
 code/build.R         | 11 ++++++++---
 4 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/R/corenet_build_OS.R b/R/corenet_build_OS.R
index e6ec8ff..19848f6 100644
--- a/R/corenet_build_OS.R
+++ b/R/corenet_build_OS.R
@@ -353,12 +353,17 @@ corenet_build_OS = function(os_scotland, osm_scotland, region_names,cities_regio
         return(NULL)
       }
     }))
-    # Define the file path for the combined GeoJSON
-    combined_CN_file = glue::glue("{output_folder}/combined_CN_{number}_{date_folder}_OS.geojson")
+    # Define the file paths for the combined GeoJSON and GeoPackage
+    combined_CN_geojson_file = glue::glue("{output_folder}/combined_CN_{number}_{date_folder}_OS.geojson")
+    combined_CN_gpkg_file = glue::glue("{output_folder}/combined_CN_{number}_{date_folder}_OS.gpkg")
 
-    # Write the combined GeoJSON to a file
-    sf::st_write(combined_CN_geojson, combined_CN_file, delete_dsn = TRUE)
-    cat("Combined cohesive networks GeoJSON file for group", number, "has been saved to:", combined_CN_file, "\n")
+    # Write the combined data to GeoJSON and GeoPackage files
+    sf::st_write(combined_CN_geojson, combined_CN_geojson_file, delete_dsn = TRUE)
+    sf::st_write(combined_CN_geojson, combined_CN_gpkg_file, delete_dsn = TRUE)
+
+    # Print messages indicating where the files have been saved
+    cat("Combined cohesive networks GeoJSON file for group", number, "has been saved to:", combined_CN_geojson_file, "\n")
+    cat("Combined cohesive networks GeoPackage file for group", number, "has been saved to:", combined_CN_gpkg_file, "\n")
 
     # Define the path for the PMtiles
     combined_CN_pmtiles = glue::glue("{output_folder}/combined_CN_{number}_{date_folder}_OS.pmtiles")
@@ -376,7 +381,7 @@ corenet_build_OS = function(os_scotland, osm_scotland, region_names,cities_regio
       ' --buffer=5',
       ' -rg',
       ' --force ',
-      combined_CN_file
+      combined_CN_geojson_file
     )
 
     # Execute the command and capture output
diff --git a/R/pkgs.R b/R/pkgs.R
index 7829073..a95e12e 100644
--- a/R/pkgs.R
+++ b/R/pkgs.R
@@ -21,6 +21,7 @@ get_pkgs = function() {
   "tidyverse", # Includes dplyr, ggplot2, tidyr, stringr etc.
   "zonebuilder", # For creating zones for spatial analysis
   "iterators", # For creating iterators
-  "doParallel" # For parallel processing
+  "doParallel", # For parallel processing
+  "httr"
   )
 }
diff --git a/_targets.R b/_targets.R
index baca7bc..3f1fd4b 100644
--- a/_targets.R
+++ b/_targets.R
@@ -19,7 +19,9 @@ if (!"corenet" %in% installed.packages()) {
 library(targets) # Needed to make targets work
 library(magrittr) # Light load of |>
 library(sf) # Needed for sf support
+library(httr)
 set.seed(2023)
+set_config(timeout(seconds = 600))
 tar_source()
 pkgs = get_pkgs()
 
diff --git a/code/build.R b/code/build.R
index 97a423e..8ec3122 100644
--- a/code/build.R
+++ b/code/build.R
@@ -198,15 +198,20 @@ if (GENERATE_CDB) {
 
       #     
       cycle_net_traffic = level_of_service(cycle_net_traffic)
-      
+            
       cbd_layer = cycle_net_traffic |>
         transmute(
           osm_id,
           highway,
-          `Traffic volume` = final_traffic,
           `Speed limit` = final_speed,
           `Infrastructure type` = cycle_segregation,
-          `Level of Service`
+          `Level of Service`,
+          `Traffic volume category` = case_when(
+            final_traffic >= 0 & final_traffic <= 1999 ~ "0 to 1999",
+            final_traffic >= 2000 & final_traffic <= 3999 ~ "2000 to 3999",
+            final_traffic >= 4000 ~ "4000+",
+            TRUE ~ NA_character_
+          )
         )
       # save file for individual district
       district_name = district_geom$LAD23NM |> 

From 7dfa7e9266feaffa91c7947f6157246ab6668f5e Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Wed, 27 Nov 2024 14:37:17 +0000
Subject: [PATCH 20/33] apply robin's suggestion

---
 code/build.R | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/code/build.R b/code/build.R
index 8ec3122..1de675c 100644
--- a/code/build.R
+++ b/code/build.R
@@ -207,9 +207,9 @@ if (GENERATE_CDB) {
           `Infrastructure type` = cycle_segregation,
           `Level of Service`,
           `Traffic volume category` = case_when(
-            final_traffic >= 0 & final_traffic <= 1999 ~ "0 to 1999",
-            final_traffic >= 2000 & final_traffic <= 3999 ~ "2000 to 3999",
-            final_traffic >= 4000 ~ "4000+",
+            final_traffic >= 0 & final_traffic <= 1999.5 ~ "0 to 1999",
+            final_traffic >= 1999.5 & final_traffic <= 3999.5 ~ "2000 to 3999",
+            final_traffic >= 3999.5 ~ "4000+",
             TRUE ~ NA_character_
           )
         )

From f65cd098bb3e3480dc5ad97de50a1681b0095521 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <74598734+wangzhao0217@users.noreply.github.com>
Date: Wed, 27 Nov 2024 14:52:50 +0000
Subject: [PATCH 21/33] Update final_traffic

Co-authored-by: Robin Lovelace <Robinlovelace@users.noreply.github.com>
---
 code/build.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/code/build.R b/code/build.R
index 1de675c..8068451 100644
--- a/code/build.R
+++ b/code/build.R
@@ -207,8 +207,8 @@ if (GENERATE_CDB) {
           `Infrastructure type` = cycle_segregation,
           `Level of Service`,
           `Traffic volume category` = case_when(
-            final_traffic >= 0 & final_traffic <= 1999.5 ~ "0 to 1999",
-            final_traffic >= 1999.5 & final_traffic <= 3999.5 ~ "2000 to 3999",
+           final_traffic >= 0 & final_traffic < 1999.5 ~ "0 to 1999",
+            final_traffic >= 1999.5 & final_traffic < 3999.5 ~ "2000 to 3999",
             final_traffic >= 3999.5 ~ "4000+",
             TRUE ~ NA_character_
           )

From b1f32e7328c9c35053d7e54a00c28afbe8e72bae Mon Sep 17 00:00:00 2001
From: wangzhao0217 <74598734+wangzhao0217@users.noreply.github.com>
Date: Wed, 27 Nov 2024 14:53:07 +0000
Subject: [PATCH 22/33] Update _targets.R

Co-authored-by: Robin Lovelace <Robinlovelace@users.noreply.github.com>
---
 _targets.R | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/_targets.R b/_targets.R
index 3f1fd4b..0f99e52 100644
--- a/_targets.R
+++ b/_targets.R
@@ -19,9 +19,8 @@ if (!"corenet" %in% installed.packages()) {
 library(targets) # Needed to make targets work
 library(magrittr) # Light load of |>
 library(sf) # Needed for sf support
-library(httr)
 set.seed(2023)
-set_config(timeout(seconds = 600))
+httr::set_config(httr::timeout(seconds = 600))
 tar_source()
 pkgs = get_pkgs()
 

From c13f93fa6091e67b5a596177f733fcc9b7b15d30 Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Wed, 27 Nov 2024 16:37:16 +0000
Subject: [PATCH 23/33] Use newer version of traffic data

---
 code/build.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/code/build.R b/code/build.R
index 8068451..8d16026 100644
--- a/code/build.R
+++ b/code/build.R
@@ -57,12 +57,12 @@ if (GENERATE_CDB) {
 
   library(osmactive)
   # See https://github.com/nptscot/osmactive/blob/main/code/classify-roads.R and traffic-volumes.R
-  f_traffic = "scottraffic/final_estimates_Scotland_higherror_discarded.gpkg"
+  f_traffic = "scottraffic/final_estimates_Scotland.gpkg"
   if (!file.exists(f_traffic)) {
     system("gh repo clone nptscot/scottraffic")
     setwd("scottraffic")
     system("gh release list")
-    system("gh release download v5 --clobber")
+    system("gh release download v6 --clobber")
     setwd("..")
   }
   traffic_volumes_scotland = sf::read_sf(f_traffic)

From 80011f8e1701467be2d4f43296f928b0d046609a Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Wed, 27 Nov 2024 16:37:48 +0000
Subject: [PATCH 24/33] Do not force osm download every time

---
 code/build.R | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/code/build.R b/code/build.R
index 8d16026..8213d3b 100644
--- a/code/build.R
+++ b/code/build.R
@@ -60,15 +60,17 @@ if (GENERATE_CDB) {
   f_traffic = "scottraffic/final_estimates_Scotland.gpkg"
   if (!file.exists(f_traffic)) {
     system("gh repo clone nptscot/scottraffic")
+    file.remove(f_traffic)
     setwd("scottraffic")
     system("gh release list")
-    system("gh release download v6 --clobber")
+    system("gh release download v6")
     setwd("..")
   }
   traffic_volumes_scotland = sf::read_sf(f_traffic)
 
-  # Generate cycle_net - this is slow, we should save the file
-  osm_national = get_travel_network("Scotland", force_download = T)
+  # Generate cycle_net: forcing update:
+  # osm_national = get_travel_network("Scotland", force_download = TRUE)
+  osm_national = get_travel_network("Scotland")
   # saveRDS(osm_national, "inputdata/osm_national_2024_05_23")
 
   # Generate road segment midpoints

From cea2fabe45fe48a92bb464e8b1614894f1a72cab Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 28 Nov 2024 10:53:43 +0000
Subject: [PATCH 25/33] Update cbd_layer traffic categories, see
 https://github.com/nptscot/nptscot.github.io/issues/267, close #496

---
 code/build.R | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/code/build.R b/code/build.R
index 8213d3b..68a6bd6 100644
--- a/code/build.R
+++ b/code/build.R
@@ -212,6 +212,8 @@ if (GENERATE_CDB) {
            final_traffic >= 0 & final_traffic < 1999.5 ~ "0 to 1999",
             final_traffic >= 1999.5 & final_traffic < 3999.5 ~ "2000 to 3999",
             final_traffic >= 3999.5 ~ "4000+",
+            `Infrastructure type` == "Off road cycleway" ~ NA_character_,
+            highway %in% c("footway", "path", "pedestrian", "steps") ~ NA_character_,
             TRUE ~ NA_character_
           )
         )
@@ -228,6 +230,10 @@ if (GENERATE_CDB) {
   }
 
   # Combine all CBD files into a single file
+  # Remove combined file if it already exists:
+  if (file.exists(cbd_filename)) {
+    file.remove(cbd_filename)
+  }
   cbd_files = list.files(output_folder, pattern = "cbd_layer_.*\\.geojson$", full.names = TRUE)
   # Create an empty cbd_layers and cbd_layer
   cbd_layers = sf::st_sf(geometry = st_sfc())
@@ -235,9 +241,6 @@ if (GENERATE_CDB) {
   cbd_layers = lapply(cbd_files, sf::read_sf)
   cbd_layer = do.call(rbind, cbd_layers)
   cbd_filename = paste0(output_folder, "/cbd_layer_", date_folder, ".geojson")
-  if (file.exists(cbd_filename)) {
-    file.remove(cbd_filename)
-  }
   sf::write_sf(cbd_layer, cbd_filename)
   fs::file_size(cbd_filename)
 
@@ -691,9 +694,6 @@ if (PUSH_TO_GITHUB) {
       parameters$max_to_route > 20e3
   is_linux = Sys.info()[["sysname"]] == "Linux"
   if (full_build) {
-    v = paste0("v", Sys.Date(), "_commit_", commit$commit)
-    v = gsub(pattern = " |:", replacement = "-", x = v)
-    # Or latest release:
     setwd(glue::glue(getwd(),"/", output_folder))
     system("gh release list")
     v = glue::glue("v{date_folder}")

From 19d85c407691b2b3afc9993fd8016c925ceef0ff Mon Sep 17 00:00:00 2001
From: robinlovelace <rob00x@gmail.com>
Date: Thu, 28 Nov 2024 16:05:55 +0000
Subject: [PATCH 26/33] Update traffic volume categorisation code

---
 code/build.R | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/code/build.R b/code/build.R
index 68a6bd6..7d9706e 100644
--- a/code/build.R
+++ b/code/build.R
@@ -212,8 +212,6 @@ if (GENERATE_CDB) {
            final_traffic >= 0 & final_traffic < 1999.5 ~ "0 to 1999",
             final_traffic >= 1999.5 & final_traffic < 3999.5 ~ "2000 to 3999",
             final_traffic >= 3999.5 ~ "4000+",
-            `Infrastructure type` == "Off road cycleway" ~ NA_character_,
-            highway %in% c("footway", "path", "pedestrian", "steps") ~ NA_character_,
             TRUE ~ NA_character_
           )
         )
@@ -241,6 +239,15 @@ if (GENERATE_CDB) {
   cbd_layers = lapply(cbd_files, sf::read_sf)
   cbd_layer = do.call(rbind, cbd_layers)
   cbd_filename = paste0(output_folder, "/cbd_layer_", date_folder, ".geojson")
+  # Update traffic volumes for off road cycleways
+  cbd_layer = cbd_layer |>
+    mutate(
+      `Traffic volume category` = case_when(
+        `Infrastructure type` == "Off Road Cycleway" ~ NA_character_,
+        highway %in% c("footway", "path", "pedestrian", "steps") ~ NA_character_,
+        TRUE ~ `Traffic volume category`
+      )
+    )
   sf::write_sf(cbd_layer, cbd_filename)
   fs::file_size(cbd_filename)
 

From 8704d08e781d8beb7d4402915c0a9ccce9058c81 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Thu, 28 Nov 2024 15:47:57 +0000
Subject: [PATCH 27/33] update oute_ids.csv

---
 _targets.R      | 2 +-
 parameters.json | 2 +-
 route_ids.csv   | 9 +++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/_targets.R b/_targets.R
index 0f99e52..b98dbfa 100644
--- a/_targets.R
+++ b/_targets.R
@@ -20,7 +20,7 @@ library(targets) # Needed to make targets work
 library(magrittr) # Light load of |>
 library(sf) # Needed for sf support
 set.seed(2023)
-httr::set_config(httr::timeout(seconds = 600))
+httr::set_config(httr::timeout(seconds = 6000))
 tar_source()
 pkgs = get_pkgs()
 
diff --git a/parameters.json b/parameters.json
index a276eed..2abf288 100644
--- a/parameters.json
+++ b/parameters.json
@@ -2,7 +2,7 @@
   "plans": ["fastest", "balanced", "quietest", "ebike"],
   "min_flow": [1],
   "max_to_route": [1000000],
-  "date_routing": ["2024-11-01"],
+  "date_routing": ["2024-12-01"],
   "geo_subset": [false],
   "open_data_build": [false],
   "disag_threshold": [100],
diff --git a/route_ids.csv b/route_ids.csv
index 517b1ed..1df0f51 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,13 @@
 nrow,plan,purpose,region,date,id
+93472,fastest,utility,Glasgow and Strathclyde,2024-12-01,11539
+93472,balanced,utility,Glasgow and Strathclyde,2024-12-01,11538
+218877,ebike,commute,Glasgow and Strathclyde,2024-12-01,11536
+218877,balanced,commute,Glasgow and Strathclyde,2024-12-01,11534
+218877,fastest,commute,Glasgow and Strathclyde,2024-12-01,11532
+29032,ebike,school,Glasgow and Strathclyde,2024-12-01,11531
+29032,balanced,school,Glasgow and Strathclyde,2024-12-01,11530
+29032,quietest,school,Glasgow and Strathclyde,2024-12-01,11529
+29032,fastest,school,Glasgow and Strathclyde,2024-12-01,11528
 93472,ebike,utility,Glasgow and Strathclyde,2024-11-01,11354
 93472,quietest,utility,Glasgow and Strathclyde,2024-11-01,11353
 93472,fastest,utility,Glasgow and Strathclyde,2024-11-01,11352

From 14fa73ade7db7af9b8a0dfc4ff5227b450489e7d Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Thu, 28 Nov 2024 16:07:13 +0000
Subject: [PATCH 28/33] update route id

---
 route_ids.csv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/route_ids.csv b/route_ids.csv
index 1df0f51..719930d 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,5 @@
 nrow,plan,purpose,region,date,id
+93472,quietest,utility,Glasgow and Strathclyde,2024-12-01,11540
 93472,fastest,utility,Glasgow and Strathclyde,2024-12-01,11539
 93472,balanced,utility,Glasgow and Strathclyde,2024-12-01,11538
 218877,ebike,commute,Glasgow and Strathclyde,2024-12-01,11536

From ca6e49914ccbc1ad6429f0c71afe7733cb07d83e Mon Sep 17 00:00:00 2001
From: Robin Lovelace <Robinlovelace@users.noreply.github.com>
Date: Thu, 28 Nov 2024 22:32:11 +0000
Subject: [PATCH 29/33] Update boundaries, to low water mark definition (#556)

---
 code/prep_admin_bounds.R | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/code/prep_admin_bounds.R b/code/prep_admin_bounds.R
index ec74816..44e693e 100644
--- a/code/prep_admin_bounds.R
+++ b/code/prep_admin_bounds.R
@@ -18,7 +18,31 @@ library(sf)
 # la = sf::read_sf("la.gpkg")
 # plot(la)
 
-la = sf::read_sf("https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/Local_Authority_Districts_December_2023_Boundaries_UK_BSC/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson")
+
+# Boundary options: 
+# https://www.ons.gov.uk/methodology/geography/geographicalproducts/digitalboundaries
+    # (BFC) Full resolution - clipped to the coastline (Mean High Water mark)
+    # (BFE) Full resolution - extent of the realm (usually this is the Mean Low Water mark but, in some cases, boundaries extend beyond this to include offshore islands)
+    # (BGC) Generalised (20m) - clipped to the coastline (Mean High Water mark)
+    # (BUC) Ultra Generalised (500m) - clipped to the coastline (Mean High Water mark)
+
+
+la_bsc = sf::read_sf("https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/Local_Authority_Districts_December_2023_Boundaries_UK_BSC/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson")
+la_bfe = sf::read_sf("https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/Local_Authority_Districts_December_2023_Boundaries_UK_BFE/FeatureServer/0/query?outFields=*&where=1%3D1&f=geojson")
+
+# BFE version is 50x larger than BSC
+object.size(la_bfe) |> as.numeric() /
+  object.size(la_bsc) |> as.numeric() 
+
+la_bfe_simplified = rmapshaper::ms_simplify(la_bfe, keep = 0.02)
+
+# BFE version is now 1.4x larger than BSC
+object.size(la_bfe_simplified) |> as.numeric() /
+  object.size(la_bsc) |> as.numeric()
+
+mapview::mapview(la_bsc) + mapview::mapview(la_bfe_simplified)
+
+la = la_bfe_simplified
 
 la = la[, c("LAD23CD", "LAD23NM")]
 # LAs in Scotland, CD starts with "S":
@@ -75,10 +99,11 @@ la_regions |>
 # check for NAs: 
 la_regions[is.na(la_regions$Region),]
 
+sf::write_sf(la_regions, "la_regions_scotland_bfe_simplified_2023.geojson", delete_dsn = TRUE)
+sf::write_sf(la, "la_uk_bfe_simplified_2023.geojson", delete_dsn = TRUE)
+
 # system("gh release upload boundaries-2024 las_scotland_2023.geojson las_2023.geojson --clobber")
-dir.create("inputdata/boundaries", showWarnings = FALSE)
-sf::write_sf(la_regions, "inputdata/boundaries/la_regions_2023.geojson", delete_dsn = TRUE)
-sf::write_sf(la, "inputdata/boundaries/las_2023.geojson", delete_dsn = TRUE)
+system("gh release upload boundaries-2024 la_regions_scotland_bfe_simplified_2023.geojson la_uk_bfe_simplified_2023.geojson --clobber")
 
 # https://github.com/nptscot/npt/releases/download/boundaries-2024/las_2023.geojson
 

From cdeb34d0926090755ff681b2f9a4b1faeb922c03 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Thu, 28 Nov 2024 22:32:40 +0000
Subject: [PATCH 30/33] update route ids

---
 route_ids.csv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/route_ids.csv b/route_ids.csv
index 719930d..e4c8149 100644
--- a/route_ids.csv
+++ b/route_ids.csv
@@ -1,4 +1,5 @@
 nrow,plan,purpose,region,date,id
+93472,ebike,utility,Glasgow and Strathclyde,2024-12-01,11541
 93472,quietest,utility,Glasgow and Strathclyde,2024-12-01,11540
 93472,fastest,utility,Glasgow and Strathclyde,2024-12-01,11539
 93472,balanced,utility,Glasgow and Strathclyde,2024-12-01,11538

From 9619e8b626fc44744d82151abcfa9dce542b3dd4 Mon Sep 17 00:00:00 2001
From: Robin Lovelace <Robinlovelace@users.noreply.github.com>
Date: Thu, 28 Nov 2024 22:43:22 +0000
Subject: [PATCH 31/33] Fix lads (#557)

* Update boundaries, to low water mark definition

* Use new lad boundaries
---
 code/build.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/build.R b/code/build.R
index 7d9706e..4d7a11f 100644
--- a/code/build.R
+++ b/code/build.R
@@ -11,7 +11,7 @@ library(doParallel)
 tar_source()
 
 parameters = jsonlite::read_json("parameters.json", simplifyVector = T)
-lads = sf::read_sf("inputdata/boundaries/la_regions_2023.geojson")
+lads = sf::read_sf("inputdata/boundaries/la_regions_scotland_bfe_simplified_2023.geojson")
 date_folder = parameters$date_routing
 output_folder = file.path("outputdata", date_folder)
 

From f5c1d55184c9c7d9fc44c6572da0d5998e67e622 Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Thu, 28 Nov 2024 22:44:36 +0000
Subject: [PATCH 32/33] set nov

---
 parameters.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parameters.json b/parameters.json
index 2abf288..a276eed 100644
--- a/parameters.json
+++ b/parameters.json
@@ -2,7 +2,7 @@
   "plans": ["fastest", "balanced", "quietest", "ebike"],
   "min_flow": [1],
   "max_to_route": [1000000],
-  "date_routing": ["2024-12-01"],
+  "date_routing": ["2024-11-01"],
   "geo_subset": [false],
   "open_data_build": [false],
   "disag_threshold": [100],

From 762ad140f014aaf19cc92a2ea6e63f85da9297ce Mon Sep 17 00:00:00 2001
From: wangzhao0217 <wangzhao0217@gmail.com>
Date: Fri, 29 Nov 2024 09:17:35 +0000
Subject: [PATCH 33/33]  remove line cause problem

---
 code/build.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/build.R b/code/build.R
index 4d7a11f..61a9c1c 100644
--- a/code/build.R
+++ b/code/build.R
@@ -321,7 +321,7 @@ if (parameters$generate_CN_start) {
 # mapview::mapview(cn_test, zcol = "road_function")
 
 # Combine regional outputs 
----------------------------------------------------
+
 GENERATE_PMTILES = TRUE
 
 if (GENERATE_PMTILES) {