From f6f6c7cd3ee126b28f6a075b5537fed7ce803123 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 09:57:33 -0700 Subject: [PATCH 01/12] output plugins return vec of results --- python/nrel/routee/compass/compass_app.py | 10 ++-- .../src/app/compass/compass_app.rs | 50 ++++++++++++------- .../output/default/edgeidlist/plugin.rs | 6 +-- .../plugin/output/default/summary/plugin.rs | 8 +-- .../plugin/output/default/traversal/plugin.rs | 8 +-- .../src/plugin/output/default/uuid/plugin.rs | 4 +- .../src/plugin/output/output_plugin.rs | 2 +- 7 files changed, 54 insertions(+), 34 deletions(-) diff --git a/python/nrel/routee/compass/compass_app.py b/python/nrel/routee/compass/compass_app.py index 4a3f546f..df96e9d2 100644 --- a/python/nrel/routee/compass/compass_app.py +++ b/python/nrel/routee/compass/compass_app.py @@ -3,11 +3,13 @@ import json from pathlib import Path -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union from nrel.routee.compass.routee_compass_py import ( CompassAppWrapper, ) +Query = Dict[str, Any] +Result = List[Dict[str, Any]] class CompassApp: """ @@ -41,8 +43,8 @@ def from_config_file(cls, config_file: Union[str, Path]) -> CompassApp: return cls(app) def run( - self, query: Union[Dict[str, Any], List[Dict[str, Any]]] - ) -> List[Dict[str, Any]]: + self, query: Union[Query, List[Query]] + ) -> Result: """ Run a query (or multiple queries) against the CompassApp @@ -86,3 +88,5 @@ def run( if single_query and len(results) == 1: return results[0] return results + + diff --git a/rust/routee-compass/src/app/compass/compass_app.rs b/rust/routee-compass/src/app/compass/compass_app.rs index 95a46185..9b25524e 100644 --- a/rust/routee-compass/src/app/compass/compass_app.rs +++ b/rust/routee-compass/src/app/compass/compass_app.rs @@ -250,13 +250,14 @@ impl CompassApp { queries .iter() .map(|q| self.run_single_query(q.clone())) - .collect::, CompassAppError>>() + .collect::>, CompassAppError>>() }) - .collect::>, CompassAppError>>()?; + .collect::>>, CompassAppError>>()?; let run_result = run_query_result .into_iter() .flatten() + .flatten() .chain(input_error_responses) .collect(); @@ -278,7 +279,7 @@ impl CompassApp { pub fn run_single_query( &self, query: serde_json::Value, - ) -> Result { + ) -> Result, CompassAppError> { let search_result = self.search_app.run_vertex_oriented(&query); let output = apply_output_processing( (&query, search_result), @@ -337,7 +338,7 @@ pub fn apply_output_processing( response_data: (&serde_json::Value, Result), search_app: &SearchApp, output_plugins: &Vec>, -) -> serde_json::Value { +) -> Vec { let (req, res) = response_data; match res { Err(e) => { @@ -345,7 +346,7 @@ pub fn apply_output_processing( "request": req, "error": e.to_string() }); - error_output + vec![error_output] } Ok(result) => { // should be moved into TraversalModel::summary, queries requesting @@ -367,20 +368,20 @@ pub fn apply_output_processing( let route = result.route.to_vec(); let last_edge_traversal = match route.last() { None => { - return serde_json::json!({ + return vec![serde_json::json!({ "request": req, "error": "route was empty" - }); + })]; } Some(et) => et, }; let tmodel = match search_app.get_traversal_model_reference(req) { Err(e) => { - return serde_json::json!({ + return vec![serde_json::json!({ "request": req, "error": e.to_string() - }) + })] } Ok(tmodel) => tmodel, }; @@ -395,20 +396,35 @@ pub fn apply_output_processing( "tree_edge_count": result.tree.len(), "traversal_summary": tmodel.serialize_state_with_info(&last_edge_traversal.result_state), }); - let init_acc: Result = Ok(init_output); - let json_result = output_plugins + let init_acc: Result, PluginError> = Ok(vec![init_output]); + let json_result = output_plugins .iter() - .fold(init_acc, move |acc, plugin| match acc { - Err(e) => Err(e), - Ok(json) => plugin.process(&json, Ok(&result)), + .fold(init_acc, |acc, p| { + acc.and_then(|outer| { + outer + .iter() + .map(|output| p.process(output, Ok(&result))) + .collect::, PluginError>>() + .map(|inner| { + inner + .into_iter() + .flatten() + .collect::>() + }) + }) }) - .map_err(CompassAppError::PluginError); - match json_result { - Err(e) => { + .map_err(|e| { serde_json::json!({ "request": req, "error": e.to_string() }) + }); + match json_result { + Err(e) => { + vec![serde_json::json!({ + "request": req, + "error": e.to_string() + })] } Ok(json) => json, } diff --git a/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs b/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs index 0a458ed8..3c34fd72 100644 --- a/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs @@ -12,9 +12,9 @@ impl OutputPlugin for EdgeIdListOutputPlugin { &self, output: &serde_json::Value, search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + ) -> Result, PluginError> { match search_result { - Err(_e) => Ok(output.clone()), + Err(_e) => Ok(vec![output.clone()]), Ok(result) => { let edge_ids = result .route @@ -24,7 +24,7 @@ impl OutputPlugin for EdgeIdListOutputPlugin { .collect::>(); let mut updated = output.clone(); updated.add_edge_list(&edge_ids)?; - Ok(updated) + Ok(vec![updated]) } } } diff --git a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs index ff0582a6..f510c090 100644 --- a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs @@ -13,9 +13,9 @@ impl OutputPlugin for SummaryOutputPlugin { &self, output: &serde_json::Value, search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + ) -> Result, PluginError> { match search_result { - Err(_e) => Ok(output.clone()), + Err(_e) => Ok(vec![output.clone()]), Ok(result) => { let mut updated_output = output.clone(); let cost = result @@ -24,7 +24,7 @@ impl OutputPlugin for SummaryOutputPlugin { .map(|traversal| traversal.edge_cost()) .sum::(); updated_output.add_cost(cost)?; - Ok(updated_output) + Ok(vec![updated_output]) } } } @@ -78,7 +78,7 @@ mod tests { let updated_output = summary_plugin .process(&output_result, Ok(&search_result)) .unwrap(); - let cost: f64 = updated_output.get_cost().unwrap().into(); + let cost: f64 = updated_output[0].get_cost().unwrap().into(); assert_eq!(cost, 6.0); } } diff --git a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs index 72697314..dcdb9103 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs @@ -59,9 +59,9 @@ impl OutputPlugin for TraversalPlugin { &self, output: &serde_json::Value, search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + ) -> Result, PluginError> { match search_result { - Err(_) => Ok(output.clone()), + Err(_) => Ok(vec![output.clone()]), Ok(result) => { let mut output_mut = output.clone(); let updated = output_mut @@ -89,7 +89,7 @@ impl OutputPlugin for TraversalPlugin { } } - Ok(serde_json::Value::Object(updated.to_owned())) + Ok(vec![serde_json::Value::Object(updated.to_owned())]) } } } @@ -182,7 +182,7 @@ mod tests { let result = geom_plugin .process(&output_result, Ok(&search_result)) .unwrap(); - let geometry_wkt = result.get_route_geometry_wkt().unwrap(); + let geometry_wkt = result[0].get_route_geometry_wkt().unwrap(); assert_eq!(geometry_wkt, expected_geometry); } } diff --git a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs index f5a406c5..788d9fcf 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs @@ -50,7 +50,7 @@ impl OutputPlugin for UUIDOutputPlugin { &self, output: &serde_json::Value, _search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + ) -> Result, PluginError> { let mut updated_output = output.clone(); let (origin_vertex_id, destination_vertex_id) = output.get_od_vertex_ids()?; let origin_uuid = self @@ -64,6 +64,6 @@ impl OutputPlugin for UUIDOutputPlugin { updated_output.add_od_uuids(origin_uuid.clone(), destination_uuid.clone())?; - Ok(updated_output) + Ok(vec![updated_output]) } } diff --git a/rust/routee-compass/src/plugin/output/output_plugin.rs b/rust/routee-compass/src/plugin/output/output_plugin.rs index e9c318d0..12c82b47 100644 --- a/rust/routee-compass/src/plugin/output/output_plugin.rs +++ b/rust/routee-compass/src/plugin/output/output_plugin.rs @@ -38,5 +38,5 @@ pub trait OutputPlugin: Send + Sync { &self, output: &serde_json::Value, result: Result<&SearchAppResult, SearchError>, - ) -> Result; + ) -> Result, PluginError>; } From 8ab4a276977ce03660a09b57e86c256a9c7b4762 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 12:07:43 -0700 Subject: [PATCH 02/12] start implementing to disk output plugin --- .../src/app/compass/compass_app.rs | 2 +- .../src/plugin/output/default/mod.rs | 1 + .../src/plugin/output/default/to_disk/mod.rs | 1 + .../plugin/output/default/to_disk/plugin.rs | 22 +++++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 rust/routee-compass/src/plugin/output/default/to_disk/mod.rs create mode 100644 rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs diff --git a/rust/routee-compass/src/app/compass/compass_app.rs b/rust/routee-compass/src/app/compass/compass_app.rs index 9b25524e..3d11fb49 100644 --- a/rust/routee-compass/src/app/compass/compass_app.rs +++ b/rust/routee-compass/src/app/compass/compass_app.rs @@ -397,7 +397,7 @@ pub fn apply_output_processing( "traversal_summary": tmodel.serialize_state_with_info(&last_edge_traversal.result_state), }); let init_acc: Result, PluginError> = Ok(vec![init_output]); - let json_result = output_plugins + let json_result = output_plugins .iter() .fold(init_acc, |acc, p| { acc.and_then(|outer| { diff --git a/rust/routee-compass/src/plugin/output/default/mod.rs b/rust/routee-compass/src/plugin/output/default/mod.rs index ef9a3837..9932f09b 100644 --- a/rust/routee-compass/src/plugin/output/default/mod.rs +++ b/rust/routee-compass/src/plugin/output/default/mod.rs @@ -1,4 +1,5 @@ pub mod edgeidlist; pub mod summary; +pub mod to_disk; pub mod traversal; pub mod uuid; diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs b/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs new file mode 100644 index 00000000..962cb1bb --- /dev/null +++ b/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs @@ -0,0 +1 @@ +pub mod plugin; diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs new file mode 100644 index 00000000..cc83750f --- /dev/null +++ b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs @@ -0,0 +1,22 @@ +use std::path::PathBuf; + +use routee_compass_core::algorithm::search::search_error::SearchError; + +use crate::{ + app::search::search_app_result::SearchAppResult, + plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}, +}; + +pub struct ToDiskOutputPlugin { + output_file: PathBuf, +} + +impl OutputPlugin for ToDiskOutputPlugin { + fn process( + &self, + output: &serde_json::Value, + _result: Result<&SearchAppResult, SearchError>, + ) -> Result, PluginError> { + Ok(Vec::new()) + } +} From 5c642686902699ab48b6a77c8e3217b947e8f4a9 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 14:07:17 -0700 Subject: [PATCH 03/12] build inital ouput before passing into output plug --- .github/workflows/python-build-test.yaml | 1 - .github/workflows/rust-build-test.yaml | 1 - .../src/app/compass/compass_app.rs | 69 ++++++++++--------- .../output/default/edgeidlist/plugin.rs | 4 +- .../plugin/output/default/summary/plugin.rs | 6 +- .../plugin/output/default/to_disk/plugin.rs | 4 +- .../plugin/output/default/traversal/plugin.rs | 5 +- .../src/plugin/output/default/uuid/plugin.rs | 3 +- .../src/plugin/output/output_plugin.rs | 3 +- 9 files changed, 50 insertions(+), 46 deletions(-) diff --git a/.github/workflows/python-build-test.yaml b/.github/workflows/python-build-test.yaml index f607d60f..51be4098 100644 --- a/.github/workflows/python-build-test.yaml +++ b/.github/workflows/python-build-test.yaml @@ -7,7 +7,6 @@ on: - "docs/**" - "**.md" pull_request: - branches: [ main ] paths-ignore: - "docs/**" - "**.md" diff --git a/.github/workflows/rust-build-test.yaml b/.github/workflows/rust-build-test.yaml index e89e0ee4..4838165e 100644 --- a/.github/workflows/rust-build-test.yaml +++ b/.github/workflows/rust-build-test.yaml @@ -7,7 +7,6 @@ on: - "docs/**" - "**.md" pull_request: - branches: [ main ] paths-ignore: - "docs/**" - "**.md" diff --git a/rust/routee-compass/src/app/compass/compass_app.rs b/rust/routee-compass/src/app/compass/compass_app.rs index 3d11fb49..8774c65f 100644 --- a/rust/routee-compass/src/app/compass/compass_app.rs +++ b/rust/routee-compass/src/app/compass/compass_app.rs @@ -340,13 +340,14 @@ pub fn apply_output_processing( output_plugins: &Vec>, ) -> Vec { let (req, res) = response_data; - match res { + + let init_output = match &res { Err(e) => { let error_output = serde_json::json!({ "request": req, "error": e.to_string() }); - vec![error_output] + error_output } Ok(result) => { // should be moved into TraversalModel::summary, queries requesting @@ -396,39 +397,41 @@ pub fn apply_output_processing( "tree_edge_count": result.tree.len(), "traversal_summary": tmodel.serialize_state_with_info(&last_edge_traversal.result_state), }); - let init_acc: Result, PluginError> = Ok(vec![init_output]); - let json_result = output_plugins - .iter() - .fold(init_acc, |acc, p| { - acc.and_then(|outer| { - outer - .iter() - .map(|output| p.process(output, Ok(&result))) - .collect::, PluginError>>() - .map(|inner| { - inner - .into_iter() - .flatten() - .collect::>() - }) - }) - }) - .map_err(|e| { - serde_json::json!({ - "request": req, - "error": e.to_string() + init_output + } + }; + + let init_acc: Result, PluginError> = Ok(vec![init_output]); + let json_result = output_plugins + .iter() + .fold(init_acc, |acc, p| { + acc.and_then(|outer| { + outer + .iter() + .map(|output| p.process(output, &res)) + .collect::, PluginError>>() + .map(|inner| { + inner + .into_iter() + .flatten() + .collect::>() }) - }); - match json_result { - Err(e) => { - vec![serde_json::json!({ - "request": req, - "error": e.to_string() - })] - } - Ok(json) => json, - } + }) + }) + .map_err(|e| { + serde_json::json!({ + "request": req, + "error": e.to_string() + }) + }); + match json_result { + Err(e) => { + vec![serde_json::json!({ + "request": req, + "error": e.to_string() + })] } + Ok(json) => json, } } diff --git a/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs b/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs index 3c34fd72..7ddb9a76 100644 --- a/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs @@ -1,8 +1,8 @@ +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::output::default::edgeidlist::json_extensions::EdgeListJsonExtensions; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::algorithm::search::search_error::SearchError; use serde_json; pub struct EdgeIdListOutputPlugin {} @@ -11,7 +11,7 @@ impl OutputPlugin for EdgeIdListOutputPlugin { fn process( &self, output: &serde_json::Value, - search_result: Result<&SearchAppResult, SearchError>, + search_result: &Result, ) -> Result, PluginError> { match search_result { Err(_e) => Ok(vec![output.clone()]), diff --git a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs index f510c090..af784909 100644 --- a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs @@ -1,5 +1,5 @@ use super::json_extensions::SummaryJsonExtensions; -use crate::app::search::search_app_result::SearchAppResult; +use crate::app::{search::search_app_result::SearchAppResult, compass::compass_app_error::CompassAppError}; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; use routee_compass_core::{algorithm::search::search_error::SearchError, model::cost::cost::Cost}; @@ -12,7 +12,7 @@ impl OutputPlugin for SummaryOutputPlugin { fn process( &self, output: &serde_json::Value, - search_result: Result<&SearchAppResult, SearchError>, + search_result: &Result, ) -> Result, PluginError> { match search_result { Err(_e) => Ok(vec![output.clone()]), @@ -76,7 +76,7 @@ mod tests { }; let summary_plugin = SummaryOutputPlugin {}; let updated_output = summary_plugin - .process(&output_result, Ok(&search_result)) + .process(&output_result, &Ok(search_result)) .unwrap(); let cost: f64 = updated_output[0].get_cost().unwrap().into(); assert_eq!(cost, 6.0); diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs index cc83750f..c2b4427f 100644 --- a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use routee_compass_core::algorithm::search::search_error::SearchError; use crate::{ - app::search::search_app_result::SearchAppResult, + app::{search::search_app_result::SearchAppResult, compass::compass_app_error::CompassAppError}, plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}, }; @@ -15,7 +15,7 @@ impl OutputPlugin for ToDiskOutputPlugin { fn process( &self, output: &serde_json::Value, - _result: Result<&SearchAppResult, SearchError>, + _result: &Result, ) -> Result, PluginError> { Ok(Vec::new()) } diff --git a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs index bba68636..9243a503 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs @@ -1,6 +1,7 @@ use super::json_extensions::TraversalJsonField; use super::traversal_output_format::TraversalOutputFormat; use super::utils::parse_linestring; +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; @@ -57,7 +58,7 @@ impl OutputPlugin for TraversalPlugin { fn process( &self, output: &serde_json::Value, - search_result: Result<&SearchAppResult, SearchError>, + search_result: &Result, ) -> Result, PluginError> { match search_result { Err(_) => Ok(vec![output.clone()]), @@ -177,7 +178,7 @@ mod tests { TraversalPlugin::from_file(&filename, Some(TraversalOutputFormat::Wkt), None).unwrap(); let result = geom_plugin - .process(&output_result, Ok(&search_result)) + .process(&output_result, &Ok(search_result)) .unwrap(); let geometry_wkt = result[0].get_route_geometry_wkt().unwrap(); assert_eq!(geometry_wkt, expected_geometry); diff --git a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs index 626e42a8..98022a15 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs @@ -1,4 +1,5 @@ use super::json_extensions::UUIDJsonExtensions; +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}; use kdam::Bar; @@ -47,7 +48,7 @@ impl OutputPlugin for UUIDOutputPlugin { fn process( &self, output: &serde_json::Value, - _search_result: Result<&SearchAppResult, SearchError>, + _search_result: &Result, ) -> Result, PluginError> { let mut updated_output = output.clone(); let (origin_vertex_id, destination_vertex_id) = output.get_od_vertex_ids()?; diff --git a/rust/routee-compass/src/plugin/output/output_plugin.rs b/rust/routee-compass/src/plugin/output/output_plugin.rs index 12c82b47..1a912972 100644 --- a/rust/routee-compass/src/plugin/output/output_plugin.rs +++ b/rust/routee-compass/src/plugin/output/output_plugin.rs @@ -1,3 +1,4 @@ +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::plugin_error::PluginError; use routee_compass_core::algorithm::search::search_error::SearchError; @@ -37,6 +38,6 @@ pub trait OutputPlugin: Send + Sync { fn process( &self, output: &serde_json::Value, - result: Result<&SearchAppResult, SearchError>, + result: &Result, ) -> Result, PluginError>; } From 6b65fc0b6caedab91ba44bff1581cfc33c3cb728 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 15:55:01 -0700 Subject: [PATCH 04/12] finish to disk output plugin --- .../routee/compass/io/generate_dataset.py | 2 +- .../src/app/compass/compass_app.rs | 13 ++---- .../app/compass/config/compass_app_builder.rs | 3 ++ .../compass/config/config_json_extension.rs | 7 +++- .../frontier_model/road_class_builder.rs | 2 +- .../src/app/compass/config/graph_builder.rs | 4 +- .../speed_grade_energy_model_builder.rs | 14 ++++--- .../traversal_model/speed_lookup_builder.rs | 6 ++- rust/routee-compass/src/main.rs | 12 ++++-- .../src/plugin/input/default/rtree/builder.rs | 2 +- .../plugin/output/default/summary/plugin.rs | 4 +- .../plugin/output/default/to_disk/builder.rs | 42 +++++++++++++++++++ .../src/plugin/output/default/to_disk/mod.rs | 1 + .../plugin/output/default/to_disk/plugin.rs | 31 +++++++++++--- .../output/default/traversal/builder.rs | 2 +- .../plugin/output/default/traversal/plugin.rs | 11 +---- .../src/plugin/output/default/uuid/builder.rs | 2 +- .../src/plugin/output/default/uuid/plugin.rs | 10 +---- .../routee-compass/src/plugin/plugin_error.rs | 4 +- 19 files changed, 118 insertions(+), 54 deletions(-) create mode 100644 rust/routee-compass/src/plugin/output/default/to_disk/builder.rs diff --git a/python/nrel/routee/compass/io/generate_dataset.py b/python/nrel/routee/compass/io/generate_dataset.py index fb35fb34..7535f657 100644 --- a/python/nrel/routee/compass/io/generate_dataset.py +++ b/python/nrel/routee/compass/io/generate_dataset.py @@ -176,7 +176,7 @@ def replace_id(vertex_uuid): if filename == "osm_default_energy.toml": if add_grade: init_toml["traversal"][ - "grade_table_file" + "grade_table_input_input_file" ] = "edges-grade-enumerated.txt.gz" init_toml["traversal"]["grade_table_grade_unit"] = "decimal" with open(output_directory / filename, "w") as f: diff --git a/rust/routee-compass/src/app/compass/compass_app.rs b/rust/routee-compass/src/app/compass/compass_app.rs index 8774c65f..fb177bfd 100644 --- a/rust/routee-compass/src/app/compass/compass_app.rs +++ b/rust/routee-compass/src/app/compass/compass_app.rs @@ -26,7 +26,7 @@ use routee_compass_core::{ }; use std::path::{Path, PathBuf}; -pub const CONFIG_FILE_KEY: &str = "config_file"; +pub const CONFIG_FILE_KEY: &str = "config_input_file"; /// Instance of RouteE Compass as an application. /// When constructed, it holds @@ -350,14 +350,6 @@ pub fn apply_output_processing( error_output } Ok(result) => { - // should be moved into TraversalModel::summary, queries requesting - // min spanning tree result will not have an acc_cost. - let mut acc_cost = Cost::ZERO; - for traversal in result.route.clone() { - let cost = traversal.edge_cost(); - acc_cost = acc_cost + cost; - } - log::debug!( "completed route for request {}: {} links, {} tree size", req, @@ -365,7 +357,8 @@ pub fn apply_output_processing( result.tree.len() ); - // should be moved into TraversalModel::summary same reason as above + // should be moved into TraversalModel::summary, queries requesting + // min spanning tree result will not have a route. let route = result.route.to_vec(); let last_edge_traversal = match route.last() { None => { diff --git a/rust/routee-compass/src/app/compass/config/compass_app_builder.rs b/rust/routee-compass/src/app/compass/config/compass_app_builder.rs index 83472436..76c9917c 100644 --- a/rust/routee-compass/src/app/compass/config/compass_app_builder.rs +++ b/rust/routee-compass/src/app/compass/config/compass_app_builder.rs @@ -24,6 +24,7 @@ use crate::plugin::{ default::{ edgeidlist::builder::EdgeIdListOutputPluginBuilder, summary::builder::SummaryOutputPluginBuilder, + to_disk::builder::ToDiskOutputPluginBuilder, traversal::builder::TraversalPluginBuilder, uuid::builder::UUIDOutputPluginBuilder, }, output_plugin::OutputPlugin, @@ -120,11 +121,13 @@ impl CompassAppBuilder { let summary: Box = Box::new(SummaryOutputPluginBuilder {}); let uuid: Box = Box::new(UUIDOutputPluginBuilder {}); let edge_id_list: Box = Box::new(EdgeIdListOutputPluginBuilder {}); + let to_disk: Box = Box::new(ToDiskOutputPluginBuilder {}); let output_plugin_builders = HashMap::from([ (String::from("traversal"), traversal), (String::from("summary"), summary), (String::from("uuid"), uuid), (String::from("edge_id_list"), edge_id_list), + (String::from("to_disk"), to_disk), ]); CompassAppBuilder { diff --git a/rust/routee-compass/src/app/compass/config/config_json_extension.rs b/rust/routee-compass/src/app/compass/config/config_json_extension.rs index 7b10341e..70929bf4 100644 --- a/rust/routee-compass/src/app/compass/config/config_json_extension.rs +++ b/rust/routee-compass/src/app/compass/config/config_json_extension.rs @@ -6,6 +6,8 @@ use std::{ str::FromStr, }; +const FILE_NORMALIZATION_POSTFIX: &str = "_input_file"; + pub trait ConfigJsonExtensions { fn get_config_section( &self, @@ -305,7 +307,10 @@ impl ConfigJsonExtensions for serde_json::Value { serde_json::Value::Object(obj) => { let mut new_obj = serde_json::map::Map::new(); for (key, value) in obj.iter() { - if key.ends_with("_file") || value.is_object() || value.is_array() { + if key.ends_with(FILE_NORMALIZATION_POSTFIX) + || value.is_object() + || value.is_array() + { new_obj.insert(key.clone(), value.normalize_file_paths(root_config_path)?); } else { new_obj.insert(key.clone(), value.clone()); diff --git a/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs b/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs index 4cfe56b3..1acc5d9c 100644 --- a/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs +++ b/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs @@ -17,7 +17,7 @@ impl FrontierModelBuilder for RoadClassBuilder { parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { let frontier_key = CompassConfigurationField::Frontier.to_string(); - let road_class_file_key = String::from("road_class_file"); + let road_class_file_key = String::from("road_class_input_file"); let valid_road_class_key = String::from("valid_road_classes"); let road_class_file = diff --git a/rust/routee-compass/src/app/compass/config/graph_builder.rs b/rust/routee-compass/src/app/compass/config/graph_builder.rs index 1ddc01a3..5efcd848 100644 --- a/rust/routee-compass/src/app/compass/config/graph_builder.rs +++ b/rust/routee-compass/src/app/compass/config/graph_builder.rs @@ -28,9 +28,9 @@ impl DefaultGraphBuilder { pub fn build(params: &serde_json::Value) -> Result { let graph_key = CompassConfigurationField::Graph.to_string(); let edge_list_csv = - params.get_config_path(String::from("edge_list_file"), graph_key.clone())?; + params.get_config_path(String::from("edge_list_input_file"), graph_key.clone())?; let vertex_list_csv = - params.get_config_path(String::from("vertex_list_file"), graph_key.clone())?; + params.get_config_path(String::from("vertex_list_input_file"), graph_key.clone())?; let n_edges = params.get_config_serde_optional(String::from("n_edges"), graph_key.clone())?; let n_vertices = diff --git a/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs b/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs index d12f143f..d84be62b 100644 --- a/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs +++ b/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs @@ -29,15 +29,19 @@ impl TraversalModelBuilder for SpeedGradeEnergyModelBuilder { ) -> Result, CompassConfigurationError> { let traversal_key = CompassConfigurationField::Traversal.to_string(); - let speed_table_path = - params.get_config_path(String::from("speed_table_file"), traversal_key.clone())?; + let speed_table_path = params.get_config_path( + String::from("speed_table_input_file"), + traversal_key.clone(), + )?; let speed_table_speed_unit = params.get_config_serde::( String::from("speed_table_speed_unit"), traversal_key.clone(), )?; - let grade_table_path = params - .get_config_path_optional(String::from("grade_table_file"), traversal_key.clone())?; + let grade_table_path = params.get_config_path_optional( + String::from("grade_table_input_file"), + traversal_key.clone(), + )?; let grade_table_grade_unit = params.get_config_serde_optional::( String::from("graph_grade_unit"), traversal_key.clone(), @@ -52,7 +56,7 @@ impl TraversalModelBuilder for SpeedGradeEnergyModelBuilder { let name = energy_model_config .get_config_string(String::from("name"), traversal_key.clone())?; let model_path = energy_model_config - .get_config_path(String::from("model_file"), traversal_key.clone())?; + .get_config_path(String::from("model_input_file"), traversal_key.clone())?; let model_type = energy_model_config .get_config_serde::(String::from("model_type"), traversal_key.clone())?; let speed_unit = energy_model_config diff --git a/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs b/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs index eaf30d18..cd152a07 100644 --- a/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs +++ b/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs @@ -22,8 +22,10 @@ impl TraversalModelBuilder for SpeedLookupBuilder { ) -> Result, CompassConfigurationError> { let traversal_key = CompassConfigurationField::Traversal.to_string(); // todo: optional output time unit - let filename = - params.get_config_path(String::from("speed_table_file"), traversal_key.clone())?; + let filename = params.get_config_path( + String::from("speed_table_input_file"), + traversal_key.clone(), + )?; let speed_unit = params .get_config_serde::(String::from("speed_unit"), traversal_key.clone())?; let distance_unit = params.get_config_serde_optional::( diff --git a/rust/routee-compass/src/main.rs b/rust/routee-compass/src/main.rs index 899a1d6f..a7147854 100644 --- a/rust/routee-compass/src/main.rs +++ b/rust/routee-compass/src/main.rs @@ -39,10 +39,14 @@ fn main() -> Result<(), Box> { let user_queries = user_json.get_queries()?; info!("Query: {:?}", user_json); - // run searches and return result - let output_rows = compass_app.run(user_queries)?; - let output_contents = serde_json::to_string(&output_rows)?; - std::fs::write("result.json", output_contents)?; + let results = compass_app.run(user_queries)?; + + // scan the results and log any json values that have "error" in them + for result in results.iter() { + if let Some(error) = result.get("error") { + error!("Error: {}", error); + } + } return Ok(()); } diff --git a/rust/routee-compass/src/plugin/input/default/rtree/builder.rs b/rust/routee-compass/src/plugin/input/default/rtree/builder.rs index 491a5c43..3b0bcde4 100644 --- a/rust/routee-compass/src/plugin/input/default/rtree/builder.rs +++ b/rust/routee-compass/src/plugin/input/default/rtree/builder.rs @@ -15,7 +15,7 @@ impl InputPluginBuilder for VertexRTreeBuilder { &self, parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { - let vertex_filename_key = String::from("vertices_file"); + let vertex_filename_key = String::from("vertices_input_file"); let vertex_path = parameters.get_config_path( vertex_filename_key, String::from("Vertex RTree Input Plugin"), diff --git a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs index af784909..5bef81ba 100644 --- a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs @@ -1,5 +1,7 @@ use super::json_extensions::SummaryJsonExtensions; -use crate::app::{search::search_app_result::SearchAppResult, compass::compass_app_error::CompassAppError}; +use crate::app::{ + compass::compass_app_error::CompassAppError, search::search_app_result::SearchAppResult, +}; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; use routee_compass_core::{algorithm::search::search_error::SearchError, model::cost::cost::Cost}; diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs new file mode 100644 index 00000000..566b49f7 --- /dev/null +++ b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs @@ -0,0 +1,42 @@ +use std::path::PathBuf; + +use crate::{ + app::compass::config::{ + builders::OutputPluginBuilder, compass_configuration_error::CompassConfigurationError, + config_json_extension::ConfigJsonExtensions, + }, + plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}, +}; + +use super::plugin::ToDiskOutputPlugin; + +pub struct ToDiskOutputPluginBuilder {} + +impl OutputPluginBuilder for ToDiskOutputPluginBuilder { + fn build( + &self, + parameters: &serde_json::Value, + ) -> Result, CompassConfigurationError> { + let output_filename_key = String::from("output_file"); + let output_filename = + parameters.get_config_string(output_filename_key, String::from("output"))?; + + let output_filepath = PathBuf::from(&output_filename); + + // create empty file + std::fs::write(output_filepath.clone(), "").map_err(|e| { + PluginError::FileReadError( + output_filepath.clone(), + format!( + "Error initializing output file for to_disk output plugin : {}", + e + ), + ) + })?; + + let to_disk_plugin = ToDiskOutputPlugin { + output_file: output_filepath, + }; + Ok(Box::new(to_disk_plugin)) + } +} diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs b/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs index 962cb1bb..dc15aad0 100644 --- a/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs +++ b/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs @@ -1 +1,2 @@ +pub mod builder; pub mod plugin; diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs index c2b4427f..4d3bd7af 100644 --- a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs @@ -1,14 +1,15 @@ -use std::path::PathBuf; - -use routee_compass_core::algorithm::search::search_error::SearchError; +use std::io::prelude::*; +use std::{fs::OpenOptions, path::PathBuf}; use crate::{ - app::{search::search_app_result::SearchAppResult, compass::compass_app_error::CompassAppError}, + app::{ + compass::compass_app_error::CompassAppError, search::search_app_result::SearchAppResult, + }, plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}, }; pub struct ToDiskOutputPlugin { - output_file: PathBuf, + pub output_file: PathBuf, } impl OutputPlugin for ToDiskOutputPlugin { @@ -17,6 +18,26 @@ impl OutputPlugin for ToDiskOutputPlugin { output: &serde_json::Value, _result: &Result, ) -> Result, PluginError> { + let mut file = OpenOptions::new() + .write(true) + .append(true) + .open(&self.output_file) + .map_err(|e| { + PluginError::FileReadError( + self.output_file.clone(), + format!("Could not open output file: {}", e), + ) + })?; + let output_json = serde_json::to_string(output)?; + + writeln!(file, "{}", output_json).map_err(|e| { + PluginError::FileReadError( + self.output_file.clone(), + format!("Could not write to output file: {}", e), + ) + })?; + + // return empty vec since we already wrote the result to a file Ok(Vec::new()) } } diff --git a/rust/routee-compass/src/plugin/output/default/traversal/builder.rs b/rust/routee-compass/src/plugin/output/default/traversal/builder.rs index be82bb4d..7bb99343 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/builder.rs @@ -40,7 +40,7 @@ impl OutputPluginBuilder for TraversalPluginBuilder { parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { let parent_key = String::from("traversal"); - let geometry_filename_key = String::from("geometry_file"); + let geometry_filename_key = String::from("geometry_input_file"); let route_geometry_key = String::from("route"); let tree_geometry_key = String::from("tree"); diff --git a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs index 9243a503..9f9f2c38 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs @@ -8,7 +8,6 @@ use crate::plugin::plugin_error::PluginError; use geo::LineString; use kdam::Bar; use kdam::BarExt; -use routee_compass_core::algorithm::search::search_error::SearchError; use routee_compass_core::util::fs::fs_utils; use routee_compass_core::util::fs::read_utils::read_raw_file; use std::path::Path; @@ -27,10 +26,7 @@ impl TraversalPlugin { ) -> Result { let count = fs_utils::line_count(filename.clone(), fs_utils::is_gzip(&filename)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; let mut pb = Bar::builder() @@ -44,10 +40,7 @@ impl TraversalPlugin { pb.update(1); }); let geoms = read_raw_file(&filename, parse_linestring, Some(cb)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; print!("\n"); Ok(TraversalPlugin { geoms, route, tree }) diff --git a/rust/routee-compass/src/plugin/output/default/uuid/builder.rs b/rust/routee-compass/src/plugin/output/default/uuid/builder.rs index 8281be3c..b1c9cfbc 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/builder.rs @@ -15,7 +15,7 @@ impl OutputPluginBuilder for UUIDOutputPluginBuilder { &self, parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { - let uuid_filename_key = String::from("uuid_file"); + let uuid_filename_key = String::from("uuid_input_file"); let uuid_filename = parameters.get_config_path(uuid_filename_key, String::from("uuid"))?; let uuid_plugin = UUIDOutputPlugin::from_file(&uuid_filename) diff --git a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs index 98022a15..17c68a62 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs @@ -16,10 +16,7 @@ impl UUIDOutputPlugin { pub fn from_file>(filename: &P) -> Result { let count = fs_utils::line_count(filename.clone(), fs_utils::is_gzip(&filename)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; let mut pb = Bar::builder() @@ -34,10 +31,7 @@ impl UUIDOutputPlugin { }); let uuids = read_raw_file(&filename, |_idx, row| Ok(row), Some(cb)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; print!("\n"); Ok(UUIDOutputPlugin { uuids }) diff --git a/rust/routee-compass/src/plugin/plugin_error.rs b/rust/routee-compass/src/plugin/plugin_error.rs index 62a89fd7..d3817a76 100644 --- a/rust/routee-compass/src/plugin/plugin_error.rs +++ b/rust/routee-compass/src/plugin/plugin_error.rs @@ -17,8 +17,8 @@ pub enum PluginError { BuildError, #[error("nearest vertex not found for coord {0:?}")] NearestVertexNotFound(Coord), - #[error("unable to read file {filename} due to {message}")] - FileReadError { filename: PathBuf, message: String }, + #[error("unable to read file {0} due to {1}")] + FileReadError(PathBuf, String), #[error(transparent)] JsonError(#[from] serde_json::Error), #[error(transparent)] From 74b1126fd3fbf0adfecfa95aa654d2d9d0c3def2 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 15:59:35 -0700 Subject: [PATCH 05/12] fix config filename keys --- docs/config.md | 28 +++---- .../resources/osm_default_distance.toml | 10 +-- .../compass/resources/osm_default_energy.toml | 80 +++++++++---------- .../compass/resources/osm_default_speed.toml | 12 +-- rust/routee-compass-powertrain/src/doc.md | 6 +- .../src/app/compass/config/test/conf.toml | 8 +- .../test/energy_test/energy_debug.toml | 8 +- .../compass/test/energy_test/energy_test.toml | 8 +- .../test/speeds_test/speeds_debug.toml | 6 +- .../compass/test/speeds_test/speeds_test.toml | 6 +- .../output/default/traversal/builder.rs | 2 +- 11 files changed, 87 insertions(+), 87 deletions(-) diff --git a/docs/config.md b/docs/config.md index 88612f74..a9b93128 100644 --- a/docs/config.md +++ b/docs/config.md @@ -13,9 +13,9 @@ parallelism = 2 # the parameters for the underlying road network graph [graph] # a file containing all the graph edges and their adjacencies -edge_list_file = "edges-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" # a file containing all the graph verticies -vertex_list_file = "vertices-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" # if verbose is true, you'll see more information when loading the graph verbose = true @@ -24,11 +24,11 @@ verbose = true # the speed_grade_energy_model can compute routes using time or energy as costs type = "speed_grade_energy_model" # speeds for each edge in the graph -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" # the units of the above speed table speed_table_speed_unit = "kilometers_per_hour" # grades for each edge in the graph -grade_table_file = "edges-grade-enumerated.txt.gz" +grade_table_input_file = "edges-grade-enumerated.txt.gz" # the units of the above graph table grade_table_grade_unit = "decimal" # the units of what the traversal model outputs for time @@ -42,7 +42,7 @@ output_distance_unit = "miles" # the name of the model that can be passed in from a query as "model_name" name = "2012_Ford_Focus" # the file for the routee-powertrain model -model_file = "models/2012_Ford_Focus.bin" +model_input_file = "models/2012_Ford_Focus.bin" # what underlying machine learn framework to use [smartcore | onnx] model_type = "smartcore" # the units of what the routee-powertrain model expects speed to be in @@ -69,7 +69,7 @@ type = "grid_search" # a vertex based RTree for matching incoming x/y coordinates to a graph vertex type = "vertex_rtree" # a file with all the graph verticies -vertices_file = "vertices-compass.csv.gz" +vertices_input_file = "vertices-compass.csv.gz" # output plugs get applied to the result of running a query [[plugin.output_plugins]] @@ -84,13 +84,13 @@ route = "geo_json" # return the full search tree in a geojson format tree = "geo_json" # geometry objects for all the edges in the graph -geometry_file = "edges-geometries-enumerated.txt.gz" +geometry_input_file = "edges-geometries-enumerated.txt.gz" [[plugin.output_plugins]] # append an map specific id(like Open Street Maps Nodes) onto the compass verticies (which use a simple integer internal index) type = "uuid" # a file with ids for each vertex in the graph -uuid_file = "vertices-uuid-enumerated.txt.gz" +uuid_input_file = "vertices-uuid-enumerated.txt.gz" ``` ## Traversal Models @@ -116,7 +116,7 @@ The speed table traversal model uses a speed lookup table to compute the fastest ```toml [traversal] type = "speed_table" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" speed_unit = "kilometers_per_hour" output_distance_unit = "miles" output_time_unit = "minutes" @@ -130,10 +130,10 @@ The speed grade energy model computes energy (with a routee-powertrain vehicle m [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "data/tomtom_metro_denver_network/edges-free-flow-speed-enumerated.txt.gz" -grade_table_file = "data/tomtom_metro_denver_network/edges-grade-enumerated.txt.gz" +speed_table_input_file = "data/tomtom_metro_denver_network/edges-free-flow-speed-enumerated.txt.gz" +grade_table_input_file = "data/tomtom_metro_denver_network/edges-grade-enumerated.txt.gz" grade_table_grade_unit = "decimal" -energy_model_file = "data/2016_TOYOTA_Camry_4cyl_2WD.bin" +energy_model_input_file = "data/2016_TOYOTA_Camry_4cyl_2WD.bin" ideal_energy_rate = 0.02857142857 speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" @@ -213,7 +213,7 @@ For example, if you specify your query origin and destination as lat/lon coordin ```toml [[plugin.input_plugins]] type = "vertex_rtree" -vertices_file = "vertices-compass.csv.gz" +vertices_input_file = "vertices-compass.csv.gz" ``` ## Output Plugins @@ -240,7 +240,7 @@ A plugin that appends various items to the result. type = "traversal" route = "geo_json" tree = "geo_json" -geometry_file = "edges-geometries-enumerated.txt.gz" +geometry_input_file = "edges-geometries-enumerated.txt.gz" ``` The `route` key will add route information to the result depending on the type. diff --git a/python/nrel/routee/compass/resources/osm_default_distance.toml b/python/nrel/routee/compass/resources/osm_default_distance.toml index c9d6d012..31e3cf34 100644 --- a/python/nrel/routee/compass/resources/osm_default_distance.toml +++ b/python/nrel/routee/compass/resources/osm_default_distance.toml @@ -1,8 +1,8 @@ parallelism = 2 [graph] -edge_list_file = "edges-compass.csv.gz" -vertex_list_file = "vertices-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" verbose = true [traversal] @@ -12,10 +12,10 @@ distance_unit = "miles" [plugin] input_plugins = [ { type = "grid_search" }, - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "traversal", route = "geo_json", tree = "geo_json", geometry_file = "edges-geometries-enumerated.txt.gz" }, - { type = "uuid", uuid_file = "vertices-uuid-enumerated.txt.gz" }, + { type = "traversal", route = "geo_json", tree = "geo_json", geometry_input_file = "edges-geometries-enumerated.txt.gz" }, + { type = "uuid", uuid_input_file = "vertices-uuid-enumerated.txt.gz" }, ] diff --git a/python/nrel/routee/compass/resources/osm_default_energy.toml b/python/nrel/routee/compass/resources/osm_default_energy.toml index e3869962..0d1926a3 100644 --- a/python/nrel/routee/compass/resources/osm_default_energy.toml +++ b/python/nrel/routee/compass/resources/osm_default_energy.toml @@ -1,31 +1,31 @@ parallelism = 2 [graph] -edge_list_file = "edges-compass.csv.gz" -vertex_list_file = "vertices-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" verbose = true [plugin] input_plugins = [ { type = "grid_search" }, - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "traversal", route = "geo_json", tree = "geo_json", geometry_file = "edges-geometries-enumerated.txt.gz" }, - { type = "uuid", uuid_file = "vertices-uuid-enumerated.txt.gz" }, + { type = "traversal", route = "geo_json", tree = "geo_json", geometry_input_file = "edges-geometries-enumerated.txt.gz" }, + { type = "uuid", uuid_input_file = "vertices-uuid-enumerated.txt.gz" }, ] [traversal] type = "speed_grade_energy_model" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" speed_table_speed_unit = "kilometers_per_hour" output_time_unit = "minutes" output_distance_unit = "miles" [[traversal.energy_models]] name = "2012_Ford_Focus" -model_file = "models/2012_Ford_Focus.bin" +model_input_file = "models/2012_Ford_Focus.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -35,7 +35,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2012_Ford_Fusion" -model_file = "models/2012_Ford_Fusion.bin" +model_input_file = "models/2012_Ford_Fusion.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -45,7 +45,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_AUDI_A3_4cyl_2WD" -model_file = "models/2016_AUDI_A3_4cyl_2WD.bin" +model_input_file = "models/2016_AUDI_A3_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -55,7 +55,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_BMW_328d_4cyl_2WD" -model_file = "models/2016_BMW_328d_4cyl_2WD.bin" +model_input_file = "models/2016_BMW_328d_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -65,7 +65,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_CHEVROLET_Malibu_4cyl_2WD" -model_file = "models/2016_CHEVROLET_Malibu_4cyl_2WD.bin" +model_input_file = "models/2016_CHEVROLET_Malibu_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -75,7 +75,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_CHEVROLET_Spark_EV" -model_file = "models/2016_CHEVROLET_Spark_EV.bin" +model_input_file = "models/2016_CHEVROLET_Spark_EV.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -85,7 +85,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_FORD_C-MAX_HEV" -model_file = "models/2016_FORD_C-MAX_HEV.bin" +model_input_file = "models/2016_FORD_C-MAX_HEV.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -95,7 +95,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2016_FORD_Escape_4cyl_2WD" -model_file = "models/2016_FORD_Escape_4cyl_2WD.bin" +model_input_file = "models/2016_FORD_Escape_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -105,7 +105,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_FORD_Explorer_4cyl_2WD" -model_file = "models/2016_FORD_Explorer_4cyl_2WD.bin" +model_input_file = "models/2016_FORD_Explorer_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -115,7 +115,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_HYUNDAI_Elantra_4cyl_2WD" -model_file = "models/2016_HYUNDAI_Elantra_4cyl_2WD.bin" +model_input_file = "models/2016_HYUNDAI_Elantra_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -125,7 +125,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_KIA_Optima_Hybrid" -model_file = "models/2016_KIA_Optima_Hybrid.bin" +model_input_file = "models/2016_KIA_Optima_Hybrid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -135,7 +135,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2016_Leaf_24_kWh" -model_file = "models/2016_Leaf_24_kWh.bin" +model_input_file = "models/2016_Leaf_24_kWh.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -145,7 +145,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_MITSUBISHI_i-MiEV" -model_file = "models/2016_MITSUBISHI_i-MiEV.bin" +model_input_file = "models/2016_MITSUBISHI_i-MiEV.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -155,7 +155,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_Nissan_Leaf_30_kWh" -model_file = "models/2016_Nissan_Leaf_30_kWh.bin" +model_input_file = "models/2016_Nissan_Leaf_30_kWh.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -165,7 +165,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_TESLA_Model_S60_2WD" -model_file = "models/2016_TESLA_Model_S60_2WD.bin" +model_input_file = "models/2016_TESLA_Model_S60_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -175,7 +175,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_TOYOTA_Camry_4cyl_2WD" -model_file = "models/2016_TOYOTA_Camry_4cyl_2WD.bin" +model_input_file = "models/2016_TOYOTA_Camry_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -185,7 +185,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_TOYOTA_Corolla_4cyl_2WD" -model_file = "models/2016_TOYOTA_Corolla_4cyl_2WD.bin" +model_input_file = "models/2016_TOYOTA_Corolla_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -195,7 +195,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_TOYOTA_Highlander_Hybrid" -model_file = "models/2016_TOYOTA_Highlander_Hybrid.bin" +model_input_file = "models/2016_TOYOTA_Highlander_Hybrid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -205,7 +205,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2016_Toyota_Prius_Two_FWD" -model_file = "models/2016_Toyota_Prius_Two_FWD.bin" +model_input_file = "models/2016_Toyota_Prius_Two_FWD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -215,7 +215,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2017_CHEVROLET_Bolt" -model_file = "models/2017_CHEVROLET_Bolt.bin" +model_input_file = "models/2017_CHEVROLET_Bolt.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -225,7 +225,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2017_Maruti_Dzire_VDI" -model_file = "models/2017_Maruti_Dzire_VDI.bin" +model_input_file = "models/2017_Maruti_Dzire_VDI.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -235,7 +235,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2017_Toyota_Highlander_3.5_L" -model_file = "models/2017_Toyota_Highlander_3.5_L.bin" +model_input_file = "models/2017_Toyota_Highlander_3.5_L.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -245,7 +245,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2020_Chevrolet_Colorado_2WD_Diesel" -model_file = "models/2020_Chevrolet_Colorado_2WD_Diesel.bin" +model_input_file = "models/2020_Chevrolet_Colorado_2WD_Diesel.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -255,7 +255,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2020_VW_Golf_1.5TSI" -model_file = "models/2020_VW_Golf_1.5TSI.bin" +model_input_file = "models/2020_VW_Golf_1.5TSI.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -265,7 +265,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2020_VW_Golf_2.0TDI" -model_file = "models/2020_VW_Golf_2.0TDI.bin" +model_input_file = "models/2020_VW_Golf_2.0TDI.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -275,7 +275,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2021_Fiat_Panda_Mild_Hybrid" -model_file = "models/2021_Fiat_Panda_Mild_Hybrid.bin" +model_input_file = "models/2021_Fiat_Panda_Mild_Hybrid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -285,7 +285,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2021_Peugot_3008" -model_file = "models/2021_Peugot_3008.bin" +model_input_file = "models/2021_Peugot_3008.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -295,7 +295,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2022_Ford_F-150_Lightning_4WD" -model_file = "models/2022_Ford_F-150_Lightning_4WD.bin" +model_input_file = "models/2022_Ford_F-150_Lightning_4WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -305,7 +305,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Renault_Zoe_ZE50_R135" -model_file = "models/2022_Renault_Zoe_ZE50_R135.bin" +model_input_file = "models/2022_Renault_Zoe_ZE50_R135.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -315,7 +315,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Tesla_Model_3_RWD" -model_file = "models/2022_Tesla_Model_3_RWD.bin" +model_input_file = "models/2022_Tesla_Model_3_RWD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -325,7 +325,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Tesla_Model_Y_RWD" -model_file = "models/2022_Tesla_Model_Y_RWD.bin" +model_input_file = "models/2022_Tesla_Model_Y_RWD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -335,7 +335,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Toyota_Yaris_Hybrid_Mid" -model_file = "models/2022_Toyota_Yaris_Hybrid_Mid.bin" +model_input_file = "models/2022_Toyota_Yaris_Hybrid_Mid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -345,7 +345,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2022_Volvo_XC40_Recharge_twin" -model_file = "models/2022_Volvo_XC40_Recharge_twin.bin" +model_input_file = "models/2022_Volvo_XC40_Recharge_twin.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -355,7 +355,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2023_Mitsubishi_Pajero_Sport" -model_file = "models/2023_Mitsubishi_Pajero_Sport.bin" +model_input_file = "models/2023_Mitsubishi_Pajero_Sport.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" diff --git a/python/nrel/routee/compass/resources/osm_default_speed.toml b/python/nrel/routee/compass/resources/osm_default_speed.toml index 8a5279ca..af226e80 100644 --- a/python/nrel/routee/compass/resources/osm_default_speed.toml +++ b/python/nrel/routee/compass/resources/osm_default_speed.toml @@ -1,13 +1,13 @@ parallelism = 2 [graph] -edge_list_file = "edges-compass.csv.gz" -vertex_list_file = "vertices-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" verbose = true [traversal] type = "speed_table" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" speed_unit = "kilometers_per_hour" output_distance_unit = "miles" output_time_unit = "minutes" @@ -15,10 +15,10 @@ output_time_unit = "minutes" [plugin] input_plugins = [ { type = "grid_search" }, - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "traversal", route = "geo_json", tree = "geo_json", geometry_file = "edges-geometries-enumerated.txt.gz" }, - { type = "uuid", uuid_file = "vertices-uuid-enumerated.txt.gz" }, + { type = "traversal", route = "geo_json", tree = "geo_json", geometry_input_file = "edges-geometries-enumerated.txt.gz" }, + { type = "uuid", uuid_input_file = "vertices-uuid-enumerated.txt.gz" }, ] diff --git a/rust/routee-compass-powertrain/src/doc.md b/rust/routee-compass-powertrain/src/doc.md index 5f87f34f..0ea5e76c 100644 --- a/rust/routee-compass-powertrain/src/doc.md +++ b/rust/routee-compass-powertrain/src/doc.md @@ -27,8 +27,8 @@ An example traversal model configuration that uses this crate may look like this [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" -energy_model_file = "2016_TOYOTA_Camry_4cyl_2WD.bin" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" +energy_model_input_file = "2016_TOYOTA_Camry_4cyl_2WD.bin" ideal_energy_rate = 0.02857142857 speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" @@ -36,7 +36,7 @@ energy_model_grade_unit = "decimal" energy_model_energy_rate_unit = "gallons_gasoline_per_mile" output_time_unit = "minutes" output_distance_unit = "miles" -grade_table_file = "edges-grade-enumerated.txt.gz" +grade_table_input_file = "edges-grade-enumerated.txt.gz" grade_table_grade_unit = "decimal" ``` diff --git a/rust/routee-compass/src/app/compass/config/test/conf.toml b/rust/routee-compass/src/app/compass/config/test/conf.toml index 3dfbe4f3..4bc9c2df 100644 --- a/rust/routee-compass/src/app/compass/config/test/conf.toml +++ b/rust/routee-compass/src/app/compass/config/test/conf.toml @@ -1,6 +1,6 @@ [graph] -edge_file = "edges-compass.csv.gz" -vertex_file = "vertices-compass.csv.gz" +edge_input_file = "edges-compass.csv.gz" +vertex_input_file = "vertices-compass.csv.gz" n_edges = 125309338 n_vertices = 56306871 verbose = true @@ -15,9 +15,9 @@ type = "distance" [plugin] input_plugins = [ - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "geometry", edge_file = "edges-geometries-enumerated.txt.gz" }, + { type = "geometry", edge_input_file = "edges-geometries-enumerated.txt.gz" }, ] diff --git a/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml b/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml index 8739ff66..82a8fe73 100644 --- a/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml +++ b/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "routee-compass/src/app/compass/test/energy_test/test_edges.csv" -vertex_list_file = "routee-compass/src/app/compass/test/energy_test/test_vertices.csv" +edge_list_input_file = "routee-compass/src/app/compass/test/energy_test/test_edges.csv" +vertex_list_input_file = "routee-compass/src/app/compass/test/energy_test/test_vertices.csv" verbose = true [algorithm] @@ -12,8 +12,8 @@ bidirectional = false [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "routee-compass/src/app/compass/test/energy_test/test_edge_speeds.csv" -energy_model_file = "../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" +speed_table_input_file = "routee-compass/src/app/compass/test/energy_test/test_edge_speeds.csv" +energy_model_input_file = "../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" energy_model_energy_rate_unit = "gallons_gasoline_per_mile" diff --git a/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml b/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml index 2baf677b..f1d305d6 100644 --- a/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml +++ b/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "src/app/compass/test/energy_test/test_edges.csv" -vertex_list_file = "src/app/compass/test/energy_test/test_vertices.csv" +edge_list_input_file = "src/app/compass/test/energy_test/test_edges.csv" +vertex_list_input_file = "src/app/compass/test/energy_test/test_vertices.csv" verbose = true [algorithm] @@ -12,8 +12,8 @@ bidirectional = false [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "src/app/compass/test/energy_test/test_edge_speeds.csv" -energy_model_file = "../../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" +speed_table_input_file = "src/app/compass/test/energy_test/test_edge_speeds.csv" +energy_model_input_file = "../../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" energy_model_energy_rate_unit = "gallons_gasoline_per_mile" diff --git a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml index 7060ba87..e152f156 100644 --- a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml +++ b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "compass-app/src/app/compass/test/speeds_test/test_edges.csv" -vertex_list_file = "compass-app/src/app/compass/test/speeds_test/test_vertices.csv" +edge_list_input_file = "compass-app/src/app/compass/test/speeds_test/test_edges.csv" +vertex_list_input_file = "compass-app/src/app/compass/test/speeds_test/test_vertices.csv" verbose = true [algorithm] @@ -11,7 +11,7 @@ bidirectional = false [traversal] type = "speed_table" -speed_table_file = "compass-app/src/app/compass/test/speeds_test/test_edge_speeds.csv" +speed_table_input_file = "compass-app/src/app/compass/test/speeds_test/test_edge_speeds.csv" speed_unit = "kilometers_per_hour" output_time_unit = "hours" diff --git a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml index 2487103b..dc0963c8 100644 --- a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml +++ b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "src/app/compass/test/speeds_test/test_edges.csv" -vertex_list_file = "src/app/compass/test/speeds_test/test_vertices.csv" +edge_list_input_file = "src/app/compass/test/speeds_test/test_edges.csv" +vertex_list_input_file = "src/app/compass/test/speeds_test/test_vertices.csv" verbose = true [algorithm] @@ -11,7 +11,7 @@ bidirectional = false [traversal] type = "speed_table" -speed_table_file = "src/app/compass/test/speeds_test/test_edge_speeds.csv" +speed_table_input_file = "src/app/compass/test/speeds_test/test_edge_speeds.csv" speed_unit = "kilometers_per_hour" output_time_unit = "hours" diff --git a/rust/routee-compass/src/plugin/output/default/traversal/builder.rs b/rust/routee-compass/src/plugin/output/default/traversal/builder.rs index 7bb99343..734291d2 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/builder.rs @@ -27,7 +27,7 @@ use crate::{ /// type = "traversal" /// route = "geo_json" /// tree = "geo_json" -/// geometry_file = "edges-geometries-enumerated.txt.gz" +/// geometry_input_file = "edges-geometries-enumerated.txt.gz" /// ``` /// pub struct TraversalPluginBuilder {} From 1c2628e9eaa5fca948550b54d73b5cf534e7bb7a Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 15:59:52 -0700 Subject: [PATCH 06/12] apply cargo fix --- rust/routee-compass-core/src/util/fs/read_utils.rs | 2 +- rust/routee-compass/src/app/compass/compass_app.rs | 2 +- rust/routee-compass/src/plugin/output/default/summary/plugin.rs | 2 +- rust/routee-compass/src/plugin/output/default/uuid/plugin.rs | 2 +- rust/routee-compass/src/plugin/output/output_plugin.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/routee-compass-core/src/util/fs/read_utils.rs b/rust/routee-compass-core/src/util/fs/read_utils.rs index aa41c05b..b8845c60 100644 --- a/rust/routee-compass-core/src/util/fs/read_utils.rs +++ b/rust/routee-compass-core/src/util/fs/read_utils.rs @@ -1,7 +1,7 @@ use super::fs_utils; use csv::ReaderBuilder; use flate2::read::GzDecoder; -use std::io::ErrorKind; + use std::{ fs::File, io::{self, BufRead, BufReader}, diff --git a/rust/routee-compass/src/app/compass/compass_app.rs b/rust/routee-compass/src/app/compass/compass_app.rs index fb177bfd..67566f39 100644 --- a/rust/routee-compass/src/app/compass/compass_app.rs +++ b/rust/routee-compass/src/app/compass/compass_app.rs @@ -21,7 +21,7 @@ use config::Config; use itertools::{Either, Itertools}; use rayon::{current_num_threads, prelude::*}; use routee_compass_core::{ - algorithm::search::search_algorithm::SearchAlgorithm, model::cost::cost::Cost, + algorithm::search::search_algorithm::SearchAlgorithm, util::duration_extension::DurationExtension, }; use std::path::{Path, PathBuf}; diff --git a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs index 5bef81ba..5f6a807c 100644 --- a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs @@ -4,7 +4,7 @@ use crate::app::{ }; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::{algorithm::search::search_error::SearchError, model::cost::cost::Cost}; +use routee_compass_core::{model::cost::cost::Cost}; use serde_json; pub struct SummaryOutputPlugin {} diff --git a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs index 17c68a62..de63f45e 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs @@ -4,7 +4,7 @@ use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}; use kdam::Bar; use kdam::BarExt; -use routee_compass_core::algorithm::search::search_error::SearchError; + use routee_compass_core::util::fs::{fs_utils, read_utils::read_raw_file}; use std::path::Path; diff --git a/rust/routee-compass/src/plugin/output/output_plugin.rs b/rust/routee-compass/src/plugin/output/output_plugin.rs index 1a912972..3aa8a417 100644 --- a/rust/routee-compass/src/plugin/output/output_plugin.rs +++ b/rust/routee-compass/src/plugin/output/output_plugin.rs @@ -1,7 +1,7 @@ use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::algorithm::search::search_error::SearchError; + /// Performs some kind of post-processing on a search result. The result JSON is available /// to the plugin as a reference which was potentially modified upstream by another output From c73ccdeb6a3c0862ecec25cf751fbc29c11630b2 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 16:00:14 -0700 Subject: [PATCH 07/12] cargo fmt --- rust/routee-compass/src/plugin/output/default/summary/plugin.rs | 2 +- rust/routee-compass/src/plugin/output/output_plugin.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs index 5f6a807c..da3a3e52 100644 --- a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs @@ -4,7 +4,7 @@ use crate::app::{ }; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::{model::cost::cost::Cost}; +use routee_compass_core::model::cost::cost::Cost; use serde_json; pub struct SummaryOutputPlugin {} diff --git a/rust/routee-compass/src/plugin/output/output_plugin.rs b/rust/routee-compass/src/plugin/output/output_plugin.rs index 3aa8a417..76347150 100644 --- a/rust/routee-compass/src/plugin/output/output_plugin.rs +++ b/rust/routee-compass/src/plugin/output/output_plugin.rs @@ -2,7 +2,6 @@ use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::plugin_error::PluginError; - /// Performs some kind of post-processing on a search result. The result JSON is available /// to the plugin as a reference which was potentially modified upstream by another output /// plugin. The plugin will output a modified version of the JSON as a result. From 113026c744adec8d3fbb9ec09219646dfd9a87cd Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 16:04:21 -0700 Subject: [PATCH 08/12] update input file error message --- .../src/app/compass/config/compass_configuration_error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs b/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs index 0f46547d..02e367af 100644 --- a/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs +++ b/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs @@ -28,14 +28,14 @@ pub enum CompassConfigurationError { Second, make sure the file exists. - Third, make sure the config key ends with '_file' which is a schema requirement + Third, make sure the config key ends with '_input_file' which is a schema requirement for the CompassApp config. "# )] FileNotFoundForComponent(String, String, String), #[error("could not normalize incoming file {0}")] FileNormalizationError(String), - #[error("Could not find incoming configuration file, tried {0} and {1}. Make sure the file exists and that the config key ends with '_file'")] + #[error("Could not find incoming configuration file, tried {0} and {1}. Make sure the file exists and that the config key ends with '_input_file'")] FileNormalizationNotFound(String, String), #[error("{0}")] InsertError(String), From 7e3e6bac3f5a8501b0275ae7d364b7d8d38ffdd8 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 16:08:52 -0700 Subject: [PATCH 09/12] put main branch back in github actions --- .github/workflows/python-build-test.yaml | 1 + .github/workflows/rust-build-test.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/python-build-test.yaml b/.github/workflows/python-build-test.yaml index 51be4098..f607d60f 100644 --- a/.github/workflows/python-build-test.yaml +++ b/.github/workflows/python-build-test.yaml @@ -7,6 +7,7 @@ on: - "docs/**" - "**.md" pull_request: + branches: [ main ] paths-ignore: - "docs/**" - "**.md" diff --git a/.github/workflows/rust-build-test.yaml b/.github/workflows/rust-build-test.yaml index 4838165e..e89e0ee4 100644 --- a/.github/workflows/rust-build-test.yaml +++ b/.github/workflows/rust-build-test.yaml @@ -7,6 +7,7 @@ on: - "docs/**" - "**.md" pull_request: + branches: [ main ] paths-ignore: - "docs/**" - "**.md" From 0a4822d873c1eea09bb3f434fa5b3acad82bd721 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Wed, 8 Nov 2023 16:12:01 -0700 Subject: [PATCH 10/12] black formatting --- python/nrel/routee/compass/compass_app.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/nrel/routee/compass/compass_app.py b/python/nrel/routee/compass/compass_app.py index df96e9d2..ae324708 100644 --- a/python/nrel/routee/compass/compass_app.py +++ b/python/nrel/routee/compass/compass_app.py @@ -11,6 +11,7 @@ Query = Dict[str, Any] Result = List[Dict[str, Any]] + class CompassApp: """ The CompassApp holds everything needed to run a route query. @@ -42,9 +43,7 @@ def from_config_file(cls, config_file: Union[str, Path]) -> CompassApp: app = CompassAppWrapper._from_config_file(str(config_path.absolute())) return cls(app) - def run( - self, query: Union[Query, List[Query]] - ) -> Result: + def run(self, query: Union[Query, List[Query]]) -> Result: """ Run a query (or multiple queries) against the CompassApp @@ -88,5 +87,3 @@ def run( if single_query and len(results) == 1: return results[0] return results - - From 40a8a3983ccd0e26dbde18573e6a49786f83015d Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Thu, 9 Nov 2023 09:03:13 -0700 Subject: [PATCH 11/12] wrap file in mutex --- .../plugin/output/default/to_disk/builder.rs | 40 +++++++++++++------ .../plugin/output/default/to_disk/plugin.rs | 27 +++++++------ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs index 566b49f7..8be97228 100644 --- a/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs @@ -1,4 +1,10 @@ -use std::path::PathBuf; +use std::{ + fs::OpenOptions, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use std::io::prelude::*; use crate::{ app::compass::config::{ @@ -21,21 +27,29 @@ impl OutputPluginBuilder for ToDiskOutputPluginBuilder { let output_filename = parameters.get_config_string(output_filename_key, String::from("output"))?; - let output_filepath = PathBuf::from(&output_filename); + let output_file_path = PathBuf::from(&output_filename); + + // initialize the file with nothing + std::fs::write(&output_file_path, "")?; + + // open the file with the option to append to it + let file = OpenOptions::new() + .write(true) + .append(true) + .open(&output_file_path) + .map_err(|e| { + PluginError::FileReadError( + output_file_path.clone(), + format!("Could not open output file: {}", e), + ) + })?; - // create empty file - std::fs::write(output_filepath.clone(), "").map_err(|e| { - PluginError::FileReadError( - output_filepath.clone(), - format!( - "Error initializing output file for to_disk output plugin : {}", - e - ), - ) - })?; + // wrap the file in a mutex so we can share it between threads + let output_file = Arc::new(Mutex::new(file)); let to_disk_plugin = ToDiskOutputPlugin { - output_file: output_filepath, + output_file_path, + output_file, }; Ok(Box::new(to_disk_plugin)) } diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs index 4d3bd7af..3ffda3e4 100644 --- a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs @@ -1,5 +1,7 @@ +use std::fs::File; use std::io::prelude::*; -use std::{fs::OpenOptions, path::PathBuf}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; use crate::{ app::{ @@ -9,7 +11,8 @@ use crate::{ }; pub struct ToDiskOutputPlugin { - pub output_file: PathBuf, + pub output_file_path: PathBuf, + pub output_file: Arc>, } impl OutputPlugin for ToDiskOutputPlugin { @@ -18,21 +21,19 @@ impl OutputPlugin for ToDiskOutputPlugin { output: &serde_json::Value, _result: &Result, ) -> Result, PluginError> { - let mut file = OpenOptions::new() - .write(true) - .append(true) - .open(&self.output_file) - .map_err(|e| { - PluginError::FileReadError( - self.output_file.clone(), - format!("Could not open output file: {}", e), - ) - })?; + let file_ref = Arc::clone(&self.output_file); + let mut file = file_ref.lock().map_err(|e| { + PluginError::FileReadError( + self.output_file_path.clone(), + format!("Could not aquire lock on output file: {}", e), + ) + })?; + let output_json = serde_json::to_string(output)?; writeln!(file, "{}", output_json).map_err(|e| { PluginError::FileReadError( - self.output_file.clone(), + self.output_file_path.clone(), format!("Could not write to output file: {}", e), ) })?; From 7b3a0175cfa772e9acb01aa9b509e34dd71a15a4 Mon Sep 17 00:00:00 2001 From: Nicholas Reinicke Date: Thu, 9 Nov 2023 09:44:27 -0700 Subject: [PATCH 12/12] include to_disk plugin in docs --- docs/config.md | 21 +++++++++++++------ .../plugin/output/default/to_disk/builder.rs | 2 -- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/config.md b/docs/config.md index a9b93128..67d1d09f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -195,8 +195,6 @@ The grid search plugin would take this single query and generate two queries tha ] ``` -#### Config - ```toml [[plugin.input_plugins]] type = "grid_search" @@ -208,8 +206,6 @@ The vertex RTree plugin uses an RTree to match coordiantes to graph verticies. For example, if you specify your query origin and destination as lat/lon coordinates (i.e. `origin_x`, `origin_y`) we need a way to match this to the graph and then insert an `origin_vertex` or a `destination_vertex` into the query. Those two fields are what the application expects when conducting a search. -#### Config - ```toml [[plugin.input_plugins]] type = "vertex_rtree" @@ -233,8 +229,6 @@ type = "summary" A plugin that appends various items to the result. -#### Config - ```toml [[plugin.output_plugins]] type = "traversal" @@ -251,3 +245,18 @@ Both the `route` and the `tree` key are optional and if omitted, the plugin will - "json": non-geometry output writing traversal metrics (cost, state) as JSON for a route or a tree - "wkt": outputs a LINESTRING for a route, or a MULTILINESTRING for a tree - "geo_json": annotated geometry data as a FeatureCollection of LineStrings with properties assigned from traversal metrics + +### To Disk + +The `to_disk` plugin writes the results to a specified output file rather than returning them when the `run` method is called. + +This plugin writes the results in newline delimited JSON. + +```toml +[[plugin.output_plugins]] +type = "to_disk" + +# where to write the results +# relative to where the application is being run +output_file = "result.json" +``` diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs index 8be97228..f51ce897 100644 --- a/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs @@ -4,8 +4,6 @@ use std::{ sync::{Arc, Mutex}, }; -use std::io::prelude::*; - use crate::{ app::compass::config::{ builders::OutputPluginBuilder, compass_configuration_error::CompassConfigurationError,