diff --git a/docs/config.md b/docs/config.md index 88612f74..67d1d09f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -13,9 +13,9 @@ parallelism = 2 # the parameters for the underlying road network graph [graph] # a file containing all the graph edges and their adjacencies -edge_list_file = "edges-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" # a file containing all the graph verticies -vertex_list_file = "vertices-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" # if verbose is true, you'll see more information when loading the graph verbose = true @@ -24,11 +24,11 @@ verbose = true # the speed_grade_energy_model can compute routes using time or energy as costs type = "speed_grade_energy_model" # speeds for each edge in the graph -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" # the units of the above speed table speed_table_speed_unit = "kilometers_per_hour" # grades for each edge in the graph -grade_table_file = "edges-grade-enumerated.txt.gz" +grade_table_input_file = "edges-grade-enumerated.txt.gz" # the units of the above graph table grade_table_grade_unit = "decimal" # the units of what the traversal model outputs for time @@ -42,7 +42,7 @@ output_distance_unit = "miles" # the name of the model that can be passed in from a query as "model_name" name = "2012_Ford_Focus" # the file for the routee-powertrain model -model_file = "models/2012_Ford_Focus.bin" +model_input_file = "models/2012_Ford_Focus.bin" # what underlying machine learn framework to use [smartcore | onnx] model_type = "smartcore" # the units of what the routee-powertrain model expects speed to be in @@ -69,7 +69,7 @@ type = "grid_search" # a vertex based RTree for matching incoming x/y coordinates to a graph vertex type = "vertex_rtree" # a file with all the graph verticies -vertices_file = "vertices-compass.csv.gz" +vertices_input_file = "vertices-compass.csv.gz" # output plugs get applied to the result of running a query [[plugin.output_plugins]] @@ -84,13 +84,13 @@ route = "geo_json" # return the full search tree in a geojson format tree = "geo_json" # geometry objects for all the edges in the graph -geometry_file = "edges-geometries-enumerated.txt.gz" +geometry_input_file = "edges-geometries-enumerated.txt.gz" [[plugin.output_plugins]] # append an map specific id(like Open Street Maps Nodes) onto the compass verticies (which use a simple integer internal index) type = "uuid" # a file with ids for each vertex in the graph -uuid_file = "vertices-uuid-enumerated.txt.gz" +uuid_input_file = "vertices-uuid-enumerated.txt.gz" ``` ## Traversal Models @@ -116,7 +116,7 @@ The speed table traversal model uses a speed lookup table to compute the fastest ```toml [traversal] type = "speed_table" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" speed_unit = "kilometers_per_hour" output_distance_unit = "miles" output_time_unit = "minutes" @@ -130,10 +130,10 @@ The speed grade energy model computes energy (with a routee-powertrain vehicle m [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "data/tomtom_metro_denver_network/edges-free-flow-speed-enumerated.txt.gz" -grade_table_file = "data/tomtom_metro_denver_network/edges-grade-enumerated.txt.gz" +speed_table_input_file = "data/tomtom_metro_denver_network/edges-free-flow-speed-enumerated.txt.gz" +grade_table_input_file = "data/tomtom_metro_denver_network/edges-grade-enumerated.txt.gz" grade_table_grade_unit = "decimal" -energy_model_file = "data/2016_TOYOTA_Camry_4cyl_2WD.bin" +energy_model_input_file = "data/2016_TOYOTA_Camry_4cyl_2WD.bin" ideal_energy_rate = 0.02857142857 speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" @@ -195,8 +195,6 @@ The grid search plugin would take this single query and generate two queries tha ] ``` -#### Config - ```toml [[plugin.input_plugins]] type = "grid_search" @@ -208,12 +206,10 @@ The vertex RTree plugin uses an RTree to match coordiantes to graph verticies. For example, if you specify your query origin and destination as lat/lon coordinates (i.e. `origin_x`, `origin_y`) we need a way to match this to the graph and then insert an `origin_vertex` or a `destination_vertex` into the query. Those two fields are what the application expects when conducting a search. -#### Config - ```toml [[plugin.input_plugins]] type = "vertex_rtree" -vertices_file = "vertices-compass.csv.gz" +vertices_input_file = "vertices-compass.csv.gz" ``` ## Output Plugins @@ -233,14 +229,12 @@ type = "summary" A plugin that appends various items to the result. -#### Config - ```toml [[plugin.output_plugins]] type = "traversal" route = "geo_json" tree = "geo_json" -geometry_file = "edges-geometries-enumerated.txt.gz" +geometry_input_file = "edges-geometries-enumerated.txt.gz" ``` The `route` key will add route information to the result depending on the type. @@ -251,3 +245,18 @@ Both the `route` and the `tree` key are optional and if omitted, the plugin will - "json": non-geometry output writing traversal metrics (cost, state) as JSON for a route or a tree - "wkt": outputs a LINESTRING for a route, or a MULTILINESTRING for a tree - "geo_json": annotated geometry data as a FeatureCollection of LineStrings with properties assigned from traversal metrics + +### To Disk + +The `to_disk` plugin writes the results to a specified output file rather than returning them when the `run` method is called. + +This plugin writes the results in newline delimited JSON. + +```toml +[[plugin.output_plugins]] +type = "to_disk" + +# where to write the results +# relative to where the application is being run +output_file = "result.json" +``` diff --git a/python/nrel/routee/compass/compass_app.py b/python/nrel/routee/compass/compass_app.py index 4a3f546f..ae324708 100644 --- a/python/nrel/routee/compass/compass_app.py +++ b/python/nrel/routee/compass/compass_app.py @@ -3,11 +3,14 @@ import json from pathlib import Path -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union from nrel.routee.compass.routee_compass_py import ( CompassAppWrapper, ) +Query = Dict[str, Any] +Result = List[Dict[str, Any]] + class CompassApp: """ @@ -40,9 +43,7 @@ def from_config_file(cls, config_file: Union[str, Path]) -> CompassApp: app = CompassAppWrapper._from_config_file(str(config_path.absolute())) return cls(app) - def run( - self, query: Union[Dict[str, Any], List[Dict[str, Any]]] - ) -> List[Dict[str, Any]]: + def run(self, query: Union[Query, List[Query]]) -> Result: """ Run a query (or multiple queries) against the CompassApp diff --git a/python/nrel/routee/compass/io/generate_dataset.py b/python/nrel/routee/compass/io/generate_dataset.py index fb35fb34..7535f657 100644 --- a/python/nrel/routee/compass/io/generate_dataset.py +++ b/python/nrel/routee/compass/io/generate_dataset.py @@ -176,7 +176,7 @@ def replace_id(vertex_uuid): if filename == "osm_default_energy.toml": if add_grade: init_toml["traversal"][ - "grade_table_file" + "grade_table_input_input_file" ] = "edges-grade-enumerated.txt.gz" init_toml["traversal"]["grade_table_grade_unit"] = "decimal" with open(output_directory / filename, "w") as f: diff --git a/python/nrel/routee/compass/resources/osm_default_distance.toml b/python/nrel/routee/compass/resources/osm_default_distance.toml index c9d6d012..31e3cf34 100644 --- a/python/nrel/routee/compass/resources/osm_default_distance.toml +++ b/python/nrel/routee/compass/resources/osm_default_distance.toml @@ -1,8 +1,8 @@ parallelism = 2 [graph] -edge_list_file = "edges-compass.csv.gz" -vertex_list_file = "vertices-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" verbose = true [traversal] @@ -12,10 +12,10 @@ distance_unit = "miles" [plugin] input_plugins = [ { type = "grid_search" }, - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "traversal", route = "geo_json", tree = "geo_json", geometry_file = "edges-geometries-enumerated.txt.gz" }, - { type = "uuid", uuid_file = "vertices-uuid-enumerated.txt.gz" }, + { type = "traversal", route = "geo_json", tree = "geo_json", geometry_input_file = "edges-geometries-enumerated.txt.gz" }, + { type = "uuid", uuid_input_file = "vertices-uuid-enumerated.txt.gz" }, ] diff --git a/python/nrel/routee/compass/resources/osm_default_energy.toml b/python/nrel/routee/compass/resources/osm_default_energy.toml index e3869962..0d1926a3 100644 --- a/python/nrel/routee/compass/resources/osm_default_energy.toml +++ b/python/nrel/routee/compass/resources/osm_default_energy.toml @@ -1,31 +1,31 @@ parallelism = 2 [graph] -edge_list_file = "edges-compass.csv.gz" -vertex_list_file = "vertices-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" verbose = true [plugin] input_plugins = [ { type = "grid_search" }, - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "traversal", route = "geo_json", tree = "geo_json", geometry_file = "edges-geometries-enumerated.txt.gz" }, - { type = "uuid", uuid_file = "vertices-uuid-enumerated.txt.gz" }, + { type = "traversal", route = "geo_json", tree = "geo_json", geometry_input_file = "edges-geometries-enumerated.txt.gz" }, + { type = "uuid", uuid_input_file = "vertices-uuid-enumerated.txt.gz" }, ] [traversal] type = "speed_grade_energy_model" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" speed_table_speed_unit = "kilometers_per_hour" output_time_unit = "minutes" output_distance_unit = "miles" [[traversal.energy_models]] name = "2012_Ford_Focus" -model_file = "models/2012_Ford_Focus.bin" +model_input_file = "models/2012_Ford_Focus.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -35,7 +35,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2012_Ford_Fusion" -model_file = "models/2012_Ford_Fusion.bin" +model_input_file = "models/2012_Ford_Fusion.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -45,7 +45,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_AUDI_A3_4cyl_2WD" -model_file = "models/2016_AUDI_A3_4cyl_2WD.bin" +model_input_file = "models/2016_AUDI_A3_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -55,7 +55,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_BMW_328d_4cyl_2WD" -model_file = "models/2016_BMW_328d_4cyl_2WD.bin" +model_input_file = "models/2016_BMW_328d_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -65,7 +65,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_CHEVROLET_Malibu_4cyl_2WD" -model_file = "models/2016_CHEVROLET_Malibu_4cyl_2WD.bin" +model_input_file = "models/2016_CHEVROLET_Malibu_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -75,7 +75,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_CHEVROLET_Spark_EV" -model_file = "models/2016_CHEVROLET_Spark_EV.bin" +model_input_file = "models/2016_CHEVROLET_Spark_EV.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -85,7 +85,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_FORD_C-MAX_HEV" -model_file = "models/2016_FORD_C-MAX_HEV.bin" +model_input_file = "models/2016_FORD_C-MAX_HEV.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -95,7 +95,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2016_FORD_Escape_4cyl_2WD" -model_file = "models/2016_FORD_Escape_4cyl_2WD.bin" +model_input_file = "models/2016_FORD_Escape_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -105,7 +105,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_FORD_Explorer_4cyl_2WD" -model_file = "models/2016_FORD_Explorer_4cyl_2WD.bin" +model_input_file = "models/2016_FORD_Explorer_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -115,7 +115,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_HYUNDAI_Elantra_4cyl_2WD" -model_file = "models/2016_HYUNDAI_Elantra_4cyl_2WD.bin" +model_input_file = "models/2016_HYUNDAI_Elantra_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -125,7 +125,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_KIA_Optima_Hybrid" -model_file = "models/2016_KIA_Optima_Hybrid.bin" +model_input_file = "models/2016_KIA_Optima_Hybrid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -135,7 +135,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2016_Leaf_24_kWh" -model_file = "models/2016_Leaf_24_kWh.bin" +model_input_file = "models/2016_Leaf_24_kWh.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -145,7 +145,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_MITSUBISHI_i-MiEV" -model_file = "models/2016_MITSUBISHI_i-MiEV.bin" +model_input_file = "models/2016_MITSUBISHI_i-MiEV.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -155,7 +155,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_Nissan_Leaf_30_kWh" -model_file = "models/2016_Nissan_Leaf_30_kWh.bin" +model_input_file = "models/2016_Nissan_Leaf_30_kWh.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -165,7 +165,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_TESLA_Model_S60_2WD" -model_file = "models/2016_TESLA_Model_S60_2WD.bin" +model_input_file = "models/2016_TESLA_Model_S60_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -175,7 +175,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2016_TOYOTA_Camry_4cyl_2WD" -model_file = "models/2016_TOYOTA_Camry_4cyl_2WD.bin" +model_input_file = "models/2016_TOYOTA_Camry_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -185,7 +185,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_TOYOTA_Corolla_4cyl_2WD" -model_file = "models/2016_TOYOTA_Corolla_4cyl_2WD.bin" +model_input_file = "models/2016_TOYOTA_Corolla_4cyl_2WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -195,7 +195,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2016_TOYOTA_Highlander_Hybrid" -model_file = "models/2016_TOYOTA_Highlander_Hybrid.bin" +model_input_file = "models/2016_TOYOTA_Highlander_Hybrid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -205,7 +205,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2016_Toyota_Prius_Two_FWD" -model_file = "models/2016_Toyota_Prius_Two_FWD.bin" +model_input_file = "models/2016_Toyota_Prius_Two_FWD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -215,7 +215,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2017_CHEVROLET_Bolt" -model_file = "models/2017_CHEVROLET_Bolt.bin" +model_input_file = "models/2017_CHEVROLET_Bolt.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -225,7 +225,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2017_Maruti_Dzire_VDI" -model_file = "models/2017_Maruti_Dzire_VDI.bin" +model_input_file = "models/2017_Maruti_Dzire_VDI.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -235,7 +235,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2017_Toyota_Highlander_3.5_L" -model_file = "models/2017_Toyota_Highlander_3.5_L.bin" +model_input_file = "models/2017_Toyota_Highlander_3.5_L.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -245,7 +245,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2020_Chevrolet_Colorado_2WD_Diesel" -model_file = "models/2020_Chevrolet_Colorado_2WD_Diesel.bin" +model_input_file = "models/2020_Chevrolet_Colorado_2WD_Diesel.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -255,7 +255,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2020_VW_Golf_1.5TSI" -model_file = "models/2020_VW_Golf_1.5TSI.bin" +model_input_file = "models/2020_VW_Golf_1.5TSI.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -265,7 +265,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2020_VW_Golf_2.0TDI" -model_file = "models/2020_VW_Golf_2.0TDI.bin" +model_input_file = "models/2020_VW_Golf_2.0TDI.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -275,7 +275,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2021_Fiat_Panda_Mild_Hybrid" -model_file = "models/2021_Fiat_Panda_Mild_Hybrid.bin" +model_input_file = "models/2021_Fiat_Panda_Mild_Hybrid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -285,7 +285,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2021_Peugot_3008" -model_file = "models/2021_Peugot_3008.bin" +model_input_file = "models/2021_Peugot_3008.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -295,7 +295,7 @@ real_world_energy_adjustment = 1.166 [[traversal.energy_models]] name = "2022_Ford_F-150_Lightning_4WD" -model_file = "models/2022_Ford_F-150_Lightning_4WD.bin" +model_input_file = "models/2022_Ford_F-150_Lightning_4WD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -305,7 +305,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Renault_Zoe_ZE50_R135" -model_file = "models/2022_Renault_Zoe_ZE50_R135.bin" +model_input_file = "models/2022_Renault_Zoe_ZE50_R135.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -315,7 +315,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Tesla_Model_3_RWD" -model_file = "models/2022_Tesla_Model_3_RWD.bin" +model_input_file = "models/2022_Tesla_Model_3_RWD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -325,7 +325,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Tesla_Model_Y_RWD" -model_file = "models/2022_Tesla_Model_Y_RWD.bin" +model_input_file = "models/2022_Tesla_Model_Y_RWD.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -335,7 +335,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2022_Toyota_Yaris_Hybrid_Mid" -model_file = "models/2022_Toyota_Yaris_Hybrid_Mid.bin" +model_input_file = "models/2022_Toyota_Yaris_Hybrid_Mid.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -345,7 +345,7 @@ real_world_energy_adjustment = 1.1252 [[traversal.energy_models]] name = "2022_Volvo_XC40_Recharge_twin" -model_file = "models/2022_Volvo_XC40_Recharge_twin.bin" +model_input_file = "models/2022_Volvo_XC40_Recharge_twin.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" @@ -355,7 +355,7 @@ real_world_energy_adjustment = 1.3958 [[traversal.energy_models]] name = "2023_Mitsubishi_Pajero_Sport" -model_file = "models/2023_Mitsubishi_Pajero_Sport.bin" +model_input_file = "models/2023_Mitsubishi_Pajero_Sport.bin" model_type = "smartcore" speed_unit = "miles_per_hour" grade_unit = "decimal" diff --git a/python/nrel/routee/compass/resources/osm_default_speed.toml b/python/nrel/routee/compass/resources/osm_default_speed.toml index 8a5279ca..af226e80 100644 --- a/python/nrel/routee/compass/resources/osm_default_speed.toml +++ b/python/nrel/routee/compass/resources/osm_default_speed.toml @@ -1,13 +1,13 @@ parallelism = 2 [graph] -edge_list_file = "edges-compass.csv.gz" -vertex_list_file = "vertices-compass.csv.gz" +edge_list_input_file = "edges-compass.csv.gz" +vertex_list_input_file = "vertices-compass.csv.gz" verbose = true [traversal] type = "speed_table" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" speed_unit = "kilometers_per_hour" output_distance_unit = "miles" output_time_unit = "minutes" @@ -15,10 +15,10 @@ output_time_unit = "minutes" [plugin] input_plugins = [ { type = "grid_search" }, - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "traversal", route = "geo_json", tree = "geo_json", geometry_file = "edges-geometries-enumerated.txt.gz" }, - { type = "uuid", uuid_file = "vertices-uuid-enumerated.txt.gz" }, + { type = "traversal", route = "geo_json", tree = "geo_json", geometry_input_file = "edges-geometries-enumerated.txt.gz" }, + { type = "uuid", uuid_input_file = "vertices-uuid-enumerated.txt.gz" }, ] diff --git a/rust/routee-compass-core/src/util/fs/read_utils.rs b/rust/routee-compass-core/src/util/fs/read_utils.rs index aa41c05b..b8845c60 100644 --- a/rust/routee-compass-core/src/util/fs/read_utils.rs +++ b/rust/routee-compass-core/src/util/fs/read_utils.rs @@ -1,7 +1,7 @@ use super::fs_utils; use csv::ReaderBuilder; use flate2::read::GzDecoder; -use std::io::ErrorKind; + use std::{ fs::File, io::{self, BufRead, BufReader}, diff --git a/rust/routee-compass-powertrain/src/doc.md b/rust/routee-compass-powertrain/src/doc.md index 5f87f34f..0ea5e76c 100644 --- a/rust/routee-compass-powertrain/src/doc.md +++ b/rust/routee-compass-powertrain/src/doc.md @@ -27,8 +27,8 @@ An example traversal model configuration that uses this crate may look like this [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "edges-posted-speed-enumerated.txt.gz" -energy_model_file = "2016_TOYOTA_Camry_4cyl_2WD.bin" +speed_table_input_file = "edges-posted-speed-enumerated.txt.gz" +energy_model_input_file = "2016_TOYOTA_Camry_4cyl_2WD.bin" ideal_energy_rate = 0.02857142857 speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" @@ -36,7 +36,7 @@ energy_model_grade_unit = "decimal" energy_model_energy_rate_unit = "gallons_gasoline_per_mile" output_time_unit = "minutes" output_distance_unit = "miles" -grade_table_file = "edges-grade-enumerated.txt.gz" +grade_table_input_file = "edges-grade-enumerated.txt.gz" grade_table_grade_unit = "decimal" ``` diff --git a/rust/routee-compass/src/app/compass/compass_app.rs b/rust/routee-compass/src/app/compass/compass_app.rs index 95a46185..67566f39 100644 --- a/rust/routee-compass/src/app/compass/compass_app.rs +++ b/rust/routee-compass/src/app/compass/compass_app.rs @@ -21,12 +21,12 @@ use config::Config; use itertools::{Either, Itertools}; use rayon::{current_num_threads, prelude::*}; use routee_compass_core::{ - algorithm::search::search_algorithm::SearchAlgorithm, model::cost::cost::Cost, + algorithm::search::search_algorithm::SearchAlgorithm, util::duration_extension::DurationExtension, }; use std::path::{Path, PathBuf}; -pub const CONFIG_FILE_KEY: &str = "config_file"; +pub const CONFIG_FILE_KEY: &str = "config_input_file"; /// Instance of RouteE Compass as an application. /// When constructed, it holds @@ -250,13 +250,14 @@ impl CompassApp { queries .iter() .map(|q| self.run_single_query(q.clone())) - .collect::, CompassAppError>>() + .collect::>, CompassAppError>>() }) - .collect::>, CompassAppError>>()?; + .collect::>>, CompassAppError>>()?; let run_result = run_query_result .into_iter() .flatten() + .flatten() .chain(input_error_responses) .collect(); @@ -278,7 +279,7 @@ impl CompassApp { pub fn run_single_query( &self, query: serde_json::Value, - ) -> Result { + ) -> Result, CompassAppError> { let search_result = self.search_app.run_vertex_oriented(&query); let output = apply_output_processing( (&query, search_result), @@ -337,9 +338,10 @@ pub fn apply_output_processing( response_data: (&serde_json::Value, Result), search_app: &SearchApp, output_plugins: &Vec>, -) -> serde_json::Value { +) -> Vec { let (req, res) = response_data; - match res { + + let init_output = match &res { Err(e) => { let error_output = serde_json::json!({ "request": req, @@ -348,14 +350,6 @@ pub fn apply_output_processing( error_output } Ok(result) => { - // should be moved into TraversalModel::summary, queries requesting - // min spanning tree result will not have an acc_cost. - let mut acc_cost = Cost::ZERO; - for traversal in result.route.clone() { - let cost = traversal.edge_cost(); - acc_cost = acc_cost + cost; - } - log::debug!( "completed route for request {}: {} links, {} tree size", req, @@ -363,24 +357,25 @@ pub fn apply_output_processing( result.tree.len() ); - // should be moved into TraversalModel::summary same reason as above + // should be moved into TraversalModel::summary, queries requesting + // min spanning tree result will not have a route. let route = result.route.to_vec(); let last_edge_traversal = match route.last() { None => { - return serde_json::json!({ + return vec![serde_json::json!({ "request": req, "error": "route was empty" - }); + })]; } Some(et) => et, }; let tmodel = match search_app.get_traversal_model_reference(req) { Err(e) => { - return serde_json::json!({ + return vec![serde_json::json!({ "request": req, "error": e.to_string() - }) + })] } Ok(tmodel) => tmodel, }; @@ -395,24 +390,41 @@ pub fn apply_output_processing( "tree_edge_count": result.tree.len(), "traversal_summary": tmodel.serialize_state_with_info(&last_edge_traversal.result_state), }); - let init_acc: Result = Ok(init_output); - let json_result = output_plugins - .iter() - .fold(init_acc, move |acc, plugin| match acc { - Err(e) => Err(e), - Ok(json) => plugin.process(&json, Ok(&result)), - }) - .map_err(CompassAppError::PluginError); - match json_result { - Err(e) => { - serde_json::json!({ - "request": req, - "error": e.to_string() + init_output + } + }; + + let init_acc: Result, PluginError> = Ok(vec![init_output]); + let json_result = output_plugins + .iter() + .fold(init_acc, |acc, p| { + acc.and_then(|outer| { + outer + .iter() + .map(|output| p.process(output, &res)) + .collect::, PluginError>>() + .map(|inner| { + inner + .into_iter() + .flatten() + .collect::>() }) - } - Ok(json) => json, - } + }) + }) + .map_err(|e| { + serde_json::json!({ + "request": req, + "error": e.to_string() + }) + }); + match json_result { + Err(e) => { + vec![serde_json::json!({ + "request": req, + "error": e.to_string() + })] } + Ok(json) => json, } } diff --git a/rust/routee-compass/src/app/compass/config/compass_app_builder.rs b/rust/routee-compass/src/app/compass/config/compass_app_builder.rs index 83472436..76c9917c 100644 --- a/rust/routee-compass/src/app/compass/config/compass_app_builder.rs +++ b/rust/routee-compass/src/app/compass/config/compass_app_builder.rs @@ -24,6 +24,7 @@ use crate::plugin::{ default::{ edgeidlist::builder::EdgeIdListOutputPluginBuilder, summary::builder::SummaryOutputPluginBuilder, + to_disk::builder::ToDiskOutputPluginBuilder, traversal::builder::TraversalPluginBuilder, uuid::builder::UUIDOutputPluginBuilder, }, output_plugin::OutputPlugin, @@ -120,11 +121,13 @@ impl CompassAppBuilder { let summary: Box = Box::new(SummaryOutputPluginBuilder {}); let uuid: Box = Box::new(UUIDOutputPluginBuilder {}); let edge_id_list: Box = Box::new(EdgeIdListOutputPluginBuilder {}); + let to_disk: Box = Box::new(ToDiskOutputPluginBuilder {}); let output_plugin_builders = HashMap::from([ (String::from("traversal"), traversal), (String::from("summary"), summary), (String::from("uuid"), uuid), (String::from("edge_id_list"), edge_id_list), + (String::from("to_disk"), to_disk), ]); CompassAppBuilder { diff --git a/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs b/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs index 0f46547d..02e367af 100644 --- a/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs +++ b/rust/routee-compass/src/app/compass/config/compass_configuration_error.rs @@ -28,14 +28,14 @@ pub enum CompassConfigurationError { Second, make sure the file exists. - Third, make sure the config key ends with '_file' which is a schema requirement + Third, make sure the config key ends with '_input_file' which is a schema requirement for the CompassApp config. "# )] FileNotFoundForComponent(String, String, String), #[error("could not normalize incoming file {0}")] FileNormalizationError(String), - #[error("Could not find incoming configuration file, tried {0} and {1}. Make sure the file exists and that the config key ends with '_file'")] + #[error("Could not find incoming configuration file, tried {0} and {1}. Make sure the file exists and that the config key ends with '_input_file'")] FileNormalizationNotFound(String, String), #[error("{0}")] InsertError(String), diff --git a/rust/routee-compass/src/app/compass/config/config_json_extension.rs b/rust/routee-compass/src/app/compass/config/config_json_extension.rs index 7b10341e..70929bf4 100644 --- a/rust/routee-compass/src/app/compass/config/config_json_extension.rs +++ b/rust/routee-compass/src/app/compass/config/config_json_extension.rs @@ -6,6 +6,8 @@ use std::{ str::FromStr, }; +const FILE_NORMALIZATION_POSTFIX: &str = "_input_file"; + pub trait ConfigJsonExtensions { fn get_config_section( &self, @@ -305,7 +307,10 @@ impl ConfigJsonExtensions for serde_json::Value { serde_json::Value::Object(obj) => { let mut new_obj = serde_json::map::Map::new(); for (key, value) in obj.iter() { - if key.ends_with("_file") || value.is_object() || value.is_array() { + if key.ends_with(FILE_NORMALIZATION_POSTFIX) + || value.is_object() + || value.is_array() + { new_obj.insert(key.clone(), value.normalize_file_paths(root_config_path)?); } else { new_obj.insert(key.clone(), value.clone()); diff --git a/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs b/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs index 4cfe56b3..1acc5d9c 100644 --- a/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs +++ b/rust/routee-compass/src/app/compass/config/frontier_model/road_class_builder.rs @@ -17,7 +17,7 @@ impl FrontierModelBuilder for RoadClassBuilder { parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { let frontier_key = CompassConfigurationField::Frontier.to_string(); - let road_class_file_key = String::from("road_class_file"); + let road_class_file_key = String::from("road_class_input_file"); let valid_road_class_key = String::from("valid_road_classes"); let road_class_file = diff --git a/rust/routee-compass/src/app/compass/config/graph_builder.rs b/rust/routee-compass/src/app/compass/config/graph_builder.rs index 1ddc01a3..5efcd848 100644 --- a/rust/routee-compass/src/app/compass/config/graph_builder.rs +++ b/rust/routee-compass/src/app/compass/config/graph_builder.rs @@ -28,9 +28,9 @@ impl DefaultGraphBuilder { pub fn build(params: &serde_json::Value) -> Result { let graph_key = CompassConfigurationField::Graph.to_string(); let edge_list_csv = - params.get_config_path(String::from("edge_list_file"), graph_key.clone())?; + params.get_config_path(String::from("edge_list_input_file"), graph_key.clone())?; let vertex_list_csv = - params.get_config_path(String::from("vertex_list_file"), graph_key.clone())?; + params.get_config_path(String::from("vertex_list_input_file"), graph_key.clone())?; let n_edges = params.get_config_serde_optional(String::from("n_edges"), graph_key.clone())?; let n_vertices = diff --git a/rust/routee-compass/src/app/compass/config/test/conf.toml b/rust/routee-compass/src/app/compass/config/test/conf.toml index 3dfbe4f3..4bc9c2df 100644 --- a/rust/routee-compass/src/app/compass/config/test/conf.toml +++ b/rust/routee-compass/src/app/compass/config/test/conf.toml @@ -1,6 +1,6 @@ [graph] -edge_file = "edges-compass.csv.gz" -vertex_file = "vertices-compass.csv.gz" +edge_input_file = "edges-compass.csv.gz" +vertex_input_file = "vertices-compass.csv.gz" n_edges = 125309338 n_vertices = 56306871 verbose = true @@ -15,9 +15,9 @@ type = "distance" [plugin] input_plugins = [ - { type = "vertex_rtree", vertices_file = "vertices-compass.csv.gz" }, + { type = "vertex_rtree", vertices_input_file = "vertices-compass.csv.gz" }, ] output_plugins = [ { type = "summary" }, - { type = "geometry", edge_file = "edges-geometries-enumerated.txt.gz" }, + { type = "geometry", edge_input_file = "edges-geometries-enumerated.txt.gz" }, ] diff --git a/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs b/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs index d12f143f..d84be62b 100644 --- a/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs +++ b/rust/routee-compass/src/app/compass/config/traversal_model/speed_grade_energy_model_builder.rs @@ -29,15 +29,19 @@ impl TraversalModelBuilder for SpeedGradeEnergyModelBuilder { ) -> Result, CompassConfigurationError> { let traversal_key = CompassConfigurationField::Traversal.to_string(); - let speed_table_path = - params.get_config_path(String::from("speed_table_file"), traversal_key.clone())?; + let speed_table_path = params.get_config_path( + String::from("speed_table_input_file"), + traversal_key.clone(), + )?; let speed_table_speed_unit = params.get_config_serde::( String::from("speed_table_speed_unit"), traversal_key.clone(), )?; - let grade_table_path = params - .get_config_path_optional(String::from("grade_table_file"), traversal_key.clone())?; + let grade_table_path = params.get_config_path_optional( + String::from("grade_table_input_file"), + traversal_key.clone(), + )?; let grade_table_grade_unit = params.get_config_serde_optional::( String::from("graph_grade_unit"), traversal_key.clone(), @@ -52,7 +56,7 @@ impl TraversalModelBuilder for SpeedGradeEnergyModelBuilder { let name = energy_model_config .get_config_string(String::from("name"), traversal_key.clone())?; let model_path = energy_model_config - .get_config_path(String::from("model_file"), traversal_key.clone())?; + .get_config_path(String::from("model_input_file"), traversal_key.clone())?; let model_type = energy_model_config .get_config_serde::(String::from("model_type"), traversal_key.clone())?; let speed_unit = energy_model_config diff --git a/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs b/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs index eaf30d18..cd152a07 100644 --- a/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs +++ b/rust/routee-compass/src/app/compass/config/traversal_model/speed_lookup_builder.rs @@ -22,8 +22,10 @@ impl TraversalModelBuilder for SpeedLookupBuilder { ) -> Result, CompassConfigurationError> { let traversal_key = CompassConfigurationField::Traversal.to_string(); // todo: optional output time unit - let filename = - params.get_config_path(String::from("speed_table_file"), traversal_key.clone())?; + let filename = params.get_config_path( + String::from("speed_table_input_file"), + traversal_key.clone(), + )?; let speed_unit = params .get_config_serde::(String::from("speed_unit"), traversal_key.clone())?; let distance_unit = params.get_config_serde_optional::( diff --git a/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml b/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml index 8739ff66..82a8fe73 100644 --- a/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml +++ b/rust/routee-compass/src/app/compass/test/energy_test/energy_debug.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "routee-compass/src/app/compass/test/energy_test/test_edges.csv" -vertex_list_file = "routee-compass/src/app/compass/test/energy_test/test_vertices.csv" +edge_list_input_file = "routee-compass/src/app/compass/test/energy_test/test_edges.csv" +vertex_list_input_file = "routee-compass/src/app/compass/test/energy_test/test_vertices.csv" verbose = true [algorithm] @@ -12,8 +12,8 @@ bidirectional = false [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "routee-compass/src/app/compass/test/energy_test/test_edge_speeds.csv" -energy_model_file = "../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" +speed_table_input_file = "routee-compass/src/app/compass/test/energy_test/test_edge_speeds.csv" +energy_model_input_file = "../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" energy_model_energy_rate_unit = "gallons_gasoline_per_mile" diff --git a/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml b/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml index 2baf677b..f1d305d6 100644 --- a/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml +++ b/rust/routee-compass/src/app/compass/test/energy_test/energy_test.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "src/app/compass/test/energy_test/test_edges.csv" -vertex_list_file = "src/app/compass/test/energy_test/test_vertices.csv" +edge_list_input_file = "src/app/compass/test/energy_test/test_edges.csv" +vertex_list_input_file = "src/app/compass/test/energy_test/test_vertices.csv" verbose = true [algorithm] @@ -12,8 +12,8 @@ bidirectional = false [traversal] type = "speed_grade_energy_model" model_type = "smartcore" -speed_table_file = "src/app/compass/test/energy_test/test_edge_speeds.csv" -energy_model_file = "../../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" +speed_table_input_file = "src/app/compass/test/energy_test/test_edge_speeds.csv" +energy_model_input_file = "../../routee-compass-powertrain/src/routee/test/Toyota_Camry.bin" speed_table_speed_unit = "kilometers_per_hour" energy_model_speed_unit = "miles_per_hour" energy_model_energy_rate_unit = "gallons_gasoline_per_mile" diff --git a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml index 7060ba87..e152f156 100644 --- a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml +++ b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_debug.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "compass-app/src/app/compass/test/speeds_test/test_edges.csv" -vertex_list_file = "compass-app/src/app/compass/test/speeds_test/test_vertices.csv" +edge_list_input_file = "compass-app/src/app/compass/test/speeds_test/test_edges.csv" +vertex_list_input_file = "compass-app/src/app/compass/test/speeds_test/test_vertices.csv" verbose = true [algorithm] @@ -11,7 +11,7 @@ bidirectional = false [traversal] type = "speed_table" -speed_table_file = "compass-app/src/app/compass/test/speeds_test/test_edge_speeds.csv" +speed_table_input_file = "compass-app/src/app/compass/test/speeds_test/test_edge_speeds.csv" speed_unit = "kilometers_per_hour" output_time_unit = "hours" diff --git a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml index 2487103b..dc0963c8 100644 --- a/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml +++ b/rust/routee-compass/src/app/compass/test/speeds_test/speeds_test.toml @@ -1,8 +1,8 @@ query_timeout_ms = 3600000 [graph] -edge_list_file = "src/app/compass/test/speeds_test/test_edges.csv" -vertex_list_file = "src/app/compass/test/speeds_test/test_vertices.csv" +edge_list_input_file = "src/app/compass/test/speeds_test/test_edges.csv" +vertex_list_input_file = "src/app/compass/test/speeds_test/test_vertices.csv" verbose = true [algorithm] @@ -11,7 +11,7 @@ bidirectional = false [traversal] type = "speed_table" -speed_table_file = "src/app/compass/test/speeds_test/test_edge_speeds.csv" +speed_table_input_file = "src/app/compass/test/speeds_test/test_edge_speeds.csv" speed_unit = "kilometers_per_hour" output_time_unit = "hours" diff --git a/rust/routee-compass/src/main.rs b/rust/routee-compass/src/main.rs index 899a1d6f..a7147854 100644 --- a/rust/routee-compass/src/main.rs +++ b/rust/routee-compass/src/main.rs @@ -39,10 +39,14 @@ fn main() -> Result<(), Box> { let user_queries = user_json.get_queries()?; info!("Query: {:?}", user_json); - // run searches and return result - let output_rows = compass_app.run(user_queries)?; - let output_contents = serde_json::to_string(&output_rows)?; - std::fs::write("result.json", output_contents)?; + let results = compass_app.run(user_queries)?; + + // scan the results and log any json values that have "error" in them + for result in results.iter() { + if let Some(error) = result.get("error") { + error!("Error: {}", error); + } + } return Ok(()); } diff --git a/rust/routee-compass/src/plugin/input/default/rtree/builder.rs b/rust/routee-compass/src/plugin/input/default/rtree/builder.rs index 491a5c43..3b0bcde4 100644 --- a/rust/routee-compass/src/plugin/input/default/rtree/builder.rs +++ b/rust/routee-compass/src/plugin/input/default/rtree/builder.rs @@ -15,7 +15,7 @@ impl InputPluginBuilder for VertexRTreeBuilder { &self, parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { - let vertex_filename_key = String::from("vertices_file"); + let vertex_filename_key = String::from("vertices_input_file"); let vertex_path = parameters.get_config_path( vertex_filename_key, String::from("Vertex RTree Input Plugin"), diff --git a/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs b/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs index 0a458ed8..7ddb9a76 100644 --- a/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/edgeidlist/plugin.rs @@ -1,8 +1,8 @@ +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::output::default::edgeidlist::json_extensions::EdgeListJsonExtensions; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::algorithm::search::search_error::SearchError; use serde_json; pub struct EdgeIdListOutputPlugin {} @@ -11,10 +11,10 @@ impl OutputPlugin for EdgeIdListOutputPlugin { fn process( &self, output: &serde_json::Value, - search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + search_result: &Result, + ) -> Result, PluginError> { match search_result { - Err(_e) => Ok(output.clone()), + Err(_e) => Ok(vec![output.clone()]), Ok(result) => { let edge_ids = result .route @@ -24,7 +24,7 @@ impl OutputPlugin for EdgeIdListOutputPlugin { .collect::>(); let mut updated = output.clone(); updated.add_edge_list(&edge_ids)?; - Ok(updated) + Ok(vec![updated]) } } } diff --git a/rust/routee-compass/src/plugin/output/default/mod.rs b/rust/routee-compass/src/plugin/output/default/mod.rs index ef9a3837..9932f09b 100644 --- a/rust/routee-compass/src/plugin/output/default/mod.rs +++ b/rust/routee-compass/src/plugin/output/default/mod.rs @@ -1,4 +1,5 @@ pub mod edgeidlist; pub mod summary; +pub mod to_disk; pub mod traversal; pub mod uuid; diff --git a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs index ff0582a6..da3a3e52 100644 --- a/rust/routee-compass/src/plugin/output/default/summary/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/summary/plugin.rs @@ -1,8 +1,10 @@ use super::json_extensions::SummaryJsonExtensions; -use crate::app::search::search_app_result::SearchAppResult; +use crate::app::{ + compass::compass_app_error::CompassAppError, search::search_app_result::SearchAppResult, +}; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::{algorithm::search::search_error::SearchError, model::cost::cost::Cost}; +use routee_compass_core::model::cost::cost::Cost; use serde_json; pub struct SummaryOutputPlugin {} @@ -12,10 +14,10 @@ impl OutputPlugin for SummaryOutputPlugin { fn process( &self, output: &serde_json::Value, - search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + search_result: &Result, + ) -> Result, PluginError> { match search_result { - Err(_e) => Ok(output.clone()), + Err(_e) => Ok(vec![output.clone()]), Ok(result) => { let mut updated_output = output.clone(); let cost = result @@ -24,7 +26,7 @@ impl OutputPlugin for SummaryOutputPlugin { .map(|traversal| traversal.edge_cost()) .sum::(); updated_output.add_cost(cost)?; - Ok(updated_output) + Ok(vec![updated_output]) } } } @@ -76,9 +78,9 @@ mod tests { }; let summary_plugin = SummaryOutputPlugin {}; let updated_output = summary_plugin - .process(&output_result, Ok(&search_result)) + .process(&output_result, &Ok(search_result)) .unwrap(); - let cost: f64 = updated_output.get_cost().unwrap().into(); + let cost: f64 = updated_output[0].get_cost().unwrap().into(); assert_eq!(cost, 6.0); } } diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs new file mode 100644 index 00000000..f51ce897 --- /dev/null +++ b/rust/routee-compass/src/plugin/output/default/to_disk/builder.rs @@ -0,0 +1,54 @@ +use std::{ + fs::OpenOptions, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use crate::{ + app::compass::config::{ + builders::OutputPluginBuilder, compass_configuration_error::CompassConfigurationError, + config_json_extension::ConfigJsonExtensions, + }, + plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}, +}; + +use super::plugin::ToDiskOutputPlugin; + +pub struct ToDiskOutputPluginBuilder {} + +impl OutputPluginBuilder for ToDiskOutputPluginBuilder { + fn build( + &self, + parameters: &serde_json::Value, + ) -> Result, CompassConfigurationError> { + let output_filename_key = String::from("output_file"); + let output_filename = + parameters.get_config_string(output_filename_key, String::from("output"))?; + + let output_file_path = PathBuf::from(&output_filename); + + // initialize the file with nothing + std::fs::write(&output_file_path, "")?; + + // open the file with the option to append to it + let file = OpenOptions::new() + .write(true) + .append(true) + .open(&output_file_path) + .map_err(|e| { + PluginError::FileReadError( + output_file_path.clone(), + format!("Could not open output file: {}", e), + ) + })?; + + // wrap the file in a mutex so we can share it between threads + let output_file = Arc::new(Mutex::new(file)); + + let to_disk_plugin = ToDiskOutputPlugin { + output_file_path, + output_file, + }; + Ok(Box::new(to_disk_plugin)) + } +} diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs b/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs new file mode 100644 index 00000000..dc15aad0 --- /dev/null +++ b/rust/routee-compass/src/plugin/output/default/to_disk/mod.rs @@ -0,0 +1,2 @@ +pub mod builder; +pub mod plugin; diff --git a/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs new file mode 100644 index 00000000..3ffda3e4 --- /dev/null +++ b/rust/routee-compass/src/plugin/output/default/to_disk/plugin.rs @@ -0,0 +1,44 @@ +use std::fs::File; +use std::io::prelude::*; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use crate::{ + app::{ + compass::compass_app_error::CompassAppError, search::search_app_result::SearchAppResult, + }, + plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}, +}; + +pub struct ToDiskOutputPlugin { + pub output_file_path: PathBuf, + pub output_file: Arc>, +} + +impl OutputPlugin for ToDiskOutputPlugin { + fn process( + &self, + output: &serde_json::Value, + _result: &Result, + ) -> Result, PluginError> { + let file_ref = Arc::clone(&self.output_file); + let mut file = file_ref.lock().map_err(|e| { + PluginError::FileReadError( + self.output_file_path.clone(), + format!("Could not aquire lock on output file: {}", e), + ) + })?; + + let output_json = serde_json::to_string(output)?; + + writeln!(file, "{}", output_json).map_err(|e| { + PluginError::FileReadError( + self.output_file_path.clone(), + format!("Could not write to output file: {}", e), + ) + })?; + + // return empty vec since we already wrote the result to a file + Ok(Vec::new()) + } +} diff --git a/rust/routee-compass/src/plugin/output/default/traversal/builder.rs b/rust/routee-compass/src/plugin/output/default/traversal/builder.rs index be82bb4d..734291d2 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/builder.rs @@ -27,7 +27,7 @@ use crate::{ /// type = "traversal" /// route = "geo_json" /// tree = "geo_json" -/// geometry_file = "edges-geometries-enumerated.txt.gz" +/// geometry_input_file = "edges-geometries-enumerated.txt.gz" /// ``` /// pub struct TraversalPluginBuilder {} @@ -40,7 +40,7 @@ impl OutputPluginBuilder for TraversalPluginBuilder { parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { let parent_key = String::from("traversal"); - let geometry_filename_key = String::from("geometry_file"); + let geometry_filename_key = String::from("geometry_input_file"); let route_geometry_key = String::from("route"); let tree_geometry_key = String::from("tree"); diff --git a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs index 6efc8a88..9f9f2c38 100644 --- a/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/traversal/plugin.rs @@ -1,13 +1,13 @@ use super::json_extensions::TraversalJsonField; use super::traversal_output_format::TraversalOutputFormat; use super::utils::parse_linestring; +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::output::output_plugin::OutputPlugin; use crate::plugin::plugin_error::PluginError; use geo::LineString; use kdam::Bar; use kdam::BarExt; -use routee_compass_core::algorithm::search::search_error::SearchError; use routee_compass_core::util::fs::fs_utils; use routee_compass_core::util::fs::read_utils::read_raw_file; use std::path::Path; @@ -26,10 +26,7 @@ impl TraversalPlugin { ) -> Result { let count = fs_utils::line_count(filename.clone(), fs_utils::is_gzip(&filename)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; let mut pb = Bar::builder() @@ -43,10 +40,7 @@ impl TraversalPlugin { pb.update(1); }); let geoms = read_raw_file(&filename, parse_linestring, Some(cb)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; print!("\n"); Ok(TraversalPlugin { geoms, route, tree }) @@ -57,10 +51,10 @@ impl OutputPlugin for TraversalPlugin { fn process( &self, output: &serde_json::Value, - search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + search_result: &Result, + ) -> Result, PluginError> { match search_result { - Err(_) => Ok(output.clone()), + Err(_) => Ok(vec![output.clone()]), Ok(result) => { let mut output_mut = output.clone(); let updated = output_mut @@ -88,7 +82,7 @@ impl OutputPlugin for TraversalPlugin { } } - Ok(serde_json::Value::Object(updated.to_owned())) + Ok(vec![serde_json::Value::Object(updated.to_owned())]) } } } @@ -177,9 +171,9 @@ mod tests { TraversalPlugin::from_file(&filename, Some(TraversalOutputFormat::Wkt), None).unwrap(); let result = geom_plugin - .process(&output_result, Ok(&search_result)) + .process(&output_result, &Ok(search_result)) .unwrap(); - let geometry_wkt = result.get_route_geometry_wkt().unwrap(); + let geometry_wkt = result[0].get_route_geometry_wkt().unwrap(); assert_eq!(geometry_wkt, expected_geometry); } } diff --git a/rust/routee-compass/src/plugin/output/default/uuid/builder.rs b/rust/routee-compass/src/plugin/output/default/uuid/builder.rs index 8281be3c..b1c9cfbc 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/builder.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/builder.rs @@ -15,7 +15,7 @@ impl OutputPluginBuilder for UUIDOutputPluginBuilder { &self, parameters: &serde_json::Value, ) -> Result, CompassConfigurationError> { - let uuid_filename_key = String::from("uuid_file"); + let uuid_filename_key = String::from("uuid_input_file"); let uuid_filename = parameters.get_config_path(uuid_filename_key, String::from("uuid"))?; let uuid_plugin = UUIDOutputPlugin::from_file(&uuid_filename) diff --git a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs index a8faa6b9..de63f45e 100644 --- a/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs +++ b/rust/routee-compass/src/plugin/output/default/uuid/plugin.rs @@ -1,9 +1,10 @@ use super::json_extensions::UUIDJsonExtensions; +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::{output::output_plugin::OutputPlugin, plugin_error::PluginError}; use kdam::Bar; use kdam::BarExt; -use routee_compass_core::algorithm::search::search_error::SearchError; + use routee_compass_core::util::fs::{fs_utils, read_utils::read_raw_file}; use std::path::Path; @@ -15,10 +16,7 @@ impl UUIDOutputPlugin { pub fn from_file>(filename: &P) -> Result { let count = fs_utils::line_count(filename.clone(), fs_utils::is_gzip(&filename)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; let mut pb = Bar::builder() @@ -33,10 +31,7 @@ impl UUIDOutputPlugin { }); let uuids = read_raw_file(&filename, |_idx, row| Ok(row), Some(cb)).map_err(|e| { - PluginError::FileReadError { - filename: filename.as_ref().to_path_buf(), - message: e.to_string(), - } + PluginError::FileReadError(filename.as_ref().to_path_buf(), e.to_string()) })?; print!("\n"); Ok(UUIDOutputPlugin { uuids }) @@ -47,8 +42,8 @@ impl OutputPlugin for UUIDOutputPlugin { fn process( &self, output: &serde_json::Value, - _search_result: Result<&SearchAppResult, SearchError>, - ) -> Result { + _search_result: &Result, + ) -> Result, PluginError> { let mut updated_output = output.clone(); let (origin_vertex_id, destination_vertex_id) = output.get_od_vertex_ids()?; let origin_uuid = self @@ -62,6 +57,6 @@ impl OutputPlugin for UUIDOutputPlugin { updated_output.add_od_uuids(origin_uuid.clone(), destination_uuid.clone())?; - Ok(updated_output) + Ok(vec![updated_output]) } } diff --git a/rust/routee-compass/src/plugin/output/output_plugin.rs b/rust/routee-compass/src/plugin/output/output_plugin.rs index e9c318d0..76347150 100644 --- a/rust/routee-compass/src/plugin/output/output_plugin.rs +++ b/rust/routee-compass/src/plugin/output/output_plugin.rs @@ -1,6 +1,6 @@ +use crate::app::compass::compass_app_error::CompassAppError; use crate::app::search::search_app_result::SearchAppResult; use crate::plugin::plugin_error::PluginError; -use routee_compass_core::algorithm::search::search_error::SearchError; /// Performs some kind of post-processing on a search result. The result JSON is available /// to the plugin as a reference which was potentially modified upstream by another output @@ -37,6 +37,6 @@ pub trait OutputPlugin: Send + Sync { fn process( &self, output: &serde_json::Value, - result: Result<&SearchAppResult, SearchError>, - ) -> Result; + result: &Result, + ) -> Result, PluginError>; } diff --git a/rust/routee-compass/src/plugin/plugin_error.rs b/rust/routee-compass/src/plugin/plugin_error.rs index 62a89fd7..d3817a76 100644 --- a/rust/routee-compass/src/plugin/plugin_error.rs +++ b/rust/routee-compass/src/plugin/plugin_error.rs @@ -17,8 +17,8 @@ pub enum PluginError { BuildError, #[error("nearest vertex not found for coord {0:?}")] NearestVertexNotFound(Coord), - #[error("unable to read file {filename} due to {message}")] - FileReadError { filename: PathBuf, message: String }, + #[error("unable to read file {0} due to {1}")] + FileReadError(PathBuf, String), #[error(transparent)] JsonError(#[from] serde_json::Error), #[error(transparent)]