Rjf/read progress uniformity

docs/developers/rust_code_style.md

@@ -6,7 +6,7 @@ while a `Vec<T>` is more common, it lacks the ability to match capacity exactly
 
 ```rust
 let path: Path = todo!();
-let data: Vec<T> = read_utils::from_csv(&file_path, True, None)?;
+let data: Vec<T> = read_utils::from_csv(&file_path, true, None, None, None)?;
 let output: Box<[T]> = data.into_boxed_slice();
 ```
 

rust/routee-compass-core/src/model/cost/network/network_cost_rate_builder.rs

@@ -7,6 +7,7 @@ use crate::{
     },
     util::fs::read_utils,
 };
+use kdam::Bar;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
@@ -27,31 +28,39 @@ impl NetworkCostRateBuilder {
         use NetworkCostRateBuilder as Builder;
         match self {
             Builder::EdgeLookupBuilder { cost_input_file } => {
-                let lookup =
-                    read_utils::from_csv::<NetworkTraversalUtilityRow>(cost_input_file, true, None)
-                        .map_err(|source| {
-                            CostModelError::BuildError(format!(
-                                "failure reading file {}: {}",
-                                cost_input_file, source
-                            ))
-                        })?
-                        .iter()
-                        .map(|row| (row.edge_id, row.cost))
-                        .collect::<HashMap<_, _>>();
+                let lookup = read_utils::from_csv::<NetworkTraversalUtilityRow>(
+                    cost_input_file,
+                    true,
+                    Some(Bar::builder().desc("network edge cost lookup")),
+                    None,
+                )
+                .map_err(|source| {
+                    CostModelError::BuildError(format!(
+                        "failure reading file {}: {}",
+                        cost_input_file, source
+                    ))
+                })?
+                .iter()
+                .map(|row| (row.edge_id, row.cost))
+                .collect::<HashMap<_, _>>();
                 Ok(NCM::EdgeLookup { lookup })
             }
             Builder::EdgeEdgeLookupBuilder { cost_input_file } => {
-                let lookup =
-                    read_utils::from_csv::<NetworkAccessUtilityRow>(cost_input_file, true, None)
-                        .map_err(|source| {
-                            CostModelError::BuildError(format!(
-                                "failure reading file {}: {}",
-                                cost_input_file, source
-                            ))
-                        })?
-                        .iter()
-                        .map(|row| ((row.source, row.destination), row.cost))
-                        .collect::<HashMap<_, _>>();
+                let lookup = read_utils::from_csv::<NetworkAccessUtilityRow>(
+                    cost_input_file,
+                    true,
+                    Some(Bar::builder().desc("network edge->edge cost lookup")),
+                    None,
+                )
+                .map_err(|source| {
+                    CostModelError::BuildError(format!(
+                        "failure reading file {}: {}",
+                        cost_input_file, source
+                    ))
+                })?
+                .iter()
+                .map(|row| ((row.source, row.destination), row.cost))
+                .collect::<HashMap<_, _>>();
 
                 Ok(NCM::EdgeEdgeLookup { lookup })
             }

rust/routee-compass-core/src/model/map/geometry_model.rs

@@ -6,7 +6,7 @@ use crate::{
     util::{fs::read_utils, geo::geo_io_utils},
 };
 use geo::LineString;
-use kdam::BarExt;
+use kdam::{Bar, BarExt};
 
 /// model for link geometries by edge id. can be constructed either
 /// from edge geometry dataset ([`GeometryModel::new_from_edges`]) or
@@ -47,20 +47,11 @@ fn read_linestrings(
     geometry_input_file: &String,
     n_edges: usize,
 ) -> Result<Vec<geo::LineString<f32>>, MapError> {
-    let mut pb = kdam::Bar::builder()
-        .total(n_edges)
-        .animation("fillup")
-        .desc("edge LineString geometry file")
-        .build()
-        .map_err(MapError::InternalError)?;
-
-    let cb = Box::new(|| {
-        let _ = pb.update(1);
-    });
     let geoms = read_utils::read_raw_file(
         geometry_input_file,
         geo_io_utils::parse_wkt_linestring,
-        Some(cb),
+        Some(Bar::builder().desc("link geometries").total(n_edges)),
+        None,
     )
     .map_err(|e: std::io::Error| {
         MapError::BuildError(format!("error loading {}: {}", geometry_input_file, e))
@@ -187,7 +178,7 @@ mod tests {
 
     #[test]
     fn test_geometry_deserialization() {
-        let result = read_raw_file(mock_geometry_file(), parse_wkt_linestring, None).unwrap();
+        let result = read_raw_file(mock_geometry_file(), parse_wkt_linestring, None, None).unwrap();
         assert_eq!(result.len(), 3);
     }
 

rust/routee-compass-core/src/model/map/spatial_index.rs

@@ -130,7 +130,7 @@ mod test {
             .join("rtree_query.json");
 
         let vertices: Box<[Vertex]> =
-            read_utils::from_csv(&vertices_filepath.as_path(), true, None).unwrap();
+            read_utils::from_csv(&vertices_filepath.as_path(), true, None, None).unwrap();
         let index = SpatialIndex::new_vertex_oriented(&vertices, None);
 
         // test nearest neighbor queries perform as expected

rust/routee-compass-core/src/model/network/graph.rs

@@ -1,12 +1,12 @@
 use super::{Edge, EdgeId, NetworkError, Vertex, VertexId};
 use crate::algorithm::search::direction::Direction;
 use crate::util::compact_ordered_hash_map::CompactOrderedHashMap;
-use std::path::Path;
-
-use super::graph_loader::graph_from_files;
-
+use crate::util::fs::read_utils;
 use allocative::Allocative;
 use itertools::Itertools;
+use kdam::Bar;
+use std::collections::HashSet;
+use std::path::Path;
 
 /// Road network topology represented as an adjacency list.
 /// The `EdgeId` and `VertexId` values correspond to edge and
@@ -37,17 +37,11 @@ pub struct Graph {
 
 impl Graph {
     /// Build a `Graph` from a pair of CSV files.
-    /// You can also pass in the number of edges and vertices to avoid
-    /// scanning the input files and potentially building vectors that
-    /// have more memory than needed.
     ///
     /// # Arguments
     ///
     /// * `edge_list_csv` - path to the CSV file containing edge attributes
     /// * `vertex_list_csv` - path to the CSV file containing vertex attributes
-    /// * `n_edges` - number of edges in the graph
-    /// * `n_vertices` - number of vertices in the graph
-    /// * `verbose` - whether to print progress information to the console
     ///
     /// # Returns
     ///
@@ -56,11 +50,56 @@ impl Graph {
     pub fn from_files<P: AsRef<Path>>(
         edge_list_csv: &P,
         vertex_list_csv: &P,
-        n_edges: Option<usize>,
-        n_vertices: Option<usize>,
-        verbose: Option<bool>,
     ) -> Result<Graph, NetworkError> {
-        graph_from_files(edge_list_csv, vertex_list_csv, n_edges, n_vertices, verbose)
+        let vertices: Box<[Vertex]> = read_utils::from_csv(
+            &vertex_list_csv,
+            true,
+            Some(Bar::builder().desc("graph vertices")),
+            None,
+        )
+        .map_err(|e| NetworkError::CsvError { source: e })?;
+
+        let mut adj: Vec<CompactOrderedHashMap<EdgeId, VertexId>> =
+            vec![CompactOrderedHashMap::empty(); vertices.len()];
+        let mut rev: Vec<CompactOrderedHashMap<EdgeId, VertexId>> =
+            vec![CompactOrderedHashMap::empty(); vertices.len()];
+        let mut missing_vertices: HashSet<VertexId> = HashSet::new();
+        let cb = Box::new(|edge: &Edge| {
+            // the Edge provides us with all id information to build our adjacency lists as well
+            match adj.get_mut(edge.src_vertex_id.0) {
+                None => {
+                    missing_vertices.insert(edge.src_vertex_id);
+                }
+                Some(out_links) => {
+                    out_links.insert(edge.edge_id, edge.dst_vertex_id);
+                }
+            }
+            match rev.get_mut(edge.dst_vertex_id.0) {
+                None => {
+                    missing_vertices.insert(edge.dst_vertex_id);
+                }
+                Some(in_links) => {
+                    in_links.insert(edge.edge_id, edge.src_vertex_id);
+                }
+            }
+        });
+
+        let edges = read_utils::from_csv(
+            &edge_list_csv,
+            true,
+            Some(Bar::builder().desc("graph edges")),
+            Some(cb),
+        )
+        .map_err(|e| NetworkError::CsvError { source: e })?;
+
+        let graph = Graph {
+            adj: adj.into_boxed_slice(),
+            rev: rev.into_boxed_slice(),
+            edges,
+            vertices,
+        };
+
+        Ok(graph)
     }
     /// number of edges in the Graph
     pub fn n_edges(&self) -> usize {