Skip to content

Commit

Permalink
Report latency at server startup (parseablehq#551) (parseablehq#562)
Browse files Browse the repository at this point in the history
- Measure the `[time]` taken to get the `.parseable.json` object from 
the store at server startup
- Print `[time]` on the banner

This is useful to display current latency at server startup time and
will help the user decide if latency is higher than expected in their
deployment.
  • Loading branch information
theteachr authored Dec 3, 2023
1 parent 3aefb9b commit 05de709
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 5 deletions.
18 changes: 14 additions & 4 deletions server/src/banner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub async fn print(config: &Config, meta: &StorageMetadata) {
print_ascii_art();
let scheme = config.parseable.get_scheme();
status_info(config, &scheme, meta.deployment_id);
storage_info(config);
storage_info(config).await;
about::print(config, meta).await;
println!();
}
Expand Down Expand Up @@ -79,16 +79,26 @@ fn status_info(config: &Config, scheme: &str, id: Uid) {
);
}

fn storage_info(config: &Config) {
/// Prints information about the `ObjectStorage`.
/// - Mode (`Local drive`, `S3 bucket`)
/// - Staging (temporary landing point for incoming events)
/// - Store (path where the data is stored)
/// - Latency
async fn storage_info(config: &Config) {
let storage = config.storage();
let latency = storage.get_object_store().get_latency().await;

eprintln!(
"
{}
Mode: \"{}\"
Staging: \"{}\"
Store: \"{}\"",
Store: \"{}\"
Latency: \"{:?}\"",
"Storage:".to_string().bold(),
config.mode_string(),
config.staging_dir().to_string_lossy(),
config.storage().get_endpoint(),
storage.get_endpoint(),
latency
)
}
21 changes: 20 additions & 1 deletion server/src/storage/object_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@ use relative_path::RelativePath;
use relative_path::RelativePathBuf;
use serde_json::Value;

use std::{collections::HashMap, fs, path::Path, sync::Arc};
use std::{
collections::HashMap,
fs,
path::Path,
sync::Arc,
time::{Duration, Instant},
};

// metadata file names in a Stream prefix
pub(super) const STREAM_METADATA_FILE_NAME: &str = ".stream.json";
Expand Down Expand Up @@ -67,6 +73,19 @@ pub trait ObjectStorage: Sync + 'static {
async fn list_streams(&self) -> Result<Vec<LogStream>, ObjectStorageError>;
async fn list_dates(&self, stream_name: &str) -> Result<Vec<String>, ObjectStorageError>;
async fn upload_file(&self, key: &str, path: &Path) -> Result<(), ObjectStorageError>;

/// Returns the amount of time taken by the `ObjectStore` to perform a get
/// call.
async fn get_latency(&self) -> Duration {
// It's Ok to `unwrap` here. The hardcoded value will always Result in
// an `Ok`.
let path = RelativePathBuf::from_path(".parseable.json").unwrap();

let start = Instant::now();
let _ = self.get_object(&path).await;
start.elapsed()
}

fn normalize_prefixes(&self, prefixes: Vec<String>) -> Vec<String>;
fn query_prefixes(&self, prefixes: Vec<String>) -> Vec<ListingTableUrl>;
fn store_url(&self) -> url::Url;
Expand Down

0 comments on commit 05de709

Please sign in to comment.