From 9eee0501598b268e94619e37becc91b7f0ba46e6 Mon Sep 17 00:00:00 2001 From: David Cook Date: Tue, 30 Apr 2024 08:40:15 -0500 Subject: [PATCH] Upgrade testcontainers to 0.16 (#3063) --- Cargo.lock | 247 ++++++++++++++++-- Cargo.toml | 2 +- aggregator/src/binary_utils.rs | 22 +- aggregator_core/src/datastore/test_util.rs | 48 +--- core/src/test_util/testcontainers.rs | 20 +- deny.toml | 1 + integration_tests/src/client.rs | 39 ++- integration_tests/src/daphne.rs | 29 +- integration_tests/src/janus.rs | 41 ++- integration_tests/tests/integration/common.rs | 4 +- integration_tests/tests/integration/daphne.rs | 35 +-- .../tests/integration/divviup_ts.rs | 36 +-- integration_tests/tests/integration/janus.rs | 39 +-- interop_binaries/src/lib.rs | 23 +- interop_binaries/tests/end_to_end.rs | 69 ++--- 15 files changed, 375 insertions(+), 280 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa9d80fdd..703d26804 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -665,13 +665,53 @@ dependencies = [ "tracing", ] +[[package]] +name = "bollard" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aed08d3adb6ebe0eff737115056652670ae290f177759aac19c30456135f94c" +dependencies = [ + "base64 0.22.0", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.1.0", + "http-body-util", + "hyper 1.2.0", + "hyper-named-pipe", + "hyper-rustls 0.26.0", + "hyper-util", + "hyperlocal-next", + "log", + "pin-project-lite", + "rustls 0.22.4", + "rustls-native-certs", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + [[package]] name = "bollard-stubs" -version = "1.42.0-rc.3" +version = "1.44.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864" +checksum = "709d9aa1c37abb89d40f19f5d0ad6f0d88cb1581264e571c9350fc5bb89cf1c5" dependencies = [ "serde", + "serde_repr", "serde_with", ] @@ -1078,9 +1118,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.4" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ "darling_core", "darling_macro", @@ -1088,27 +1128,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.4" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn 1.0.109", + "syn 2.0.55", ] [[package]] name = "darling_macro" -version = "0.13.4" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.55", ] [[package]] @@ -1230,6 +1270,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "divviup-client" version = "0.2.1" @@ -1251,6 +1312,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "docker_credential" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31951f49556e34d90ed28342e1df7e1cb7a229c4cab0aecc627b5d91edd41d07" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -2106,6 +2178,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.2.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.26.0" @@ -2116,7 +2203,9 @@ dependencies = [ "http 1.1.0", "hyper 1.2.0", "hyper-util", + "log", "rustls 0.22.4", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", @@ -2187,6 +2276,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal-next" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2234,6 +2338,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -2244,6 +2349,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", + "serde", ] [[package]] @@ -2905,6 +3011,16 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.27.0" @@ -3311,6 +3427,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "2.10.1" @@ -3389,6 +3511,31 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "parse-display" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06af5f9333eb47bd9ba8462d612e37a8328a5cb80b13f0af4de4c3b89f52dee5" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax 0.8.2", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc9252f259500ee570c75adcc4e317fa6f57a1e47747d622e0bf838002a7b790" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax 0.8.2", + "structmeta", + "syn 2.0.55", +] + [[package]] name = "paste" version = "1.0.14" @@ -3932,6 +4079,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.10.4" @@ -4467,6 +4625,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "serde_spanned" version = "0.6.5" @@ -4499,24 +4668,32 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" dependencies = [ + "base64 0.22.0", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", "serde", + "serde_derive", + "serde_json", "serde_with_macros", + "time", ] [[package]] name = "serde_with_macros" -version = "1.5.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.55", ] [[package]] @@ -5014,6 +5191,29 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.55", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.55", +] + [[package]] name = "subtle" version = "2.5.0" @@ -5074,19 +5274,24 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.15.0" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d2931d7f521af5bae989f716c3fa43a6af9af7ec7a5e21b59ae40878cec00" +checksum = "279c6fb2b8db23b09ce3a68ec5dd720530e5743af060a65be8ab9acfb7394aef" dependencies = [ + "async-trait", + "bollard", "bollard-stubs", + "dirs", + "docker_credential", "futures", - "hex", - "hmac", "log", - "rand", + "parse-display", "serde", "serde_json", - "sha2", + "serde_with", + "tokio", + "tokio-util", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 425f372f7..e9908028b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,7 +98,7 @@ signal-hook-tokio = "0.3.1" sqlx = "0.7.4" stopper = "0.2.7" tempfile = "3.10.1" -testcontainers = "0.15.0" +testcontainers = "0.16.5" thiserror = "1.0" tracing = "0.1.40" tracing-chrome = "0.7.2" diff --git a/aggregator/src/binary_utils.rs b/aggregator/src/binary_utils.rs index dcd7ee4ed..39a3284e3 100644 --- a/aggregator/src/binary_utils.rs +++ b/aggregator/src/binary_utils.rs @@ -556,8 +556,8 @@ mod tests { use crate::{ aggregator::http_handlers::test_util::take_response_body, binary_utils::{ - database_pool, register_database_pool_status_metrics, zpages_handler, - CommonBinaryOptions, + database_pool, initialize_rustls, register_database_pool_status_metrics, + zpages_handler, CommonBinaryOptions, }, config::DbConfig, }; @@ -565,7 +565,7 @@ mod tests { use janus_aggregator_core::datastore::test_util::ephemeral_datastore; use janus_core::test_util::{ install_test_trace_subscriber, - testcontainers::{container_client, Postgres, Volume}, + testcontainers::{Postgres, Volume}, }; use opentelemetry::metrics::MeterProvider as _; use opentelemetry_sdk::{ @@ -574,7 +574,7 @@ mod tests { testing::metrics::InMemoryMetricsExporter, }; use std::{collections::HashMap, fs}; - use testcontainers::RunnableImage; + use testcontainers::{core::Mount, runners::AsyncRunner, RunnableImage}; use tokio::task::spawn_blocking; use tracing_subscriber::{reload, EnvFilter}; use trillium::Status; @@ -654,8 +654,8 @@ mod tests { #[tokio::test] async fn postgres_tls_connection() { install_test_trace_subscriber(); + initialize_rustls(); - let client = container_client(); // We need to be careful about providing the certificate and private key to the Postgres // container. The key must have '-rw-------' permissions, and both must be readable by the // postgres user, which has UID 70 inside the container at time of writing. Merely mounting @@ -680,7 +680,7 @@ mod tests { .to_string(), ]), )) - .with_volume(( + .with_mount(Mount::bind_mount( fs::canonicalize("tests/tls_files") .unwrap() .into_os_string() @@ -688,8 +688,8 @@ mod tests { .unwrap(), "/etc/ssl/postgresql_host", )) - .with_volume((volume.name(), "/etc/ssl/postgresql")); - let setup_container = client.run(setup_image); + .with_mount(Mount::volume_mount(volume.name(), "/etc/ssl/postgresql")); + let setup_container = setup_image.start().await; drop(setup_container); let image = RunnableImage::from(( @@ -703,10 +703,10 @@ mod tests { "ssl_key_file=/etc/ssl/postgresql/127.0.0.1-key.pem".to_string(), ]), )) - .with_volume((volume.name(), "/etc/ssl/postgresql")); - let db_container = client.run(image); + .with_mount(Mount::volume_mount(volume.name(), "/etc/ssl/postgresql")); + let db_container = image.start().await; const POSTGRES_DEFAULT_PORT: u16 = 5432; - let port = db_container.get_host_port_ipv4(POSTGRES_DEFAULT_PORT); + let port = db_container.get_host_port_ipv4(POSTGRES_DEFAULT_PORT).await; let db_config = DbConfig { url: format!("postgres://postgres@127.0.0.1:{port}/postgres?sslmode=require") diff --git a/aggregator_core/src/datastore/test_util.rs b/aggregator_core/src/datastore/test_util.rs index 874f2f55d..1d1b3d9b0 100644 --- a/aggregator_core/src/datastore/test_util.rs +++ b/aggregator_core/src/datastore/test_util.rs @@ -19,21 +19,19 @@ use sqlx::{ use std::{ path::PathBuf, str::FromStr, - sync::{Arc, Barrier, Weak}, - thread::{self, JoinHandle}, + sync::{Arc, Weak}, time::Duration, }; -use testcontainers::RunnableImage; -use tokio::sync::{oneshot, Mutex}; +use testcontainers::{runners::AsyncRunner, ContainerAsync, RunnableImage}; +use tokio::sync::Mutex; use tokio_postgres::{connect, Config, NoTls}; use tracing::trace; use super::SUPPORTED_SCHEMA_VERSIONS; struct EphemeralDatabase { + _db_container: ContainerAsync, port_number: u16, - shutdown_barrier: Arc, - join_handle: Option>, } impl EphemeralDatabase { @@ -51,30 +49,15 @@ impl EphemeralDatabase { } async fn start() -> Self { - let (port_tx, port_rx) = oneshot::channel(); - let shutdown_barrier = Arc::new(Barrier::new(2)); - let join_handle = thread::spawn({ - let shutdown_barrier = Arc::clone(&shutdown_barrier); - move || { - // Start an instance of Postgres running in a container. - let container_client = testcontainers::clients::Cli::default(); - let db_container = container_client.run(RunnableImage::from(Postgres::default())); - const POSTGRES_DEFAULT_PORT: u16 = 5432; - let port_number = db_container.get_host_port_ipv4(POSTGRES_DEFAULT_PORT); - trace!("Postgres container is up with port {port_number}"); - port_tx.send(port_number).unwrap(); - - // Wait for the barrier as a shutdown signal. - shutdown_barrier.wait(); - trace!("Shutting down Postgres container with port {port_number}"); - } - }); - let port_number = port_rx.await.unwrap(); + // Start an instance of Postgres running in a container. + let db_container = RunnableImage::from(Postgres::default()).start().await; + const POSTGRES_DEFAULT_PORT: u16 = 5432; + let port_number = db_container.get_host_port_ipv4(POSTGRES_DEFAULT_PORT).await; + trace!("Postgres container is up with port {port_number}"); Self { + _db_container: db_container, port_number, - shutdown_barrier, - join_handle: Some(join_handle), } } @@ -86,17 +69,6 @@ impl EphemeralDatabase { } } -impl Drop for EphemeralDatabase { - fn drop(&mut self) { - // Wait on the shutdown barrier, which will cause the container-management thread to - // begin shutdown. Then wait for the container-management thread itself to terminate. - // This guarantees container shutdown finishes before dropping the EphemeralDatabase - // completes. - self.shutdown_barrier.wait(); - self.join_handle.take().unwrap().join().unwrap(); - } -} - /// EphemeralDatastore represents an ephemeral datastore instance. It has methods allowing /// creation of Datastores, as well as the ability to retrieve the underlying connection pool. /// diff --git a/core/src/test_util/testcontainers.rs b/core/src/test_util/testcontainers.rs index 29b1c7568..7f9add493 100644 --- a/core/src/test_util/testcontainers.rs +++ b/core/src/test_util/testcontainers.rs @@ -1,23 +1,7 @@ //! Testing functionality that interacts with the testcontainers library. -use std::{ - collections::HashMap, - process::Command, - sync::{Arc, Mutex, Weak}, -}; -use testcontainers::{clients::Cli, core::WaitFor, Image}; - -/// Returns a container client, possibly shared with other callers to this function. -pub fn container_client() -> Arc { - static CONTAINER_CLIENT_MU: Mutex> = Mutex::new(Weak::new()); - - let mut container_client = CONTAINER_CLIENT_MU.lock().unwrap(); - container_client.upgrade().unwrap_or_else(|| { - let client = Arc::new(Cli::default()); - *container_client = Arc::downgrade(&client); - client - }) -} +use std::{collections::HashMap, process::Command}; +use testcontainers::{core::WaitFor, Image}; /// A [`testcontainers::Image`] that provides a Postgres server. #[derive(Debug)] diff --git a/deny.toml b/deny.toml index 4950c7535..9942fd4fc 100644 --- a/deny.toml +++ b/deny.toml @@ -34,6 +34,7 @@ allow = [ "Unicode-DFS-2016", "OpenSSL", "Unlicense", + "CC0-1.0", ] [[licenses.clarify]] diff --git a/integration_tests/src/client.rs b/integration_tests/src/client.rs index 9fd586400..36fc7731e 100644 --- a/integration_tests/src/client.rs +++ b/integration_tests/src/client.rs @@ -17,7 +17,7 @@ use prio::{ use rand::random; use serde_json::{json, Value}; use std::env; -use testcontainers::{clients::Cli, core::WaitFor, Image, RunnableImage}; +use testcontainers::{core::WaitFor, runners::AsyncRunner, Image, RunnableImage}; use url::Url; /// Extension trait to encode measurements for VDAFs as JSON objects, according to @@ -182,7 +182,6 @@ pub enum ClientBackend<'a> { /// Uploads reports by starting a containerized client implementation, and sending it requests /// using draft-dcook-ppm-dap-interop-test-design. Container { - container_client: &'a Cli, container_image: InteropClient, network: &'a str, }, @@ -195,7 +194,7 @@ impl<'a> ClientBackend<'a> { task_parameters: &TaskParameters, (leader_port, helper_port): (u16, u16), vdaf: V, - ) -> anyhow::Result> + ) -> anyhow::Result> where V: vdaf::Client<16> + InteropClientEncoding, { @@ -208,26 +207,25 @@ impl<'a> ClientBackend<'a> { .await .map_err(Into::into), ClientBackend::Container { - container_client, container_image, network, } => Ok(ClientImplementation::new_container( test_name, - container_client, container_image.clone(), network, task_parameters, vdaf, - )), + ) + .await), } } } -pub struct ContainerClientImplementation<'d, V> +pub struct ContainerClientImplementation where V: vdaf::Client<16>, { - _container: ContainerLogsDropGuard<'d, InteropClient>, + _container: ContainerLogsDropGuard, leader: Url, helper: Url, task_id: TaskId, @@ -240,15 +238,15 @@ where /// A DAP client implementation, specialized to work with a particular VDAF. See also /// [`ClientBackend`]. -pub enum ClientImplementation<'d, V> +pub enum ClientImplementation where V: vdaf::Client<16>, { InProcess { client: Client }, - Container(Box>), + Container(Box>), } -impl<'d, V> ClientImplementation<'d, V> +impl ClientImplementation where V: vdaf::Client<16> + InteropClientEncoding, { @@ -256,7 +254,7 @@ where task_parameters: &TaskParameters, (leader_port, helper_port): (u16, u16), vdaf: V, - ) -> Result, janus_client::Error> { + ) -> Result, janus_client::Error> { let (leader_aggregator_endpoint, helper_aggregator_endpoint) = task_parameters .endpoint_fragments .endpoints_for_host_client(leader_port, helper_port); @@ -271,9 +269,8 @@ where Ok(ClientImplementation::InProcess { client }) } - pub fn new_container( + pub async fn new_container( test_name: &str, - container_client: &'d Cli, container_image: InteropClient, network: &str, task_parameters: &TaskParameters, @@ -283,14 +280,14 @@ where let client_container_name = format!("client-{random_part}"); let container = ContainerLogsDropGuard::new_janus( test_name, - container_client.run( - RunnableImage::from(container_image) - .with_network(network) - .with_env_var(get_rust_log_level()) - .with_container_name(client_container_name), - ), + RunnableImage::from(container_image) + .with_network(network) + .with_env_var(get_rust_log_level()) + .with_container_name(client_container_name) + .start() + .await, ); - let host_port = container.get_host_port_ipv4(8080); + let host_port = container.get_host_port_ipv4(8080).await; let http_client = reqwest::Client::new(); let (leader_aggregator_endpoint, helper_aggregator_endpoint) = task_parameters .endpoint_fragments diff --git a/integration_tests/src/daphne.rs b/integration_tests/src/daphne.rs index 2f1237413..e169ea3ad 100644 --- a/integration_tests/src/daphne.rs +++ b/integration_tests/src/daphne.rs @@ -7,29 +7,29 @@ use janus_interop_binaries::{ }; use janus_messages::{Role, Time}; use serde_json::json; -use testcontainers::{clients::Cli, GenericImage, RunnableImage}; +use testcontainers::{runners::AsyncRunner, GenericImage, RunnableImage}; use url::Url; const DAPHNE_HELPER_IMAGE_NAME_AND_TAG: &str = "cloudflare/daphne-worker-helper:sha-f6b3ef1"; /// Represents a running Daphne test instance. -pub struct Daphne<'a> { - daphne_container: Option>, +pub struct Daphne { + _daphne_container: Option>, + port: u16, } -impl<'a> Daphne<'a> { +impl Daphne { const INTERNAL_SERVING_PORT: u16 = 8788; /// Create and start a new hermetic Daphne test instance in the given Docker network, configured /// to service the given task. The aggregator port is also exposed to the host. pub async fn new( test_name: &str, - container_client: &'a Cli, network: &str, task: &Task, role: Role, start_container: bool, - ) -> Daphne<'a> { + ) -> Daphne { let (endpoint, image_name_and_tag) = match role { Role::Leader => panic!("A leader container image for Daphne is not yet available"), Role::Helper => ( @@ -51,10 +51,12 @@ impl<'a> Daphne<'a> { .with_container_name(endpoint.host_str().unwrap()); let daphne_container = ContainerLogsDropGuard::new( test_name, - container_client.run(runnable_image), + runnable_image.start().await, ContainerLogsSource::Path("/logs".into()), ); - let port = daphne_container.get_host_port_ipv4(Self::INTERNAL_SERVING_PORT); + let port = daphne_container + .get_host_port_ipv4(Self::INTERNAL_SERVING_PORT) + .await; (port, Some(daphne_container)) } else { (Self::INTERNAL_SERVING_PORT, None) @@ -123,15 +125,14 @@ impl<'a> Daphne<'a> { // Write the given task to the Daphne instance we started. interop_api::aggregator_add_task(port, task, role).await; - Self { daphne_container } + Self { + _daphne_container: daphne_container, + port, + } } /// Returns the port of the aggregator on the host. pub fn port(&self) -> u16 { - self.daphne_container - .as_ref() - .map_or(Self::INTERNAL_SERVING_PORT, |container| { - container.get_host_port_ipv4(Self::INTERNAL_SERVING_PORT) - }) + self.port } } diff --git a/integration_tests/src/janus.rs b/integration_tests/src/janus.rs index 0f3c48cbf..23d5cd34a 100644 --- a/integration_tests/src/janus.rs +++ b/integration_tests/src/janus.rs @@ -37,26 +37,21 @@ use janus_interop_binaries::{ use janus_messages::Role; use std::net::{Ipv4Addr, SocketAddr}; #[cfg(feature = "testcontainer")] -use testcontainers::{clients::Cli, RunnableImage}; +use testcontainers::{runners::AsyncRunner, RunnableImage}; use trillium_tokio::Stopper; /// Represents a running Janus test instance in a container. #[cfg(feature = "testcontainer")] -pub struct JanusContainer<'a> { - container: ContainerLogsDropGuard<'a, Aggregator>, +pub struct JanusContainer { + _container: ContainerLogsDropGuard, + port: u16, } #[cfg(feature = "testcontainer")] -impl<'a> JanusContainer<'a> { +impl JanusContainer { /// Create and start a new hermetic Janus test instance in the given Docker network, configured /// to service the given task. The aggregator port is also exposed to the host. - pub async fn new( - test_name: &str, - container_client: &'a Cli, - network: &str, - task: &Task, - role: Role, - ) -> JanusContainer<'a> { + pub async fn new(test_name: &str, network: &str, task: &Task, role: Role) -> JanusContainer { // Start the Janus interop aggregator container running. let endpoint = match role { Role::Leader => task.leader_aggregator_endpoint(), @@ -65,14 +60,16 @@ impl<'a> JanusContainer<'a> { }; let container = ContainerLogsDropGuard::new_janus( test_name, - container_client.run( - RunnableImage::from(Aggregator::default()) - .with_network(network) - .with_env_var(get_rust_log_level()) - .with_container_name(endpoint.host_str().unwrap()), - ), + RunnableImage::from(Aggregator::default()) + .with_network(network) + .with_env_var(get_rust_log_level()) + .with_container_name(endpoint.host_str().unwrap()) + .start() + .await, ); - let port = container.get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT); + let port = container + .get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT) + .await; // Wait for the container to start listening on its port. await_http_server(port).await; @@ -80,13 +77,15 @@ impl<'a> JanusContainer<'a> { // Write the given task to the Janus instance we started. interop_api::aggregator_add_task(port, task.clone(), role).await; - Self { container } + Self { + _container: container, + port, + } } /// Returns the port of the aggregator on the host. pub fn port(&self) -> u16 { - self.container - .get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT) + self.port } } diff --git a/integration_tests/tests/integration/common.rs b/integration_tests/tests/integration/common.rs index b29307ab8..015e4b3cf 100644 --- a/integration_tests/tests/integration/common.rs +++ b/integration_tests/tests/integration/common.rs @@ -173,7 +173,7 @@ pub async fn submit_measurements_and_verify_aggregate_generic( leader_port: u16, vdaf: V, test_case: &AggregationTestCase, - client_implementation: &ClientImplementation<'_, V>, + client_implementation: &ClientImplementation, ) where V: vdaf::Client<16> + vdaf::Collector + InteropClientEncoding, V::AggregateResult: PartialEq, @@ -193,7 +193,7 @@ pub async fn submit_measurements_and_verify_aggregate_generic( pub async fn submit_measurements_generic( measurements: &[V::Measurement], - client_implementation: &ClientImplementation<'_, V>, + client_implementation: &ClientImplementation, ) -> Time where V: vdaf::Client<16> + vdaf::Collector + InteropClientEncoding, diff --git a/integration_tests/tests/integration/daphne.rs b/integration_tests/tests/integration/daphne.rs index 75ddd1497..48724921f 100644 --- a/integration_tests/tests/integration/daphne.rs +++ b/integration_tests/tests/integration/daphne.rs @@ -3,10 +3,7 @@ use crate::{ initialize_rustls, }; use janus_aggregator_core::task::{test_util::TaskBuilder, QueryType}; -use janus_core::{ - test_util::{install_test_trace_subscriber, testcontainers::container_client}, - vdaf::VdafInstance, -}; +use janus_core::{test_util::install_test_trace_subscriber, vdaf::VdafInstance}; #[cfg(feature = "testcontainer")] use janus_integration_tests::janus::JanusContainer; use janus_integration_tests::{ @@ -47,18 +44,8 @@ async fn daphne_janus() { .with_leader_aggregator_endpoint(leader_aggregator_endpoint) .build(); - let container_client = container_client(); - let leader = Daphne::new( - TEST_NAME, - &container_client, - &network, - &task, - Role::Leader, - true, - ) - .await; - let helper = - JanusContainer::new(TEST_NAME, &container_client, &network, &task, Role::Helper).await; + let leader = Daphne::new(TEST_NAME, &network, &task, Role::Leader, true).await; + let helper = JanusContainer::new(TEST_NAME, &network, &task, Role::Helper).await; // Run the behavioral test. submit_measurements_and_verify_aggregate( @@ -99,18 +86,8 @@ async fn janus_daphne() { .with_helper_aggregator_endpoint(helper_aggregator_endpoint) .build(); - let container_client = container_client(); - let leader = - JanusContainer::new(TEST_NAME, &container_client, &network, &task, Role::Leader).await; - let helper = Daphne::new( - TEST_NAME, - &container_client, - &network, - &task, - Role::Helper, - true, - ) - .await; + let leader = JanusContainer::new(TEST_NAME, &network, &task, Role::Leader).await; + let helper = Daphne::new(TEST_NAME, &network, &task, Role::Helper, true).await; // Run the behavioral test. submit_measurements_and_verify_aggregate( @@ -133,7 +110,6 @@ async fn janus_in_process_daphne() { // Start servers. let network = generate_network_name(); - let container_client = container_client(); let (mut task_parameters, mut task_builder) = build_test_task( TaskBuilder::new(QueryType::TimeInterval, VdafInstance::Prio3Count), TestContext::VirtualNetwork, @@ -149,7 +125,6 @@ async fn janus_in_process_daphne() { .set_path(VERSION_PATH.to_owned()); let helper = Daphne::new( TEST_NAME, - &container_client, &network, &task_builder.clone().build(), Role::Helper, diff --git a/integration_tests/tests/integration/divviup_ts.rs b/integration_tests/tests/integration/divviup_ts.rs index 01b52ed92..8aac8b0db 100644 --- a/integration_tests/tests/integration/divviup_ts.rs +++ b/integration_tests/tests/integration/divviup_ts.rs @@ -6,10 +6,7 @@ use crate::{ initialize_rustls, }; use janus_aggregator_core::task::{test_util::TaskBuilder, QueryType}; -use janus_core::{ - test_util::{install_test_trace_subscriber, testcontainers::container_client}, - vdaf::VdafInstance, -}; +use janus_core::{test_util::install_test_trace_subscriber, vdaf::VdafInstance}; use janus_integration_tests::{ client::{ClientBackend, InteropClient}, janus::JanusContainer, @@ -17,13 +14,8 @@ use janus_integration_tests::{ use janus_interop_binaries::test_util::generate_network_name; use janus_messages::Role; use std::time::Duration; -use testcontainers::clients::Cli; -async fn run_divviup_ts_integration_test( - test_name: &str, - container_client: &Cli, - vdaf: VdafInstance, -) { +async fn run_divviup_ts_integration_test(test_name: &str, vdaf: VdafInstance) { let (task_parameters, task_builder) = build_test_task( TaskBuilder::new(QueryType::TimeInterval, vdaf), TestContext::VirtualNetwork, @@ -32,13 +24,10 @@ async fn run_divviup_ts_integration_test( ); let task = task_builder.build(); let network = generate_network_name(); - let leader = - JanusContainer::new(test_name, container_client, &network, &task, Role::Leader).await; - let helper = - JanusContainer::new(test_name, container_client, &network, &task, Role::Helper).await; + let leader = JanusContainer::new(test_name, &network, &task, Role::Leader).await; + let helper = JanusContainer::new(test_name, &network, &task, Role::Helper).await; let client_backend = ClientBackend::Container { - container_client, container_image: InteropClient::divviup_ts(), network: &network, }; @@ -57,12 +46,7 @@ async fn janus_divviup_ts_count() { install_test_trace_subscriber(); initialize_rustls(); - run_divviup_ts_integration_test( - "janus_divviup_ts_count", - &container_client(), - VdafInstance::Prio3Count, - ) - .await; + run_divviup_ts_integration_test("janus_divviup_ts_count", VdafInstance::Prio3Count).await; } #[tokio::test(flavor = "multi_thread")] @@ -71,12 +55,8 @@ async fn janus_divviup_ts_sum() { install_test_trace_subscriber(); initialize_rustls(); - run_divviup_ts_integration_test( - "janus_divviup_ts_sum", - &container_client(), - VdafInstance::Prio3Sum { bits: 8 }, - ) - .await; + run_divviup_ts_integration_test("janus_divviup_ts_sum", VdafInstance::Prio3Sum { bits: 8 }) + .await; } #[tokio::test(flavor = "multi_thread")] @@ -87,7 +67,6 @@ async fn janus_divviup_ts_histogram() { run_divviup_ts_integration_test( "janus_divviup_ts_histogram", - &container_client(), VdafInstance::Prio3Histogram { length: 4, chunk_length: 2, @@ -104,7 +83,6 @@ async fn janus_divviup_ts_sumvec() { run_divviup_ts_integration_test( "janus_divviup_ts_sumvec", - &container_client(), VdafInstance::Prio3SumVec { bits: 16, length: 15, diff --git a/integration_tests/tests/integration/janus.rs b/integration_tests/tests/integration/janus.rs index 50498c982..806744826 100644 --- a/integration_tests/tests/integration/janus.rs +++ b/integration_tests/tests/integration/janus.rs @@ -6,8 +6,6 @@ use crate::{ initialize_rustls, }; use janus_aggregator_core::task::{test_util::TaskBuilder, QueryType}; -#[cfg(feature = "testcontainer")] -use janus_core::test_util::testcontainers::container_client; use janus_core::{test_util::install_test_trace_subscriber, vdaf::VdafInstance}; #[cfg(feature = "testcontainer")] use janus_integration_tests::janus::JanusContainer; @@ -17,32 +15,29 @@ use janus_interop_binaries::test_util::generate_network_name; use janus_messages::Role; use prio::vdaf::dummy; use std::time::Duration; -#[cfg(feature = "testcontainer")] -use testcontainers::clients::Cli; /// A pair of Janus instances, running in containers, against which integration tests may be run. #[cfg(feature = "testcontainer")] -struct JanusContainerPair<'a> { +struct JanusContainerPair { /// Task parameters needed by the client and collector, for the task configured in both Janus /// aggregators. task_parameters: TaskParameters, /// Handle to the leader's resources, which are released on drop. - leader: JanusContainer<'a>, + leader: JanusContainer, /// Handle to the helper's resources, which are released on drop. - helper: JanusContainer<'a>, + helper: JanusContainer, } #[cfg(feature = "testcontainer")] -impl<'a> JanusContainerPair<'a> { +impl JanusContainerPair { /// Set up a new pair of containerized Janus test instances, and set up a new task in each using /// the given VDAF and query type. pub async fn new( test_name: &str, - container_client: &'a Cli, vdaf: VdafInstance, query_type: QueryType, - ) -> JanusContainerPair<'a> { + ) -> JanusContainerPair { let (task_parameters, task_builder) = build_test_task( TaskBuilder::new(query_type, vdaf), TestContext::VirtualNetwork, @@ -52,10 +47,8 @@ impl<'a> JanusContainerPair<'a> { let task = task_builder.build(); let network = generate_network_name(); - let leader = - JanusContainer::new(test_name, container_client, &network, &task, Role::Leader).await; - let helper = - JanusContainer::new(test_name, container_client, &network, &task, Role::Helper).await; + let leader = JanusContainer::new(test_name, &network, &task, Role::Leader).await; + let helper = JanusContainer::new(test_name, &network, &task, Role::Helper).await; Self { task_parameters, @@ -113,14 +106,8 @@ async fn janus_janus_count() { initialize_rustls(); // Start servers. - let container_client = container_client(); - let janus_pair = JanusContainerPair::new( - TEST_NAME, - &container_client, - VdafInstance::Prio3Count, - QueryType::TimeInterval, - ) - .await; + let janus_pair = + JanusContainerPair::new(TEST_NAME, VdafInstance::Prio3Count, QueryType::TimeInterval).await; // Run the behavioral test. submit_measurements_and_verify_aggregate( @@ -164,10 +151,8 @@ async fn janus_janus_sum_16() { initialize_rustls(); // Start servers. - let container_client = container_client(); let janus_pair = JanusContainerPair::new( TEST_NAME, - &container_client, VdafInstance::Prio3Sum { bits: 16 }, QueryType::TimeInterval, ) @@ -215,10 +200,8 @@ async fn janus_janus_histogram_4_buckets() { initialize_rustls(); // Start servers. - let container_client = container_client(); let janus_pair = JanusContainerPair::new( TEST_NAME, - &container_client, VdafInstance::Prio3Histogram { length: 4, chunk_length: 2, @@ -272,10 +255,8 @@ async fn janus_janus_fixed_size() { initialize_rustls(); // Start servers. - let container_client = container_client(); let janus_pair = JanusContainerPair::new( TEST_NAME, - &container_client, VdafInstance::Prio3Count, QueryType::FixedSize { max_batch_size: Some(50), @@ -328,10 +309,8 @@ async fn janus_janus_sum_vec() { install_test_trace_subscriber(); initialize_rustls(); - let container_client = container_client(); let janus_pair = JanusContainerPair::new( TEST_NAME, - &container_client, VdafInstance::Prio3SumVec { bits: 16, length: 15, diff --git a/interop_binaries/src/lib.rs b/interop_binaries/src/lib.rs index e3a845095..5361e3358 100644 --- a/interop_binaries/src/lib.rs +++ b/interop_binaries/src/lib.rs @@ -27,7 +27,7 @@ use std::{ str::FromStr, sync::Arc, }; -use testcontainers::{Container, Image}; +use testcontainers::{ContainerAsync, Image}; use tokio::sync::Mutex; use tracing_log::LogTracer; use tracing_subscriber::{prelude::*, EnvFilter, Registry}; @@ -416,9 +416,9 @@ struct ContainerInspectEntry { name: String, } -pub struct ContainerLogsDropGuard<'d, I: Image> { +pub struct ContainerLogsDropGuard { test_name: String, - container: Container<'d, I>, + container: ContainerAsync, source: ContainerLogsSource, } @@ -429,12 +429,12 @@ pub enum ContainerLogsSource { Path(String), } -impl<'d, I: Image> ContainerLogsDropGuard<'d, I> { +impl ContainerLogsDropGuard { pub fn new( test_name: &str, - container: Container<'d, I>, + container: ContainerAsync, source: ContainerLogsSource, - ) -> ContainerLogsDropGuard<'d, I> { + ) -> ContainerLogsDropGuard { ContainerLogsDropGuard { test_name: test_name.into(), container, @@ -442,10 +442,7 @@ impl<'d, I: Image> ContainerLogsDropGuard<'d, I> { } } - pub fn new_janus( - test_name: &str, - container: Container<'d, I>, - ) -> ContainerLogsDropGuard<'d, I> { + pub fn new_janus(test_name: &str, container: ContainerAsync) -> ContainerLogsDropGuard { ContainerLogsDropGuard { test_name: test_name.into(), container, @@ -454,7 +451,7 @@ impl<'d, I: Image> ContainerLogsDropGuard<'d, I> { } } -impl<'d, I: Image> Drop for ContainerLogsDropGuard<'d, I> { +impl Drop for ContainerLogsDropGuard { fn drop(&mut self) { // The unwraps in this code block would induce a double panic, but we accept this risk // since it happens only in test code. This is also our main method of debugging @@ -510,8 +507,8 @@ impl<'d, I: Image> Drop for ContainerLogsDropGuard<'d, I> { } } -impl<'d, I: Image> Deref for ContainerLogsDropGuard<'d, I> { - type Target = Container<'d, I>; +impl Deref for ContainerLogsDropGuard { + type Target = ContainerAsync; fn deref(&self) -> &Self::Target { &self.container diff --git a/interop_binaries/tests/end_to_end.rs b/interop_binaries/tests/end_to_end.rs index f112828b8..e1ec71e34 100644 --- a/interop_binaries/tests/end_to_end.rs +++ b/interop_binaries/tests/end_to_end.rs @@ -4,7 +4,7 @@ use backoff::{backoff::Backoff, ExponentialBackoffBuilder}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use futures::future::join_all; use janus_core::{ - test_util::{install_test_trace_subscriber, testcontainers::container_client}, + test_util::install_test_trace_subscriber, time::{Clock, RealClock, TimeExt}, vdaf::VERIFY_KEY_LENGTH, }; @@ -23,7 +23,7 @@ use rand::random; use reqwest::{header::CONTENT_TYPE, StatusCode, Url}; use serde_json::{json, Value}; use std::time::Duration as StdDuration; -use testcontainers::RunnableImage; +use testcontainers::{runners::AsyncRunner, RunnableImage}; use tokio::time::sleep; #[cfg(feature = "fpvec_bounded_l2")] @@ -61,54 +61,61 @@ async fn run( }; // Create and start containers. - let container_client = container_client(); let network = generate_network_name(); let client_container = ContainerLogsDropGuard::new_janus( test_name, - container_client.run( - RunnableImage::from(Client::default()) - .with_network(&network) - .with_env_var(get_rust_log_level()) - .with_container_name(generate_unique_name("client")), - ), + RunnableImage::from(Client::default()) + .with_network(&network) + .with_env_var(get_rust_log_level()) + .with_container_name(generate_unique_name("client")) + .start() + .await, ); - let client_port = client_container.get_host_port_ipv4(Client::INTERNAL_SERVING_PORT); + let client_port = client_container + .get_host_port_ipv4(Client::INTERNAL_SERVING_PORT) + .await; let leader_name = generate_unique_name("leader"); let leader_container = ContainerLogsDropGuard::new_janus( test_name, - container_client.run( - RunnableImage::from(Aggregator::default()) - .with_network(&network) - .with_env_var(get_rust_log_level()) - .with_container_name(leader_name.clone()), - ), + RunnableImage::from(Aggregator::default()) + .with_network(&network) + .with_env_var(get_rust_log_level()) + .with_container_name(leader_name.clone()) + .start() + .await, ); - let leader_port = leader_container.get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT); + let leader_port = leader_container + .get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT) + .await; let helper_name = generate_unique_name("helper"); let helper_container = ContainerLogsDropGuard::new_janus( test_name, - container_client.run( - RunnableImage::from(Aggregator::default()) - .with_network(&network) - .with_env_var(get_rust_log_level()) - .with_container_name(helper_name.clone()), - ), + RunnableImage::from(Aggregator::default()) + .with_network(&network) + .with_env_var(get_rust_log_level()) + .with_container_name(helper_name.clone()) + .start() + .await, ); - let helper_port = helper_container.get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT); + let helper_port = helper_container + .get_host_port_ipv4(Aggregator::INTERNAL_SERVING_PORT) + .await; let collector_container = ContainerLogsDropGuard::new_janus( test_name, - container_client.run( - RunnableImage::from(Collector::default()) - .with_network(&network) - .with_env_var(get_rust_log_level()) - .with_container_name(generate_unique_name("collector")), - ), + RunnableImage::from(Collector::default()) + .with_network(&network) + .with_env_var(get_rust_log_level()) + .with_container_name(generate_unique_name("collector")) + .start() + .await, ); - let collector_port = collector_container.get_host_port_ipv4(Collector::INTERNAL_SERVING_PORT); + let collector_port = collector_container + .get_host_port_ipv4(Collector::INTERNAL_SERVING_PORT) + .await; // Wait for all containers to sucessfully respond to HTTP requests. join_all(