diff --git a/compiler/base/orchestrator/src/coordinator.rs b/compiler/base/orchestrator/src/coordinator.rs index 2d58a0c10..2e4cf73f1 100644 --- a/compiler/base/orchestrator/src/coordinator.rs +++ b/compiler/base/orchestrator/src/coordinator.rs @@ -35,6 +35,134 @@ use crate::{ DropErrorDetailsExt, }; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Versions { + pub stable: ChannelVersions, + pub beta: ChannelVersions, + pub nightly: ChannelVersions, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChannelVersions { + pub rustc: Version, + pub rustfmt: Version, + pub clippy: Version, + pub miri: Option, +} + +/// Parsing this struct is very lenient — we'd rather return some +/// partial data instead of absolutely nothing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Version { + pub release: String, + pub commit_hash: String, + pub commit_date: String, +} + +impl Version { + fn parse_rustc_version_verbose(rustc_version: &str) -> Self { + let mut release = ""; + let mut commit_hash = ""; + let mut commit_date = ""; + + let fields = rustc_version.lines().skip(1).filter_map(|line| { + let mut pieces = line.splitn(2, ':'); + let key = pieces.next()?.trim(); + let value = pieces.next()?.trim(); + Some((key, value)) + }); + + for (k, v) in fields { + match k { + "release" => release = v, + "commit-hash" => commit_hash = v, + "commit-date" => commit_date = v, + _ => {} + } + } + + Self { + release: release.into(), + commit_hash: commit_hash.into(), + commit_date: commit_date.into(), + } + } + + // Parses versions of the shape `toolname 0.0.0 (0000000 0000-00-00)` + fn parse_tool_version(tool_version: &str) -> Self { + let mut parts = tool_version.split_whitespace().fuse().skip(1); + + let release = parts.next().unwrap_or("").into(); + let commit_hash = parts.next().unwrap_or("").trim_start_matches('(').into(); + let commit_date = parts.next().unwrap_or("").trim_end_matches(')').into(); + + Self { + release, + commit_hash, + commit_date, + } + } +} + +#[derive(Debug, Snafu)] +#[snafu(module)] +pub enum VersionsError { + #[snafu(display("Unable to determine versions for the stable channel"))] + Stable { source: VersionsChannelError }, + + #[snafu(display("Unable to determine versions for the beta channel"))] + Beta { source: VersionsChannelError }, + + #[snafu(display("Unable to determine versions for the nightly channel"))] + Nightly { source: VersionsChannelError }, +} + +#[derive(Debug, Snafu)] +pub enum VersionsChannelError { + #[snafu(context(false))] // transparent + Channel { source: Error }, + + #[snafu(context(false))] // transparent + Versions { source: ContainerVersionsError }, +} + +#[derive(Debug, Snafu)] +#[snafu(module)] +pub enum ContainerVersionsError { + #[snafu(display("Failed to get `rustc` version"))] + Rustc { source: VersionError }, + + #[snafu(display("`rustc` not executable"))] + RustcMissing, + + #[snafu(display("Failed to get `rustfmt` version"))] + Rustfmt { source: VersionError }, + + #[snafu(display("`cargo fmt` not executable"))] + RustfmtMissing, + + #[snafu(display("Failed to get clippy version"))] + Clippy { source: VersionError }, + + #[snafu(display("`cargo clippy` not executable"))] + ClippyMissing, + + #[snafu(display("Failed to get miri version"))] + Miri { source: VersionError }, +} + +#[derive(Debug, Snafu)] +#[snafu(module)] +pub enum VersionError { + #[snafu(display("Could not start the process"))] + #[snafu(context(false))] + SpawnProcess { source: SpawnCargoError }, + + #[snafu(display("The task panicked"))] + #[snafu(context(false))] + TaskPanic { source: tokio::task::JoinError }, +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum AssemblyFlavor { Att, @@ -639,6 +767,28 @@ where } } + pub async fn versions(&self) -> Result { + use versions_error::*; + + let [stable, beta, nightly] = + [Channel::Stable, Channel::Beta, Channel::Nightly].map(|c| async move { + let c = self.select_channel(c).await?; + c.versions().await.map_err(VersionsChannelError::from) + }); + + let (stable, beta, nightly) = join!(stable, beta, nightly); + + let stable = stable.context(StableSnafu)?; + let beta = beta.context(BetaSnafu)?; + let nightly = nightly.context(NightlySnafu)?; + + Ok(Versions { + stable, + beta, + nightly, + }) + } + pub async fn execute( &self, request: ExecuteRequest, @@ -904,6 +1054,72 @@ impl Container { }) } + async fn versions(&self) -> Result { + use container_versions_error::*; + + let token = CancellationToken::new(); + + let rustc = self.rustc_version(token.clone()); + let rustfmt = self.tool_version(token.clone(), "fmt"); + let clippy = self.tool_version(token.clone(), "clippy"); + let miri = self.tool_version(token, "miri"); + + let (rustc, rustfmt, clippy, miri) = join!(rustc, rustfmt, clippy, miri); + + let rustc = rustc.context(RustcSnafu)?.context(RustcMissingSnafu)?; + let rustfmt = rustfmt + .context(RustfmtSnafu)? + .context(RustfmtMissingSnafu)?; + let clippy = clippy.context(ClippySnafu)?.context(ClippyMissingSnafu)?; + let miri = miri.context(MiriSnafu)?; + + Ok(ChannelVersions { + rustc, + rustfmt, + clippy, + miri, + }) + } + + async fn rustc_version( + &self, + token: CancellationToken, + ) -> Result, VersionError> { + let rustc_cmd = ExecuteCommandRequest::simple("rustc", ["--version", "--verbose"]); + let output = self.version_output(token, rustc_cmd).await?; + + Ok(output.map(|o| Version::parse_rustc_version_verbose(&o))) + } + + async fn tool_version( + &self, + token: CancellationToken, + subcommand_name: &str, + ) -> Result, VersionError> { + let tool_cmd = ExecuteCommandRequest::simple("cargo", [subcommand_name, "--version"]); + let output = self.version_output(token, tool_cmd).await?; + + Ok(output.map(|o| Version::parse_tool_version(&o))) + } + + async fn version_output( + &self, + token: CancellationToken, + cmd: ExecuteCommandRequest, + ) -> Result, VersionError> { + let v = self.spawn_cargo_task(token.clone(), cmd).await?; + let SpawnCargo { + task, + stdin_tx, + stdout_rx, + stderr_rx, + } = v; + drop(stdin_tx); + let task = async { task.await?.map_err(VersionError::from) }; + let o = WithOutput::try_absorb(task, stdout_rx, stderr_rx).await?; + Ok(if o.success { Some(o.stdout) } else { None }) + } + async fn execute( &self, request: ExecuteRequest, @@ -2416,6 +2632,20 @@ mod tests { } } + #[tokio::test] + #[snafu::report] + async fn versions() -> Result<()> { + let coordinator = new_coordinator().await; + + let versions = coordinator.versions().with_timeout().await.unwrap(); + + assert_starts_with!(versions.stable.rustc.release, "1."); + + coordinator.shutdown().await?; + + Ok(()) + } + const ARBITRARY_EXECUTE_REQUEST: ExecuteRequest = ExecuteRequest { channel: Channel::Stable, mode: Mode::Debug, diff --git a/compiler/base/orchestrator/src/message.rs b/compiler/base/orchestrator/src/message.rs index 36fd2227c..fe728d5b4 100644 --- a/compiler/base/orchestrator/src/message.rs +++ b/compiler/base/orchestrator/src/message.rs @@ -116,6 +116,20 @@ pub struct ExecuteCommandRequest { pub cwd: Option, // None means in project direcotry. } +impl ExecuteCommandRequest { + pub fn simple( + cmd: impl Into, + args: impl IntoIterator>, + ) -> Self { + Self { + cmd: cmd.into(), + args: args.into_iter().map(Into::into).collect(), + envs: Default::default(), + cwd: None, + } + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct ExecuteCommandResponse { pub success: bool, diff --git a/ui/src/main.rs b/ui/src/main.rs index 51f988061..1fbcf1d52 100644 --- a/ui/src/main.rs +++ b/ui/src/main.rs @@ -219,6 +219,14 @@ enum Error { #[snafu(display("The WebSocket worker panicked: {}", text))] WebSocketTaskPanic { text: String }, + #[snafu(display("Unable to find the available versions"))] + Versions { + source: orchestrator::coordinator::VersionsError, + }, + + #[snafu(display("The Miri version was missing"))] + MiriVersion, + #[snafu(display("Unable to shutdown the coordinator"))] ShutdownCoordinator { source: orchestrator::coordinator::Error, @@ -471,16 +479,6 @@ impl From> for MetaCratesResponse { } } -impl From for MetaVersionResponse { - fn from(me: sandbox::Version) -> Self { - MetaVersionResponse { - version: me.release.into(), - hash: me.commit_hash.into(), - date: me.commit_date.into(), - } - } -} - impl From for MetaGistResponse { fn from(me: gist::Gist) -> Self { MetaGistResponse { diff --git a/ui/src/metrics.rs b/ui/src/metrics.rs index 214c39b12..e1da4f64c 100644 --- a/ui/src/metrics.rs +++ b/ui/src/metrics.rs @@ -227,12 +227,6 @@ impl SuccessDetails for Vec { } } -impl SuccessDetails for sandbox::Version { - fn success_details(&self) -> Outcome { - Outcome::Success - } -} - pub(crate) async fn track_metric_no_request_async( endpoint: Endpoint, body: B, diff --git a/ui/src/sandbox.rs b/ui/src/sandbox.rs index d7804e068..27539aa85 100644 --- a/ui/src/sandbox.rs +++ b/ui/src/sandbox.rs @@ -1,6 +1,6 @@ use serde_derive::Deserialize; use snafu::prelude::*; -use std::{collections::BTreeMap, io, string, time::Duration}; +use std::{io, time::Duration}; use tempfile::TempDir; use tokio::{process::Command, time}; @@ -28,13 +28,6 @@ impl From for CrateInformation { } } -#[derive(Debug, Clone)] -pub struct Version { - pub release: String, - pub commit_hash: String, - pub commit_date: String, -} - #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Unable to create temporary directory: {}", source))] @@ -58,22 +51,10 @@ pub enum Error { #[snafu(display("Unable to read crate information: {}", source))] UnableToParseCrateInformation { source: ::serde_json::Error }, - #[snafu(display("Output was not valid UTF-8: {}", source))] - OutputNotUtf8 { source: string::FromUtf8Error }, - #[snafu(display("Release was missing from the version output"))] - VersionReleaseMissing, - #[snafu(display("Commit hash was missing from the version output"))] - VersionHashMissing, - #[snafu(display("Commit date was missing from the version output"))] - VersionDateMissing, } pub type Result = ::std::result::Result; -fn vec_to_str(v: Vec) -> Result { - String::from_utf8(v).context(OutputNotUtf8Snafu) -} - macro_rules! docker_command { ($($arg:expr),* $(,)?) => ({ let mut cmd = Command::new("docker"); @@ -151,76 +132,6 @@ impl Sandbox { Ok(crates) } - - pub async fn version(&self, channel: Channel) -> Result { - let mut command = basic_secure_docker_command(); - command.args(&[channel.container_name()]); - command.args(&["rustc", "--version", "--verbose"]); - - let output = run_command_with_timeout(command).await?; - let version_output = vec_to_str(output.stdout)?; - - let mut info: BTreeMap = version_output - .lines() - .skip(1) - .filter_map(|line| { - let mut pieces = line.splitn(2, ':').fuse(); - match (pieces.next(), pieces.next()) { - (Some(name), Some(value)) => Some((name.trim().into(), value.trim().into())), - _ => None, - } - }) - .collect(); - - let release = info.remove("release").context(VersionReleaseMissingSnafu)?; - let commit_hash = info - .remove("commit-hash") - .context(VersionHashMissingSnafu)?; - let commit_date = info - .remove("commit-date") - .context(VersionDateMissingSnafu)?; - - Ok(Version { - release, - commit_hash, - commit_date, - }) - } - - pub async fn version_rustfmt(&self) -> Result { - let mut command = basic_secure_docker_command(); - command.args(&["rustfmt", "cargo", "fmt", "--version"]); - self.cargo_tool_version(command).await - } - - pub async fn version_clippy(&self) -> Result { - let mut command = basic_secure_docker_command(); - command.args(&["clippy", "cargo", "clippy", "--version"]); - self.cargo_tool_version(command).await - } - - pub async fn version_miri(&self) -> Result { - let mut command = basic_secure_docker_command(); - command.args(&["miri", "cargo", "miri", "--version"]); - self.cargo_tool_version(command).await - } - - // Parses versions of the shape `toolname 0.0.0 (0000000 0000-00-00)` - async fn cargo_tool_version(&self, command: Command) -> Result { - let output = run_command_with_timeout(command).await?; - let version_output = vec_to_str(output.stdout)?; - let mut parts = version_output.split_whitespace().fuse().skip(1); - - let release = parts.next().unwrap_or("").into(); - let commit_hash = parts.next().unwrap_or("").trim_start_matches('(').into(); - let commit_date = parts.next().unwrap_or("").trim_end_matches(')').into(); - - Ok(Version { - release, - commit_hash, - commit_date, - }) - } } async fn run_command_with_timeout(mut command: Command) -> Result { diff --git a/ui/src/server_axum.rs b/ui/src/server_axum.rs index f4101c95a..17acf3ee9 100644 --- a/ui/src/server_axum.rs +++ b/ui/src/server_axum.rs @@ -4,14 +4,14 @@ use crate::{ record_metric, track_metric_no_request_async, Endpoint, HasLabelsCore, Outcome, UNAVAILABLE_WS, }, - sandbox::{Channel, Sandbox, DOCKER_PROCESS_TIMEOUT_SOFT}, + sandbox::{Sandbox, DOCKER_PROCESS_TIMEOUT_SOFT}, CachingSnafu, ClippyRequest, ClippyResponse, ClippySnafu, CompileRequest, CompileResponse, CompileSnafu, Config, Error, ErrorJson, EvaluateRequest, EvaluateResponse, EvaluateSnafu, ExecuteRequest, ExecuteResponse, ExecuteSnafu, FormatRequest, FormatResponse, FormatSnafu, GhToken, GistCreationSnafu, GistLoadingSnafu, MacroExpansionRequest, MacroExpansionResponse, MacroExpansionSnafu, MetaCratesResponse, MetaGistCreateRequest, MetaGistResponse, - MetaVersionResponse, MetricsToken, MiriRequest, MiriResponse, MiriSnafu, Result, - SandboxCreationSnafu, ShutdownCoordinatorSnafu, TimeoutSnafu, + MetaVersionResponse, MetricsToken, MiriRequest, MiriResponse, MiriSnafu, MiriVersionSnafu, + Result, SandboxCreationSnafu, ShutdownCoordinatorSnafu, TimeoutSnafu, VersionsSnafu, }; use async_trait::async_trait; use axum::{ @@ -27,7 +27,7 @@ use axum::{ Router, }; use futures::{future::BoxFuture, FutureExt}; -use orchestrator::coordinator::{self, DockerBackend}; +use orchestrator::coordinator::{self, Coordinator, DockerBackend, Versions}; use snafu::prelude::*; use std::{ convert::TryInto, @@ -556,12 +556,7 @@ type Stamped = (T, SystemTime); #[derive(Debug, Default)] struct SandboxCache { crates: CacheOne, - version_stable: CacheOne, - version_beta: CacheOne, - version_nightly: CacheOne, - version_rustfmt: CacheOne, - version_clippy: CacheOne, - version_miri: CacheOne, + versions: CacheOne>, } impl SandboxCache { @@ -573,65 +568,53 @@ impl SandboxCache { .await } - async fn version_stable(&self) -> Result> { - self.version_stable - .fetch(|sandbox| async move { - let version = sandbox - .version(Channel::Stable) - .await - .context(CachingSnafu)?; - Ok(version.into()) + async fn versions(&self) -> Result>> { + let coordinator = Coordinator::new_docker().await; + + self.versions + .fetch2(|| async { + Ok(Arc::new( + coordinator.versions().await.context(VersionsSnafu)?, + )) }) .await } + async fn version_stable(&self) -> Result> { + let (v, t) = self.versions().await?; + let v = (&v.stable.rustc).into(); + Ok((v, t)) + } + async fn version_beta(&self) -> Result> { - self.version_beta - .fetch(|sandbox| async move { - let version = sandbox.version(Channel::Beta).await.context(CachingSnafu)?; - Ok(version.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.beta.rustc).into(); + Ok((v, t)) } async fn version_nightly(&self) -> Result> { - self.version_nightly - .fetch(|sandbox| async move { - let version = sandbox - .version(Channel::Nightly) - .await - .context(CachingSnafu)?; - Ok(version.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.nightly.rustc).into(); + Ok((v, t)) } async fn version_rustfmt(&self) -> Result> { - self.version_rustfmt - .fetch(|sandbox| async move { - Ok(sandbox - .version_rustfmt() - .await - .context(CachingSnafu)? - .into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.nightly.rustfmt).into(); + Ok((v, t)) } async fn version_clippy(&self) -> Result> { - self.version_clippy - .fetch(|sandbox| async move { - Ok(sandbox.version_clippy().await.context(CachingSnafu)?.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.nightly.clippy).into(); + Ok((v, t)) } async fn version_miri(&self) -> Result> { - self.version_miri - .fetch(|sandbox| async move { - Ok(sandbox.version_miri().await.context(CachingSnafu)?.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = v.nightly.miri.as_ref().context(MiriVersionSnafu)?; + let v = v.into(); + Ok((v, t)) } } @@ -698,6 +681,59 @@ where Ok(value) } + + async fn fetch2(&self, generator: F) -> Result> + where + F: FnOnce() -> FFut, + FFut: Future>, + { + let data = &mut *self.0.lock().await; + match data { + Some(info) => { + if info.validation_time.elapsed() <= SANDBOX_CACHE_TIME_TO_LIVE { + Ok(info.stamped_value()) + } else { + Self::set_value2(data, generator).await + } + } + None => Self::set_value2(data, generator).await, + } + } + + async fn set_value2( + data: &mut Option>, + generator: F, + ) -> Result> + where + F: FnOnce() -> FFut, + FFut: Future>, + { + let value = generator().await?; + + let old_info = data.take(); + let new_info = CacheInfo::build(value); + + let info = match old_info { + Some(mut old_value) => { + if old_value.value == new_info.value { + // The value hasn't changed; record that we have + // checked recently, but keep the creation time to + // preserve caching. + old_value.validation_time = new_info.validation_time; + old_value + } else { + new_info + } + } + None => new_info, + }; + + let value = info.stamped_value(); + + *data = Some(info); + + Ok(value) + } } #[derive(Debug)] @@ -776,6 +812,16 @@ pub(crate) mod api_orchestrator_integration_impls { use snafu::prelude::*; use std::convert::TryFrom; + impl From<&Version> for crate::MetaVersionResponse { + fn from(other: &Version) -> Self { + Self { + version: (&*other.release).into(), + hash: (&*other.commit_hash).into(), + date: (&*other.commit_date).into(), + } + } + } + impl TryFrom for ExecuteRequest { type Error = ParseEvaluateRequestError;