From f10827357ef6033c80f16ce9830ec77a0ad872c1 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Thu, 2 Jun 2022 18:52:28 +0300 Subject: [PATCH 0001/1953] workflow: require PR num input on test-kata-deploy workflow_dispatch this will require to set a PR number when triggering the test-kata-deploy workflow manually also make sure user variables are set correctly when workflow_dispatch is used Fixes: #4349 Signed-off-by: Snir Sheriber --- .github/workflows/kata-deploy-test.yaml | 36 ++++++++++++++++++------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index e0d6afd7c122..c2270ebd05fb 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -1,5 +1,10 @@ on: workflow_dispatch: # this is used to trigger the workflow on non-main branches + inputs: + pr: + description: 'PR number from the selected branch to test' + type: string + required: true issue_comment: types: [created, edited] @@ -13,19 +18,20 @@ jobs: && github.event_name == 'issue_comment' && github.event.action == 'created' && startsWith(github.event.comment.body, '/test_kata_deploy') + || github.event_name == 'workflow_dispatch' steps: - - name: Check membership + - name: Check membership on comment or dispatch uses: kata-containers/is-organization-member@1.0.1 id: is_organization_member with: organization: kata-containers - username: ${{ github.event.comment.user.login }} + username: ${{ github.event.comment.user.login || github.event.sender.login }} token: ${{ secrets.GITHUB_TOKEN }} - name: Fail if not member run: | result=${{ steps.is_organization_member.outputs.result }} if [ $result == false ]; then - user=${{ github.event.comment.user.login }} + user=${{ github.event.comment.user.login || github.event.sender.login }} echo Either ${user} is not part of the kata-containers organization echo or ${user} has its Organization Visibility set to Private at echo https://github.com/orgs/kata-containers/people?query=${user} @@ -53,8 +59,12 @@ jobs: - name: get-PR-ref id: get-PR-ref run: | - ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - echo "reference for PR: " ${ref} + if [ ${{ github.event_name }} == 'issue_comment' ]; then + ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') + else # workflow_dispatch + ref="refs/pull/${{ github.event.inputs.pr }}/merge" + fi + echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" - uses: actions/checkout@v2 with: @@ -89,8 +99,12 @@ jobs: - name: get-PR-ref id: get-PR-ref run: | - ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - echo "reference for PR: " ${ref} + if [ ${{ github.event_name }} == 'issue_comment' ]; then + ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') + else # workflow_dispatch + ref="refs/pull/${{ github.event.inputs.pr }}/merge" + fi + echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" - uses: actions/checkout@v2 with: @@ -116,8 +130,12 @@ jobs: - name: get-PR-ref id: get-PR-ref run: | - ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - echo "reference for PR: " ${ref} + if [ ${{ github.event_name }} == 'issue_comment' ]; then + ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') + else # workflow_dispatch + ref="refs/pull/${{ github.event.inputs.pr }}/merge" + fi + echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" - uses: actions/checkout@v2 with: From 7676cde0c5c4e5a99bca626770e534cab48c1db7 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Thu, 9 Jun 2022 09:17:04 +0300 Subject: [PATCH 0002/1953] workflow: trigger test-kata-deploy with pull_request event that changes VERSION (i.e. a release PR) Signed-off-by: Snir Sheriber --- .github/workflows/kata-deploy-test.yaml | 27 +++++++++++++++++++------ docs/Release-Process.md | 2 +- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index c2270ebd05fb..e7ff49e1a537 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -1,11 +1,19 @@ on: - workflow_dispatch: # this is used to trigger the workflow on non-main branches + pull_request: # this will trigger the workflow on PRs that changes release version + types: + - opened + - edited + - reopened + - synchronize + paths: + - VERSION + workflow_dispatch: # this allows to trigger the workflow manually on non-main branches inputs: pr: description: 'PR number from the selected branch to test' type: string required: true - issue_comment: + issue_comment: # this allows to trigger the workflow from main by commenting "/test_kata_deploy" types: [created, edited] name: test-kata-deploy @@ -19,13 +27,14 @@ jobs: && github.event.action == 'created' && startsWith(github.event.comment.body, '/test_kata_deploy') || github.event_name == 'workflow_dispatch' + || github.event_name == 'pull_request' steps: - name: Check membership on comment or dispatch uses: kata-containers/is-organization-member@1.0.1 id: is_organization_member with: organization: kata-containers - username: ${{ github.event.comment.user.login || github.event.sender.login }} + username: ${{ github.event.comment.user.login || github.event.sender.login }} # first one applies only on issue_comment token: ${{ secrets.GITHUB_TOKEN }} - name: Fail if not member run: | @@ -61,8 +70,10 @@ jobs: run: | if [ ${{ github.event_name }} == 'issue_comment' ]; then ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - else # workflow_dispatch + elif [ ${{ github.event_name }} == 'workflow_dispatch' ]; then ref="refs/pull/${{ github.event.inputs.pr }}/merge" + elif [ ${{ github.event_name }} == 'pull_request' ]; then + ref="refs/pull/${{ github.event.number }}/merge" fi echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" @@ -101,8 +112,10 @@ jobs: run: | if [ ${{ github.event_name }} == 'issue_comment' ]; then ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - else # workflow_dispatch + elif [ ${{ github.event_name }} == 'workflow_dispatch' ]; then ref="refs/pull/${{ github.event.inputs.pr }}/merge" + elif [ ${{ github.event_name }} == 'pull_request' ]; then + ref="refs/pull/${{ github.event.number }}/merge" fi echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" @@ -132,8 +145,10 @@ jobs: run: | if [ ${{ github.event_name }} == 'issue_comment' ]; then ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed 's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#') - else # workflow_dispatch + elif [ ${{ github.event_name }} == 'workflow_dispatch' ]; then ref="refs/pull/${{ github.event.inputs.pr }}/merge" + elif [ ${{ github.event_name }} == 'pull_request' ]; then + ref="refs/pull/${{ github.event.number }}/merge" fi echo "reference for PR: " ${ref} "event:" ${{ github.event_name }} echo "##[set-output name=pr-ref;]${ref}" diff --git a/docs/Release-Process.md b/docs/Release-Process.md index 7dcfb84a30ce..c9991d4b0914 100644 --- a/docs/Release-Process.md +++ b/docs/Release-Process.md @@ -48,7 +48,7 @@ ### Merge all bump version Pull requests - The above step will create a GitHub pull request in the Kata projects. Trigger the CI using `/test` command on each bump Pull request. - - Trigger the `test-kata-deploy` workflow which is under the `Actions` tab on the repository GitHub page (make sure to select the correct branch and validate it passes). + - `test-kata-deploy` workflow should be triggered automatically, validate it passes under the `Actions` tab on the repository GitHub page (you're also able to run it manually from there). - Check any failures and fix if needed. - Work with the Kata approvers to verify that the CI works and the pull requests are merged. From 575df4dc4d76ad142206ef18dbcc7a47858f2dfc Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Wed, 15 Dec 2021 22:13:10 +0800 Subject: [PATCH 0003/1953] static-checks: Allow Merge commit to be >75 chars Some generated merge commit messages are >75 chars Allow these to not trigger the subject line length failure Signed-off-by: Liu Jiang Signed-off-by: stevenhorsman --- .github/workflows/commit-message-check.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/commit-message-check.yaml b/.github/workflows/commit-message-check.yaml index 2062e51b9c01..a4852c0798a4 100644 --- a/.github/workflows/commit-message-check.yaml +++ b/.github/workflows/commit-message-check.yaml @@ -47,7 +47,7 @@ jobs: uses: tim-actions/commit-message-checker-with-regex@v0.3.1 with: commits: ${{ steps.get-pr-commits.outputs.commits }} - pattern: '^.{0,75}(\n.*)*$' + pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*' error: 'Subject too long (max 75)' post_error: ${{ env.error_msg }} @@ -94,6 +94,6 @@ jobs: uses: tim-actions/commit-message-checker-with-regex@v0.3.1 with: commits: ${{ steps.get-pr-commits.outputs.commits }} - pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:' + pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*' error: 'Failed to find subsystem in subject' post_error: ${{ env.error_msg }} From 392f1ecdf59f271fca8f4dd75d9a50aafa6897b7 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 16 Dec 2021 16:26:37 +0800 Subject: [PATCH 0004/1953] libs: convert to a cargo workspace Convert libs into a Cargo workspace, so all libraries could share the build infrastructure. Fixes #3282 Signed-off-by: Liu Jiang --- Makefile | 8 ++++---- README.md | 2 ++ src/agent/Makefile | 8 ++++---- src/libs/{logging => }/Makefile | 0 src/tools/agent-ctl/Makefile | 8 ++++---- src/tools/trace-forwarder/Makefile | 8 ++++---- 6 files changed, 18 insertions(+), 16 deletions(-) rename src/libs/{logging => }/Makefile (100%) diff --git a/Makefile b/Makefile index 2b6f6a748f48..0ca42a509c84 100644 --- a/Makefile +++ b/Makefile @@ -21,10 +21,10 @@ STANDARD_TARGETS = build check clean install test vendor default: all -all: logging-crate-tests build +all: libs-crate-tests build -logging-crate-tests: - make -C src/libs/logging +libs-crate-tests: + make -C src/libs include utils.mk include ./tools/packaging/kata-deploy/local-build/Makefile @@ -49,7 +49,7 @@ docs-url-alive-check: binary-tarball \ default \ install-binary-tarball \ - logging-crate-tests \ + libs-crate-tests \ static-checks \ docs-url-alive-check diff --git a/README.md b/README.md index 90a5c9209a4a..b8fa30996f64 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ The table below lists the core parts of the project: |-|-|-| | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. | | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. | +| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.ht +ml) | | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). | | [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) | | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. | diff --git a/src/agent/Makefile b/src/agent/Makefile index 9d1327c00448..6c09b5dfac1f 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -107,10 +107,10 @@ endef ##TARGET default: build code default: $(TARGET) show-header -$(TARGET): $(GENERATED_CODE) logging-crate-tests $(TARGET_PATH) +$(TARGET): $(GENERATED_CODE) libs-crate-tests $(TARGET_PATH) -logging-crate-tests: - make -C $(CWD)/../libs/logging +libs-crate-tests: + make -C $(CWD)/../libs $(TARGET_PATH): show-summary @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) @@ -203,7 +203,7 @@ codecov-html: check_tarpaulin .PHONY: \ help \ - logging-crate-tests \ + libs-crate-tests \ optimize \ show-header \ show-summary \ diff --git a/src/libs/logging/Makefile b/src/libs/Makefile similarity index 100% rename from src/libs/logging/Makefile rename to src/libs/Makefile diff --git a/src/tools/agent-ctl/Makefile b/src/tools/agent-ctl/Makefile index df3eacf243dd..b020563dbef5 100644 --- a/src/tools/agent-ctl/Makefile +++ b/src/tools/agent-ctl/Makefile @@ -8,11 +8,11 @@ include ../../../utils.mk .DEFAULT_GOAL := default default: build -build: logging-crate-tests +build: libs-crate-tests @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) -logging-crate-tests: - make -C $(CWD)/../../libs/logging +libs-crate-tests: + make -C $(CWD)/../../libs clean: cargo clean @@ -32,6 +32,6 @@ check: standard_rust_check check \ clean \ install \ - logging-crate-tests \ + libs-crate-tests \ test \ vendor diff --git a/src/tools/trace-forwarder/Makefile b/src/tools/trace-forwarder/Makefile index 5b1c53849b97..f597f891fa25 100644 --- a/src/tools/trace-forwarder/Makefile +++ b/src/tools/trace-forwarder/Makefile @@ -8,11 +8,11 @@ include ../../../utils.mk .DEFAULT_GOAL := default default: build -build: logging-crate-tests +build: libs-crate-tests @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) -logging-crate-tests: - make -C $(CWD)/../../libs/logging +libs-crate-tests: + make -C $(CWD)/../../libs clean: cargo clean @@ -32,6 +32,6 @@ check: standard_rust_check check \ clean \ install \ - logging-crate-tests \ + libs-crate-tests \ test \ vendor From 426f38de9428ff9faa554c2794b634fc2d606427 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 16 Dec 2021 14:06:12 +0800 Subject: [PATCH 0005/1953] libs/logging: implement rotator for log files Add FileRotator to rotate log files. The FileRotator structure may be used as writer for create_logger() and limits the storage space occupied by log files. Fixes: #3304 Signed-off-by: Liu Jiang Signed-off-by: Wei Yang Signed-off-by: yanlei --- src/libs/logging/src/file_rotate.rs | 315 ++++++++++++++++++++++++++++ src/libs/logging/src/lib.rs | 4 + 2 files changed, 319 insertions(+) create mode 100644 src/libs/logging/src/file_rotate.rs diff --git a/src/libs/logging/src/file_rotate.rs b/src/libs/logging/src/file_rotate.rs new file mode 100644 index 000000000000..2c2182b90756 --- /dev/null +++ b/src/libs/logging/src/file_rotate.rs @@ -0,0 +1,315 @@ +// Copyright (c) 2020 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 or MIT +// +// Partial code are extracted from +// https://github.com/sile/sloggers/blob/153c00a59f7218c1d96f522fb7a95c80bb0d530c/src/file.rs +// with following license and copyright. +// The MIT License +// +// Copyright (c) 2017 Takeru Ohta +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +use std::fs::{self, File, OpenOptions}; +use std::io::{self, LineWriter, Result, Write}; +use std::path::{Path, PathBuf}; + +/// Default rotate size for logger files. +const DEFAULT_LOG_FILE_SIZE_TO_ROTATE: u64 = 10485760; + +/// Default number of log files to keep. +const DEFAULT_HISTORY_LOG_FILES: usize = 3; + +/// Writer with file rotation for log files. +/// +/// This is a modified version of `FileAppender` from +/// https://github.com/sile/sloggers/blob/153c00a59f7218c1d96f522fb7a95c80bb0d530c/src/file.rs#L190 +#[derive(Debug)] +pub struct FileRotator { + path: PathBuf, + file: Option>, + ignore_errors: bool, + rotate_size: u64, + rotate_keep: usize, + truncate: bool, + written_size: u64, + #[cfg(test)] + fail_rename: bool, +} + +impl FileRotator { + /// Create a new instance of [`FileRotator`] to write log file at `path`. + /// + /// It returns `std::io::Error` if the path is not a normal file or the parent directory does + /// not exist. + pub fn new>(path: P) -> Result { + let p = Path::new(path.as_ref()); + match p.metadata() { + Ok(md) => { + if !md.is_file() { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("path '{}' is not a file", p.to_string_lossy()), + )); + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + if let Some(parent) = p.parent() { + if p.has_root() || !parent.as_os_str().is_empty() { + let md = parent.metadata()?; + if !md.is_dir() { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("'{}' is not a directory", parent.to_string_lossy()), + )); + } + } + } + + Ok(FileRotator { + path: p.to_path_buf(), + file: None, + ignore_errors: false, + rotate_size: DEFAULT_LOG_FILE_SIZE_TO_ROTATE, + rotate_keep: DEFAULT_HISTORY_LOG_FILES, + truncate: false, + written_size: 0, + #[cfg(test)] + fail_rename: false, + }) + } + + /// Use "truncate" or "append" mode when opening the log file. + pub fn truncate_mode(&mut self, truncate: bool) -> &mut Self { + self.truncate = truncate; + self + } + + /// Set the threshold size to rotate log files. + pub fn rotate_threshold(&mut self, size: u64) -> &mut Self { + self.rotate_size = size; + self + } + + /// Set number of rotated log files to keep. + pub fn rotate_count(&mut self, count: usize) -> &mut Self { + self.rotate_keep = count; + self + } + + /// Ignore all errors and try best effort to log messages but without guarantee. + pub fn ignore_errors(&mut self, ignore_errors: bool) -> &mut Self { + self.ignore_errors = ignore_errors; + self + } + + /// Open the log file if + /// - it hasn't been opened yet. + /// - current log file has been rotated and needs to open a new log file. + fn reopen_if_needed(&mut self) -> Result<()> { + if self.file.is_none() || !self.path.exists() { + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(self.truncate) + .append(!self.truncate) + .open(&self.path)?; + match file.metadata() { + Ok(md) => self.written_size = md.len(), + Err(e) => { + if self.ignore_errors { + // Pretend as an empty file. + // It's better to permit over-sized log file instead of disabling rotation. + self.written_size = 0; + } else { + return Err(e); + } + } + } + self.file = Some(LineWriter::new(file)); + } + + Ok(()) + } + + /// Try to rotate log files. + /// + /// When failed to rotate the log files, we choose to ignore the error instead of possibly + /// panicking the whole program. This may cause over-sized log files, but that should be easy + /// to recover. + fn rotate(&mut self) -> Result<()> { + for i in (1..=self.rotate_keep).rev() { + let from = self.rotated_path(i); + let to = self.rotated_path(i + 1); + if from.exists() { + let _ = fs::rename(from, to); + } + } + + #[cfg(test)] + if !self.fail_rename && self.path.exists() { + let rotated_path = self.rotated_path(1); + let _ = fs::rename(&self.path, &rotated_path); + } + #[cfg(not(test))] + if self.path.exists() { + let rotated_path = self.rotated_path(1); + let _ = fs::rename(&self.path, &rotated_path); + } + + let delete_path = self.rotated_path(self.rotate_keep + 1); + if delete_path.exists() { + let _ = fs::remove_file(delete_path); + } + + // Reset the `written_size` so only try to rotate again when another `rotate_size` bytes + // of log messages have been written to the lo file. + self.written_size = 0; + self.reopen_if_needed()?; + + Ok(()) + } + + fn rotated_path(&self, i: usize) -> PathBuf { + let mut path = self.path.clone().into_os_string(); + path.push(format!(".{}", i)); + PathBuf::from(path) + } +} + +impl Write for FileRotator { + fn write(&mut self, buf: &[u8]) -> Result { + if self.ignore_errors { + let _ = self.reopen_if_needed(); + if let Some(file) = self.file.as_mut() { + let _ = file.write_all(buf); + } + } else { + self.reopen_if_needed()?; + match self.file.as_mut() { + Some(file) => file.write_all(buf)?, + None => { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Cannot open file: {:?}", self.path), + )) + } + } + } + + self.written_size += buf.len() as u64; + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<()> { + if let Some(f) = self.file.as_mut() { + if let Err(e) = f.flush() { + if !self.ignore_errors { + return Err(e); + } + } + } + if self.written_size >= self.rotate_size { + if let Err(e) = self.rotate() { + if !self.ignore_errors { + return Err(e); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::MetadataExt; + + #[test] + fn test_rotator_valid_path() { + FileRotator::new("/proc/self").unwrap_err(); + FileRotator::new("/proc/self/__does_not_exist__/log.txt").unwrap_err(); + + let _ = FileRotator::new("log.txt").unwrap(); + } + + #[test] + fn test_rotator_rotate() { + let tmpdir = tempfile::tempdir().unwrap(); + let mut path = tmpdir.path().to_path_buf(); + path.push("log.txt"); + + let mut rotator = FileRotator::new(&path).unwrap(); + rotator.truncate_mode(false); + rotator.rotate_threshold(4); + rotator.rotate_count(1); + assert_eq!(rotator.rotate_size, 4); + assert_eq!(rotator.rotate_keep, 1); + assert_eq!(rotator.truncate, false); + + rotator.write("test".as_bytes()).unwrap(); + rotator.flush().unwrap(); + rotator.write("test1".as_bytes()).unwrap(); + rotator.flush().unwrap(); + rotator.write("t2".as_bytes()).unwrap(); + rotator.flush().unwrap(); + + let content = fs::read_to_string(path).unwrap(); + assert_eq!(content, "t2"); + + let mut path1 = tmpdir.path().to_path_buf(); + path1.push("log.txt.1"); + let content = fs::read_to_string(path1).unwrap(); + assert_eq!(content, "test1"); + + let mut path2 = tmpdir.path().to_path_buf(); + path2.push("log.txt.2"); + fs::read_to_string(path2).unwrap_err(); + } + + #[test] + fn test_rotator_rotate_fail() { + let tmpdir = tempfile::tempdir().unwrap(); + let mut path = tmpdir.path().to_path_buf(); + path.push("log.txt"); + + let mut rotator = FileRotator::new(&path).unwrap(); + rotator.truncate_mode(false); + rotator.rotate_threshold(1); + rotator.rotate_count(1); + rotator.fail_rename = true; + + rotator.write("test".as_bytes()).unwrap(); + rotator.flush().unwrap(); + let size1 = path.metadata().unwrap().size(); + + rotator.write("test1".as_bytes()).unwrap(); + rotator.flush().unwrap(); + let size2 = path.metadata().unwrap().size(); + assert!(size2 > size1); + + rotator.write("test2".as_bytes()).unwrap(); + rotator.flush().unwrap(); + let size3 = path.metadata().unwrap().size(); + assert!(size3 > size2); + } +} diff --git a/src/libs/logging/src/lib.rs b/src/libs/logging/src/lib.rs index 5d92acabfff9..e13948ba3043 100644 --- a/src/libs/logging/src/lib.rs +++ b/src/libs/logging/src/lib.rs @@ -11,6 +11,10 @@ use std::process; use std::result; use std::sync::Mutex; +mod file_rotate; + +pub use file_rotate::FileRotator; + const LOG_LEVELS: &[(&str, slog::Level)] = &[ ("trace", slog::Level::Trace), ("debug", slog::Level::Debug), From 7cdee4980c6729801a2c4aede4986cb3e3106a55 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Mon, 20 Dec 2021 16:36:02 +0800 Subject: [PATCH 0006/1953] libs/logging: introduce a wrapper writer for logging Introduce a wrapper writer `LogWriter` which converts every line written to it into a log record. Signed-off-by: Liu Jiang Signed-off-by: Wei Yang Signed-off-by: yanlei --- src/libs/logging/src/lib.rs | 2 + src/libs/logging/src/log_writer.rs | 66 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/libs/logging/src/log_writer.rs diff --git a/src/libs/logging/src/lib.rs b/src/libs/logging/src/lib.rs index e13948ba3043..f83af90a7875 100644 --- a/src/libs/logging/src/lib.rs +++ b/src/libs/logging/src/lib.rs @@ -12,8 +12,10 @@ use std::result; use std::sync::Mutex; mod file_rotate; +mod log_writer; pub use file_rotate::FileRotator; +pub use log_writer::LogWriter; const LOG_LEVELS: &[(&str, slog::Level)] = &[ ("trace", slog::Level::Trace), diff --git a/src/libs/logging/src/log_writer.rs b/src/libs/logging/src/log_writer.rs new file mode 100644 index 000000000000..a5fc7152e9b4 --- /dev/null +++ b/src/libs/logging/src/log_writer.rs @@ -0,0 +1,66 @@ +// Copyright (c) 2020 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{Result, Write}; + +use slog::{info, Logger}; + +/// Writer to convert each line written to it to a log record. +#[derive(Debug)] +pub struct LogWriter(Logger); + +impl LogWriter { + /// Create a new isntance of ['LogWriter']. + pub fn new(logger: Logger) -> Self { + LogWriter(logger) + } +} + +impl Write for LogWriter { + fn write(&mut self, buf: &[u8]) -> Result { + buf.split(|b| *b == b'\n').for_each(|it| { + if !it.is_empty() { + info!(self.0, "{}", String::from_utf8_lossy(it)) + } + }); + + Ok(buf.len()) + } + + fn flush(&mut self) -> Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{create_logger, FileRotator}; + use std::fs; + + #[test] + fn test_log_writer() { + let tmpdir = tempfile::tempdir().unwrap(); + let mut path = tmpdir.path().to_path_buf(); + path.push("log.txt"); + + let mut rotator = FileRotator::new(&path).unwrap(); + rotator.truncate_mode(false); + rotator.rotate_threshold(4); + rotator.rotate_count(1); + + let (logger, guard) = create_logger("test", "hi", slog::Level::Info, rotator); + let mut writer = LogWriter::new(logger); + + writer.write("test1\nblabla".as_bytes()).unwrap(); + writer.flush().unwrap(); + writer.write("test2".as_bytes()).unwrap(); + writer.flush().unwrap(); + drop(guard); + + let content = fs::read_to_string(path).unwrap(); + assert!(!content.is_empty()); + } +} From 6f8acb94c2feb253e4a75294078609c04dabc931 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 6 Jan 2022 14:52:59 +0800 Subject: [PATCH 0007/1953] libs: refine Makefile rules Refine Makefile rules to better support the KATA ci env. Fixes: #3536 Signed-off-by: Liu Jiang --- Makefile | 7 +---- src/agent/Makefile | 6 +---- src/libs/Makefile | 42 +++++++++++++++++++++++------- src/tools/agent-ctl/Makefile | 6 +---- src/tools/trace-forwarder/Makefile | 6 +---- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 0ca42a509c84..7411a4c29707 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ # List of available components COMPONENTS = +COMPONENTS += libs COMPONENTS += agent COMPONENTS += runtime @@ -21,11 +22,6 @@ STANDARD_TARGETS = build check clean install test vendor default: all -all: libs-crate-tests build - -libs-crate-tests: - make -C src/libs - include utils.mk include ./tools/packaging/kata-deploy/local-build/Makefile @@ -49,7 +45,6 @@ docs-url-alive-check: binary-tarball \ default \ install-binary-tarball \ - libs-crate-tests \ static-checks \ docs-url-alive-check diff --git a/src/agent/Makefile b/src/agent/Makefile index 6c09b5dfac1f..0a537aa55909 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -107,10 +107,7 @@ endef ##TARGET default: build code default: $(TARGET) show-header -$(TARGET): $(GENERATED_CODE) libs-crate-tests $(TARGET_PATH) - -libs-crate-tests: - make -C $(CWD)/../libs +$(TARGET): $(GENERATED_CODE) $(TARGET_PATH) $(TARGET_PATH): show-summary @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) @@ -203,7 +200,6 @@ codecov-html: check_tarpaulin .PHONY: \ help \ - libs-crate-tests \ optimize \ show-header \ show-summary \ diff --git a/src/libs/Makefile b/src/libs/Makefile index 74c917ab882f..9ce0be19de1a 100644 --- a/src/libs/Makefile +++ b/src/libs/Makefile @@ -3,16 +3,40 @@ # SPDX-License-Identifier: Apache-2.0 # -# It is not necessary to have a build target as this crate is built -# automatically by the consumers of it. -# -# However, it is essential that the crate be tested. -default: test +EXTRA_RUSTFEATURES := + +EXTRA_TEST_FLAGS := +USERID=$(shell id -u) +ifeq ($(USERID), 0) + override EXTRA_TEST_FLAGS = --ignored +endif + +default: build + +build: + cargo build --all-features + +check: clippy format + +clippy: + @echo "INFO: cargo clippy..." + cargo clippy --all-targets --all-features --release \ + -- \ + -D warnings + +format: + @echo "INFO: cargo fmt..." + cargo fmt -- --check + +clean: + cargo clean # It is essential to run these tests using *both* build profiles. # See the `test_logger_levels()` test for further information. test: - @echo "INFO: testing log levels for development build" - @cargo test - @echo "INFO: testing log levels for release build" - @cargo test --release + @echo "INFO: testing libraries for development build" + cargo test --all $(EXTRA_RUSTFEATURES) -- --nocapture $(EXTRA_TEST_FLAGS) + @echo "INFO: testing libraries for release build" + cargo test --release --all $(EXTRA_RUSTFEATURES) -- --nocapture $(EXTRA_TEST_FLAGS) + +.PHONY: install vendor diff --git a/src/tools/agent-ctl/Makefile b/src/tools/agent-ctl/Makefile index b020563dbef5..99713f6c8d0d 100644 --- a/src/tools/agent-ctl/Makefile +++ b/src/tools/agent-ctl/Makefile @@ -8,12 +8,9 @@ include ../../../utils.mk .DEFAULT_GOAL := default default: build -build: libs-crate-tests +build: @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) -libs-crate-tests: - make -C $(CWD)/../../libs - clean: cargo clean @@ -32,6 +29,5 @@ check: standard_rust_check check \ clean \ install \ - libs-crate-tests \ test \ vendor diff --git a/src/tools/trace-forwarder/Makefile b/src/tools/trace-forwarder/Makefile index f597f891fa25..b6b223c001e6 100644 --- a/src/tools/trace-forwarder/Makefile +++ b/src/tools/trace-forwarder/Makefile @@ -8,12 +8,9 @@ include ../../../utils.mk .DEFAULT_GOAL := default default: build -build: libs-crate-tests +build: @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) -libs-crate-tests: - make -C $(CWD)/../../libs - clean: cargo clean @@ -32,6 +29,5 @@ check: standard_rust_check check \ clean \ install \ - libs-crate-tests \ test \ vendor From 4f62a7618c1a800eae900bd68b873bcdee7d13d3 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Fri, 7 Jan 2022 13:46:24 +0800 Subject: [PATCH 0008/1953] libs/logging: fix clippy warnings Fix clippy warnings of libs/logging. Signed-off-by: Liu Jiang --- src/libs/logging/src/file_rotate.rs | 14 +++++++------- src/libs/logging/src/lib.rs | 26 +++++++++++++------------- src/libs/logging/src/log_writer.rs | 4 ++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libs/logging/src/file_rotate.rs b/src/libs/logging/src/file_rotate.rs index 2c2182b90756..444297e53d58 100644 --- a/src/libs/logging/src/file_rotate.rs +++ b/src/libs/logging/src/file_rotate.rs @@ -264,13 +264,13 @@ mod tests { rotator.rotate_count(1); assert_eq!(rotator.rotate_size, 4); assert_eq!(rotator.rotate_keep, 1); - assert_eq!(rotator.truncate, false); + assert!(!rotator.truncate); - rotator.write("test".as_bytes()).unwrap(); + rotator.write_all("test".as_bytes()).unwrap(); rotator.flush().unwrap(); - rotator.write("test1".as_bytes()).unwrap(); + rotator.write_all("test1".as_bytes()).unwrap(); rotator.flush().unwrap(); - rotator.write("t2".as_bytes()).unwrap(); + rotator.write_all("t2".as_bytes()).unwrap(); rotator.flush().unwrap(); let content = fs::read_to_string(path).unwrap(); @@ -298,16 +298,16 @@ mod tests { rotator.rotate_count(1); rotator.fail_rename = true; - rotator.write("test".as_bytes()).unwrap(); + rotator.write_all("test".as_bytes()).unwrap(); rotator.flush().unwrap(); let size1 = path.metadata().unwrap().size(); - rotator.write("test1".as_bytes()).unwrap(); + rotator.write_all("test1".as_bytes()).unwrap(); rotator.flush().unwrap(); let size2 = path.metadata().unwrap().size(); assert!(size2 > size1); - rotator.write("test2".as_bytes()).unwrap(); + rotator.write_all("test2".as_bytes()).unwrap(); rotator.flush().unwrap(); let size3 = path.metadata().unwrap().size(); assert!(size3 > size2); diff --git a/src/libs/logging/src/lib.rs b/src/libs/logging/src/lib.rs index f83af90a7875..33f9fee3ee2c 100644 --- a/src/libs/logging/src/lib.rs +++ b/src/libs/logging/src/lib.rs @@ -534,13 +534,13 @@ mod tests { let msg = format!("test[{}]", i); // Create a writer for the logger drain to use - let writer = - NamedTempFile::new().expect(&format!("{:}: failed to create tempfile", msg)); + let writer = NamedTempFile::new() + .unwrap_or_else(|_| panic!("{:}: failed to create tempfile", msg)); // Used to check file contents before the temp file is unlinked let mut writer_ref = writer .reopen() - .expect(&format!("{:?}: failed to clone tempfile", msg)); + .unwrap_or_else(|e| panic!("{:?}: failed to clone tempfile, {}", msg, e)); let (logger, logger_guard) = create_logger(name, source, d.slog_level, writer); @@ -554,52 +554,52 @@ mod tests { let mut contents = String::new(); writer_ref .read_to_string(&mut contents) - .expect(&format!("{:?}: failed to read tempfile contents", msg)); + .unwrap_or_else(|e| panic!("{:?}: failed to read tempfile contents, {}", msg, e)); // Convert file to JSON let fields: Value = serde_json::from_str(&contents) - .expect(&format!("{:?}: failed to convert logfile to json", msg)); + .unwrap_or_else(|e| panic!("{:?}: failed to convert logfile to json, {}", msg, e)); // Check the expected JSON fields let field_ts = fields .get("ts") - .expect(&format!("{:?}: failed to find timestamp field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find timestamp field", msg)); assert_ne!(field_ts, "", "{}", msg); let field_version = fields .get("version") - .expect(&format!("{:?}: failed to find version field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find version field", msg)); assert_eq!(field_version, env!("CARGO_PKG_VERSION"), "{}", msg); let field_pid = fields .get("pid") - .expect(&format!("{:?}: failed to find pid field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find pid field", msg)); assert_ne!(field_pid, "", "{}", msg); let field_level = fields .get("level") - .expect(&format!("{:?}: failed to find level field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find level field", msg)); assert_eq!(field_level, d.slog_level_tag, "{}", msg); let field_msg = fields .get("msg") - .expect(&format!("{:?}: failed to find msg field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find msg field", msg)); assert_eq!(field_msg, &json!(d.msg), "{}", msg); let field_name = fields .get("name") - .expect(&format!("{:?}: failed to find name field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find name field", msg)); assert_eq!(field_name, name, "{}", msg); let field_source = fields .get("source") - .expect(&format!("{:?}: failed to find source field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find source field", msg)); assert_eq!(field_source, source, "{}", msg); let field_subsystem = fields .get("subsystem") - .expect(&format!("{:?}: failed to find subsystem field", msg)); + .unwrap_or_else(|| panic!("{:?}: failed to find subsystem field", msg)); // No explicit subsystem, so should be the default assert_eq!(field_subsystem, &json!(DEFAULT_SUBSYSTEM), "{}", msg); diff --git a/src/libs/logging/src/log_writer.rs b/src/libs/logging/src/log_writer.rs index a5fc7152e9b4..53e6d541e06c 100644 --- a/src/libs/logging/src/log_writer.rs +++ b/src/libs/logging/src/log_writer.rs @@ -54,9 +54,9 @@ mod tests { let (logger, guard) = create_logger("test", "hi", slog::Level::Info, rotator); let mut writer = LogWriter::new(logger); - writer.write("test1\nblabla".as_bytes()).unwrap(); + writer.write_all("test1\nblabla".as_bytes()).unwrap(); writer.flush().unwrap(); - writer.write("test2".as_bytes()).unwrap(); + writer.write_all("test2".as_bytes()).unwrap(); writer.flush().unwrap(); drop(guard); From 5b89c1df2fb105ed3afbf68d946ee7b19cdd5263 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 9 Dec 2021 11:51:38 +0800 Subject: [PATCH 0009/1953] libs/types: add kata-types crate under src/libs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add kata-types crate to host constants and data types shared by multiple Kata Containers components. Fixes: #3305 Signed-off-by: Liu Jiang Signed-off-by: Fupan Li  Signed-off-by: Huamin Tang Signed-off-by: Lei Wang Signed-off-by: yanlei --- src/libs/Cargo.lock | 19 ++ src/libs/Cargo.toml | 2 + src/libs/README.md | 5 +- src/libs/kata-types/Cargo.toml | 19 ++ src/libs/kata-types/README.md | 18 ++ .../src/annotations/cri_containerd.rs | 13 ++ src/libs/kata-types/src/annotations/crio.rs | 13 ++ .../kata-types/src/annotations/dockershim.rs | 13 ++ src/libs/kata-types/src/annotations/mod.rs | 14 ++ src/libs/kata-types/src/container.rs | 203 ++++++++++++++++++ src/libs/kata-types/src/k8s.rs | 102 +++++++++ src/libs/kata-types/src/lib.rs | 20 ++ src/libs/kata-types/src/mount.rs | 88 ++++++++ 13 files changed, 527 insertions(+), 2 deletions(-) create mode 100644 src/libs/kata-types/Cargo.toml create mode 100644 src/libs/kata-types/README.md create mode 100644 src/libs/kata-types/src/annotations/cri_containerd.rs create mode 100644 src/libs/kata-types/src/annotations/crio.rs create mode 100644 src/libs/kata-types/src/annotations/dockershim.rs create mode 100644 src/libs/kata-types/src/annotations/mod.rs create mode 100644 src/libs/kata-types/src/container.rs create mode 100644 src/libs/kata-types/src/k8s.rs create mode 100644 src/libs/kata-types/src/lib.rs create mode 100644 src/libs/kata-types/src/mount.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index fc7fc5da5fc1..f2ec072546b3 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -283,6 +283,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "kata-types" +version = "0.1.0" +dependencies = [ + "oci", + "serde", + "thiserror", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -414,6 +423,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "oci" +version = "0.1.0" +dependencies = [ + "libc", + "serde", + "serde_derive", + "serde_json", +] + [[package]] name = "once_cell" version = "1.9.0" diff --git a/src/libs/Cargo.toml b/src/libs/Cargo.toml index 16eedb91f272..10887de3c1f5 100644 --- a/src/libs/Cargo.toml +++ b/src/libs/Cargo.toml @@ -1,7 +1,9 @@ [workspace] members = [ "logging", + "kata-types", "safe-path", "protocols", + "oci", ] resolver = "2" diff --git a/src/libs/README.md b/src/libs/README.md index 36f2f00d73cc..0593749a09f1 100644 --- a/src/libs/README.md +++ b/src/libs/README.md @@ -5,6 +5,7 @@ or published to [`crates.io`](https://crates.io/index.html). Currently it provides following library crates: | Library | Description | -|-|-|-| -| [logging](logging/) | Facilities to setup logging subsystem based slog. | +|-|-| +| [logging](logging/) | Facilities to setup logging subsystem based on slog. | +| [types](kata-types/) | Collection of constants and data types shared by multiple Kata Containers components. | | [safe-path](safe-path/) | Utilities to safely resolve filesystem paths. | diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml new file mode 100644 index 000000000000..807791dc0850 --- /dev/null +++ b/src/libs/kata-types/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "kata-types" +version = "0.1.0" +description = "Constants and data types shared by Kata Containers components" +keywords = ["kata", "container", "runtime"] +authors = ["The Kata Containers community "] +repository = "https://github.com/kata-containers/kata-containers.git" +homepage = "https://katacontainers.io/" +readme = "README.md" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +serde = { version = "1.0.100", features = ["derive"] } +thiserror = "1.0" + +oci = { path = "../oci" } + +[dev-dependencies] diff --git a/src/libs/kata-types/README.md b/src/libs/kata-types/README.md new file mode 100644 index 000000000000..334c879e20a3 --- /dev/null +++ b/src/libs/kata-types/README.md @@ -0,0 +1,18 @@ +# kata-types + +This crate is a collection of constants and data types shared by multiple +[Kata Containers](https://github.com/kata-containers/kata-containers/) components. + +It defines constants and data types used by multiple Kata Containers components. Those constants +and data types may be defined by Kata Containers or by other projects/specifications, such as: +- [Containerd](https://github.com/containerd/containerd) +- [Kubelet](https://github.com/kubernetes/kubelet) + +## Support + +**Operating Systems**: +- Linux + +## License + +This code is licensed under [Apache-2.0](../../../LICENSE). diff --git a/src/libs/kata-types/src/annotations/cri_containerd.rs b/src/libs/kata-types/src/annotations/cri_containerd.rs new file mode 100644 index 000000000000..db6462a8c8e3 --- /dev/null +++ b/src/libs/kata-types/src/annotations/cri_containerd.rs @@ -0,0 +1,13 @@ +// Copyright (c) 2019 Alibaba Cloud +// Copyright (c) 2019 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#![allow(missing_docs)] + +pub const CONTAINER_TYPE_LABEL_KEY: &str = "io.kubernetes.cri.container-type"; +pub const SANDBOX: &str = "sandbox"; +pub const CONTAINER: &str = "container"; + +pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id"; diff --git a/src/libs/kata-types/src/annotations/crio.rs b/src/libs/kata-types/src/annotations/crio.rs new file mode 100644 index 000000000000..c8b2311f844d --- /dev/null +++ b/src/libs/kata-types/src/annotations/crio.rs @@ -0,0 +1,13 @@ +// Copyright (c) 2019 Alibaba Cloud +// Copyright (c) 2019 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#![allow(missing_docs)] + +pub const CONTAINER_TYPE_LABEL_KEY: &str = "io.kubernetes.cri.container-type"; +pub const SANDBOX: &str = "sandbox"; +pub const CONTAINER: &str = "container"; + +pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri-o.SandboxID"; diff --git a/src/libs/kata-types/src/annotations/dockershim.rs b/src/libs/kata-types/src/annotations/dockershim.rs new file mode 100644 index 000000000000..1558983d9a1b --- /dev/null +++ b/src/libs/kata-types/src/annotations/dockershim.rs @@ -0,0 +1,13 @@ +// Copyright (c) 2019 Alibaba Cloud +// Copyright (c) 2019 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#![allow(missing_docs)] + +pub const CONTAINER_TYPE_LABEL_KEY: &str = "io.kubernetes.docker.type"; +pub const SANDBOX: &str = "podsandbox"; +pub const CONTAINER: &str = "container"; + +pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.sandbox.id"; diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs new file mode 100644 index 000000000000..f1cd1432046a --- /dev/null +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2019 Alibaba Cloud +// Copyright (c) 2019 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +/// CRI-containerd specific annotations. +pub mod cri_containerd; + +/// CRI-O specific annotations. +pub mod crio; + +/// Dockershim specific annotations. +pub mod dockershim; diff --git a/src/libs/kata-types/src/container.rs b/src/libs/kata-types/src/container.rs new file mode 100644 index 000000000000..aee94f95d37c --- /dev/null +++ b/src/libs/kata-types/src/container.rs @@ -0,0 +1,203 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +const CONTAINER: &str = "container"; +const SANDBOX: &str = "sandbox"; +const POD_CONTAINER: &str = "pod_container"; +const POD_SANDBOX: &str = "pod_sandbox"; +const POD_SANDBOX2: &str = "podsandbox"; + +const STATE_READY: &str = "ready"; +const STATE_RUNNING: &str = "running"; +const STATE_STOPPED: &str = "stopped"; +const STATE_PAUSED: &str = "paused"; + +/// Error codes for container related operations. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// Invalid container type + #[error("Invalid container type {0}")] + InvalidContainerType(String), + /// Invalid container state + #[error("Invalid sandbox state {0}")] + InvalidState(String), + /// Invalid container state transition + #[error("Can not transit from {0} to {1}")] + InvalidStateTransition(State, State), +} + +/// Types of pod containers: container or sandbox. +#[derive(PartialEq, Debug, Clone)] +pub enum ContainerType { + /// A pod container. + PodContainer, + /// A pod sandbox. + PodSandbox, +} + +impl ContainerType { + /// Check whether it's a pod container. + pub fn is_pod_container(&self) -> bool { + matches!(self, ContainerType::PodContainer) + } + + /// Check whether it's a pod container. + pub fn is_pod_sandbox(&self) -> bool { + matches!(self, ContainerType::PodSandbox) + } +} + +impl Display for ContainerType { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + match self { + ContainerType::PodContainer => write!(f, "{}", POD_CONTAINER), + ContainerType::PodSandbox => write!(f, "{}", POD_SANDBOX), + } + } +} + +impl FromStr for ContainerType { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + POD_CONTAINER | CONTAINER => Ok(ContainerType::PodContainer), + POD_SANDBOX | POD_SANDBOX2 | SANDBOX => Ok(ContainerType::PodSandbox), + _ => Err(Error::InvalidContainerType(value.to_owned())), + } + } +} + +/// Process states. +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum State { + /// The container is ready to run. + Ready, + /// The container executed the user-specified program but has not exited + Running, + /// The container has exited + Stopped, + /// The container has been paused. + Paused, +} + +impl Display for State { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + match self { + State::Ready => write!(f, "{}", STATE_READY), + State::Running => write!(f, "{}", STATE_RUNNING), + State::Stopped => write!(f, "{}", STATE_STOPPED), + State::Paused => write!(f, "{}", STATE_PAUSED), + } + } +} + +impl FromStr for State { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + STATE_READY => Ok(State::Ready), + STATE_RUNNING => Ok(State::Running), + STATE_STOPPED => Ok(State::Stopped), + STATE_PAUSED => Ok(State::Paused), + _ => Err(Error::InvalidState(value.to_owned())), + } + } +} + +impl State { + /// Check whether it's a valid state transition from self to the `new_state`. + pub fn check_transition(self, new_state: State) -> Result<(), Error> { + match self { + State::Ready if new_state == State::Running || new_state == State::Stopped => Ok(()), + State::Running if new_state == State::Stopped => Ok(()), + State::Stopped if new_state == State::Running => Ok(()), + State::Paused if new_state == State::Paused => Ok(()), + _ => Err(Error::InvalidStateTransition(self, new_state)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_container_type() { + assert!(ContainerType::PodContainer.is_pod_container()); + assert!(!ContainerType::PodContainer.is_pod_sandbox()); + + assert!(ContainerType::PodSandbox.is_pod_sandbox()); + assert!(!ContainerType::PodSandbox.is_pod_container()); + } + + #[test] + fn test_container_type_display() { + assert_eq!(format!("{}", ContainerType::PodContainer), POD_CONTAINER); + assert_eq!(format!("{}", ContainerType::PodSandbox), POD_SANDBOX); + } + + #[test] + fn test_container_type_from_str() { + assert_eq!( + ContainerType::from_str("pod_container").unwrap(), + ContainerType::PodContainer + ); + assert_eq!( + ContainerType::from_str("container").unwrap(), + ContainerType::PodContainer + ); + assert_eq!( + ContainerType::from_str("pod_sandbox").unwrap(), + ContainerType::PodSandbox + ); + assert_eq!( + ContainerType::from_str("podsandbox").unwrap(), + ContainerType::PodSandbox + ); + assert_eq!( + ContainerType::from_str("sandbox").unwrap(), + ContainerType::PodSandbox + ); + ContainerType::from_str("test").unwrap_err(); + } + + #[test] + fn test_valid() { + let mut state = State::from_str("invalid_state"); + assert!(state.is_err()); + + state = State::from_str("ready"); + assert!(state.is_ok()); + + state = State::from_str("running"); + assert!(state.is_ok()); + + state = State::from_str("stopped"); + assert!(state.is_ok()); + } + + #[test] + fn test_valid_transition() { + use State::*; + + assert!(Ready.check_transition(Ready).is_err()); + assert!(Ready.check_transition(Running).is_ok()); + assert!(Ready.check_transition(Stopped).is_ok()); + + assert!(Running.check_transition(Ready).is_err()); + assert!(Running.check_transition(Running).is_err()); + assert!(Running.check_transition(Stopped).is_ok()); + + assert!(Stopped.check_transition(Ready).is_err()); + assert!(Stopped.check_transition(Running).is_ok()); + assert!(Stopped.check_transition(Stopped).is_err()); + } +} diff --git a/src/libs/kata-types/src/k8s.rs b/src/libs/kata-types/src/k8s.rs new file mode 100644 index 000000000000..43f5ba839d24 --- /dev/null +++ b/src/libs/kata-types/src/k8s.rs @@ -0,0 +1,102 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::Path; + +use crate::annotations; +use crate::container::ContainerType; +use std::str::FromStr; + +// K8S_EMPTY_DIR is the k8s specific path for `empty-dir` volumes +const K8S_EMPTY_DIR: &str = "kubernetes.io~empty-dir"; + +/// Check whether the path is a K8S empty directory. +/// +/// For a K8S EmptyDir, Kubernetes mounts +/// "/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/" +/// to "/". +pub fn is_empty_dir>(path: P) -> bool { + let path = path.as_ref(); + + if let Some(parent) = path.parent() { + if let Some(pname) = parent.file_name() { + if pname == K8S_EMPTY_DIR && parent.parent().is_some() { + return true; + } + } + } + + false +} + +/// Get K8S container type from OCI annotations. +pub fn container_type(spec: &oci::Spec) -> ContainerType { + // PodSandbox: "sandbox" (Containerd & CRI-O), "podsandbox" (dockershim) + // PodContainer: "container" (Containerd & CRI-O & dockershim) + for k in [ + annotations::crio::CONTAINER_TYPE_LABEL_KEY, + annotations::cri_containerd::CONTAINER_TYPE_LABEL_KEY, + annotations::dockershim::CONTAINER_TYPE_LABEL_KEY, + ] + .iter() + { + if let Some(v) = spec.annotations.get(k.to_owned()) { + if let Ok(t) = ContainerType::from_str(v) { + return t; + } + } + } + + ContainerType::PodSandbox +} + +/// Determine the k8s sandbox ID from OCI annotations. +/// +/// This function is expected to be called only when the container type is "PodContainer". +pub fn sandbox_id(spec: &oci::Spec) -> Result, String> { + if container_type(spec) != ContainerType::PodSandbox { + return Err("Not a sandbox container".to_string()); + } + for k in [ + annotations::crio::SANDBOX_ID_LABEL_KEY, + annotations::cri_containerd::SANDBOX_ID_LABEL_KEY, + annotations::dockershim::SANDBOX_ID_LABEL_KEY, + ] + .iter() + { + if let Some(id) = spec.annotations.get(k.to_owned()) { + return Ok(Some(id.to_string())); + } + } + + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_empty_dir() { + let empty_dir = "/volumes/kubernetes.io~empty-dir/shm"; + assert!(is_empty_dir(empty_dir)); + + let empty_dir = "/volumes/kubernetes.io~empty-dir//shm"; + assert!(is_empty_dir(empty_dir)); + + let empty_dir = "/volumes/kubernetes.io~empty-dir-test/shm"; + assert!(!is_empty_dir(empty_dir)); + + let empty_dir = "/volumes/kubernetes.io~empty-dir"; + assert!(!is_empty_dir(empty_dir)); + + let empty_dir = "kubernetes.io~empty-dir"; + assert!(!is_empty_dir(empty_dir)); + + let empty_dir = "/kubernetes.io~empty-dir/shm"; + assert!(is_empty_dir(empty_dir)); + } +} diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs new file mode 100644 index 000000000000..5fde5061509c --- /dev/null +++ b/src/libs/kata-types/src/lib.rs @@ -0,0 +1,20 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Constants and Data Types shared by Kata Containers components. + +#[deny(missing_docs)] + +/// Constants and data types annotations. +pub mod annotations; + +/// Constants and data types related to container. +pub mod container; + +/// Constants and data types related to Kubernetes/kubelet. +pub mod k8s; + +/// Constants and data types related to mount point. +pub mod mount; diff --git a/src/libs/kata-types/src/mount.rs b/src/libs/kata-types/src/mount.rs new file mode 100644 index 000000000000..2ccc0feed28a --- /dev/null +++ b/src/libs/kata-types/src/mount.rs @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::PathBuf; + +/// Prefix to mark a volume as Kata special. +pub const KATA_VOLUME_TYPE_PREFIX: &str = "kata:"; + +/// The Mount should be ignored by the host and handled by the guest. +pub const KATA_GUEST_MOUNT_PREFIX: &str = "kata:guest-mount:"; + +/// KATA_EPHEMERAL_DEV_TYPE creates a tmpfs backed volume for sharing files between containers. +pub const KATA_EPHEMERAL_VOLUME_TYPE: &str = "kata:ephemeral"; + +/// KATA_HOST_DIR_TYPE use for host empty dir +pub const KATA_HOST_DIR_VOLUME_TYPE: &str = "kata:hostdir"; + +/// Information about a mount. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct Mount { + /// A device name, but can also be a file or directory name for bind mounts or a dummy. + /// Path values for bind mounts are either absolute or relative to the bundle. A mount is a + /// bind mount if it has either bind or rbind in the options. + pub source: String, + /// Destination of mount point: path inside container. This value MUST be an absolute path. + pub destination: PathBuf, + /// The type of filesystem for the mountpoint. + pub fs_type: String, + /// Mount options for the mountpoint. + pub options: Vec, + /// Optional device id for the block device when: + /// - the source is a block device or a mountpoint for a block device + /// - block device direct assignment is enabled + pub device_id: Option, + /// Intermediate path to mount the source on host side and then passthrough to vm by shared fs. + pub host_shared_fs_path: Option, + /// Whether to mount the mountpoint in readonly mode + pub read_only: bool, +} + +impl Mount { + /// Get size of mount options. + pub fn option_size(&self) -> usize { + self.options.iter().map(|v| v.len() + 1).sum() + } +} + +/// Check whether a mount type is a marker for Kata specific volume. +pub fn is_kata_special_volume(ty: &str) -> bool { + ty.len() > KATA_VOLUME_TYPE_PREFIX.len() && ty.starts_with(KATA_VOLUME_TYPE_PREFIX) +} + +/// Check whether a mount type is a marker for Kata guest mount volume. +pub fn is_kata_guest_mount_volume(ty: &str) -> bool { + ty.len() > KATA_GUEST_MOUNT_PREFIX.len() && ty.starts_with(KATA_GUEST_MOUNT_PREFIX) +} + +/// Check whether a mount type is a marker for Kata ephemeral volume. +pub fn is_kata_ephemeral_volume(ty: &str) -> bool { + ty == KATA_EPHEMERAL_VOLUME_TYPE +} + +/// Check whether a mount type is a marker for Kata hostdir volume. +pub fn is_kata_host_dir_volume(ty: &str) -> bool { + ty == KATA_HOST_DIR_VOLUME_TYPE +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_kata_special_volume() { + assert!(is_kata_special_volume("kata:guest-mount:nfs")); + assert!(!is_kata_special_volume("kata:")); + } + + #[test] + fn test_is_kata_guest_mount_volume() { + assert!(is_kata_guest_mount_volume("kata:guest-mount:nfs")); + assert!(!is_kata_guest_mount_volume("kata:guest-mount")); + assert!(!is_kata_guest_mount_volume("kata:guest-moun")); + assert!(!is_kata_guest_mount_volume("Kata:guest-mount:nfs")); + } +} From 21cc02d724c454c8b6e65615ecc8307e1c0985fc Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Sun, 26 Dec 2021 10:28:02 +0800 Subject: [PATCH 0010/1953] libs/types: support load Kata runtime configuration from file Add structures to load Kata runtime configuration from configuration files. Also define a mechanism for vendor to extend the Kata configuration structure. Signed-off-by: Liu Jiang Signed-off-by: Zhongtao Hu --- src/libs/Cargo.lock | 13 + src/libs/kata-types/Cargo.toml | 7 + src/libs/kata-types/src/config/default.rs | 23 ++ src/libs/kata-types/src/config/mod.rs | 125 ++++++++ src/libs/kata-types/src/config/runtime.rs | 272 ++++++++++++++++++ .../kata-types/src/config/runtime_vendor.rs | 88 ++++++ src/libs/kata-types/src/lib.rs | 62 +++- 7 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 src/libs/kata-types/src/config/default.rs create mode 100644 src/libs/kata-types/src/config/mod.rs create mode 100644 src/libs/kata-types/src/config/runtime.rs create mode 100644 src/libs/kata-types/src/config/runtime_vendor.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index f2ec072546b3..c9888c433d76 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -287,9 +287,13 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" name = "kata-types" version = "0.1.0" dependencies = [ + "lazy_static", "oci", "serde", + "slog", + "slog-scope", "thiserror", + "toml", ] [[package]] @@ -802,6 +806,15 @@ dependencies = [ "vsock", ] +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + [[package]] name = "ttrpc" version = "0.5.2" diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 807791dc0850..5c6035e36a85 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -11,9 +11,16 @@ license = "Apache-2.0" edition = "2018" [dependencies] +lazy_static = "1.4.0" serde = { version = "1.0.100", features = ["derive"] } +slog = "2.5.2" +slog-scope = "4.4.0" thiserror = "1.0" +toml = "0.5.8" oci = { path = "../oci" } [dev-dependencies] +[features] +default = [] +enable-vendor = [] diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs new file mode 100644 index 000000000000..4c501c06f598 --- /dev/null +++ b/src/libs/kata-types/src/config/default.rs @@ -0,0 +1,23 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Default configuration values. +#![allow(missing_docs)] + +use lazy_static::lazy_static; + +lazy_static! { + /// Default configuration file paths. + pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![ + "/etc/kata-containers2/configuration.toml", + "/usr/share/defaults/kata-containers2/configuration.toml", + "/etc/kata-containers/configuration_v2.toml", + "/usr/share/defaults/kata-containers/configuration_v2.toml", + "/etc/kata-containers/configuration.toml", + "/usr/share/defaults/kata-containers/configuration.toml", + ]; +} + +pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter"; diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs new file mode 100644 index 000000000000..5d13263016a6 --- /dev/null +++ b/src/libs/kata-types/src/config/mod.rs @@ -0,0 +1,125 @@ +// Copyright (c) 2019-2021 Ant Financial +// Copyright (c) 2019-2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fs; +use std::io::{self, Result}; +use std::path::{Path, PathBuf}; + +use crate::sl; + +/// Default configuration values. +pub mod default; + +mod runtime; +pub use self::runtime::{Runtime, RuntimeVendor}; + +/// Trait to manipulate global Kata configuration information. +pub trait ConfigOps { + /// Adjust the configuration information after loading from configuration file. + fn adjust_configuration(_conf: &mut TomlConfig) -> Result<()> { + Ok(()) + } + + /// Validate the configuration information. + fn validate(_conf: &TomlConfig) -> Result<()> { + Ok(()) + } +} + +/// Trait to manipulate global Kata configuration information. +pub trait ConfigObjectOps { + /// Adjust the configuration information after loading from configuration file. + fn adjust_configuration(&mut self) -> Result<()> { + Ok(()) + } + + /// Validate the configuration information. + fn validate(&self) -> Result<()> { + Ok(()) + } +} + +/// Kata configuration information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct TomlConfig { + /// Kata runtime configuration information. + #[serde(default)] + pub runtime: Runtime, +} + +impl TomlConfig { + /// Load Kata configuration information from configuration files. + /// + /// If `config_file` is valid, it will used, otherwise a built-in default path list will be + /// scanned. + pub fn load_from_file>(config_file: P) -> Result<(TomlConfig, PathBuf)> { + let file_path = if !config_file.as_ref().as_os_str().is_empty() { + fs::canonicalize(config_file)? + } else { + Self::get_default_config_file()? + }; + + info!( + sl!(), + "load configuration from: {}", + file_path.to_string_lossy() + ); + let content = fs::read_to_string(&file_path)?; + let config = Self::load(&content)?; + + Ok((config, file_path)) + } + + /// Load raw Kata configuration information from configuration files. + /// + /// If `config_file` is valid, it will used, otherwise a built-in default path list will be + /// scanned. + pub fn load_raw_from_file>(config_file: P) -> Result<(TomlConfig, PathBuf)> { + let file_path = if !config_file.as_ref().as_os_str().is_empty() { + fs::canonicalize(config_file)? + } else { + Self::get_default_config_file()? + }; + + info!( + sl!(), + "load configuration from: {}", + file_path.to_string_lossy() + ); + let content = fs::read_to_string(&file_path)?; + let config: TomlConfig = toml::from_str(&content)?; + + Ok((config, file_path)) + } + + /// Load Kata configuration information from string. + pub fn load(content: &str) -> Result { + let mut config: TomlConfig = toml::from_str(content)?; + + Runtime::adjust_configuration(&mut config)?; + info!(sl!(), "get kata config: {:?}", config); + + Ok(config) + } + + /// Validate Kata configuration information. + pub fn validate(&self) -> Result<()> { + Runtime::validate(self)?; + + Ok(()) + } + + /// Probe configuration file according to the default configuration file list. + fn get_default_config_file() -> Result { + for f in default::DEFAULT_RUNTIME_CONFIGURATIONS.iter() { + if let Ok(path) = fs::canonicalize(f) { + return Ok(path); + } + } + + Err(io::Error::from(io::ErrorKind::NotFound)) + } +} diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs new file mode 100644 index 000000000000..9784662f98f2 --- /dev/null +++ b/src/libs/kata-types/src/config/runtime.rs @@ -0,0 +1,272 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::Result; +use std::path::Path; + +use super::default; +use crate::config::{ConfigOps, TomlConfig}; +use crate::{eother, resolve_path, validate_path}; + +/// Kata runtime configuration information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Runtime { + /// If enabled, the runtime will log additional debug messages to the system log. + #[serde(default, rename = "enable_debug")] + pub debug: bool, + + /// Enabled experimental feature list, format: ["a", "b"]. + /// + /// Experimental features are features not stable enough for production, they may break + /// compatibility, and are prepared for a big version bump. + #[serde(default)] + pub experimental: Vec, + + /// Determines how the VM should be connected to the container network interface. + /// + /// Options: + /// - macvtap: used when the Container network interface can be bridged using macvtap. + /// - none: used when customize network. Only creates a tap device. No veth pair. + /// - tcfilter: uses tc filter rules to redirect traffic from the network interface provided + /// by plugin to a tap interface connected to the VM. + #[serde(default)] + pub internetworking_model: String, + + /// If enabled, the runtime won't create a network namespace for shim and hypervisor processes. + /// + /// This option may have some potential impacts to your host. It should only be used when you + /// know what you're doing. + /// + /// `disable_new_netns` conflicts with `internetworking_model=tcfilter` and + /// `internetworking_model=macvtap`. It works only with `internetworking_model=none`. + /// The tap device will be in the host network namespace and can connect to a bridge (like OVS) + /// directly. + /// + /// If you are using docker, `disable_new_netns` only works with `docker run --net=none` + #[serde(default)] + pub disable_new_netns: bool, + + /// If specified, sandbox_bind_mounts identifies host paths to be mounted into the sandboxes + /// shared path. + /// + /// This is only valid if filesystem sharing is utilized. The provided path(s) will be bind + /// mounted into the shared fs directory. If defaults are utilized, these mounts should be + /// available in the guest at `/run/kata-containers/shared/containers/passthrough/sandbox-mounts`. + /// These will not be exposed to the container workloads, and are only provided for potential + /// guest services. + #[serde(default)] + pub sandbox_bind_mounts: Vec, + + /// If enabled, the runtime will add all the kata processes inside one dedicated cgroup. + /// + /// The container cgroups in the host are not created, just one single cgroup per sandbox. + /// The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. + /// The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. + /// The sandbox cgroup is constrained if there is no container type annotation. + /// See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType + #[serde(default)] + pub sandbox_cgroup_only: bool, + + /// If enabled, the runtime will create opentracing.io traces and spans. + /// See https://www.jaegertracing.io/docs/getting-started. + #[serde(default)] + pub enable_tracing: bool, + /// The full url to the Jaeger HTTP Thrift collector. + #[serde(default)] + pub jaeger_endpoint: String, + /// The username to be used if basic auth is required for Jaeger. + #[serde(default)] + pub jaeger_user: String, + /// The password to be used if basic auth is required for Jaeger. + #[serde(default)] + pub jaeger_password: String, + + /// If enabled, user can run pprof tools with shim v2 process through kata-monitor. + #[serde(default)] + pub enable_pprof: bool, + + /// Determines whether container seccomp profiles are passed to the virtual machine and + /// applied by the kata agent. If set to true, seccomp is not applied within the guest. + #[serde(default)] + pub disable_guest_seccomp: bool, + + /// Determines how VFIO devices should be be presented to the container. + /// + /// Options: + /// - vfio: Matches behaviour of OCI runtimes (e.g. runc) as much as possible. VFIO devices + /// will appear in the container as VFIO character devices under /dev/vfio. The exact names + /// may differ from the host (they need to match the VM's IOMMU group numbers rather than + /// the host's) + /// - guest-kernel: This is a Kata-specific behaviour that's useful in certain cases. + /// The VFIO device is managed by whatever driver in the VM kernel claims it. This means + /// it will appear as one or more device nodes or network interfaces depending on the nature + /// of the device. Using this mode requires specially built workloads that know how to locate + /// the relevant device interfaces within the VM. + #[serde(default)] + pub vfio_mode: String, + + /// Vendor customized runtime configuration. + #[serde(default, flatten)] + pub vendor: RuntimeVendor, +} + +impl ConfigOps for Runtime { + fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { + RuntimeVendor::adjust_configuration(conf)?; + + if conf.runtime.internetworking_model.is_empty() { + conf.runtime.internetworking_model = default::DEFAULT_INTERNETWORKING_MODEL.to_owned(); + } + + for bind in conf.runtime.sandbox_bind_mounts.iter_mut() { + resolve_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?; + } + + Ok(()) + } + + fn validate(conf: &TomlConfig) -> Result<()> { + RuntimeVendor::validate(conf)?; + + let net_model = &conf.runtime.internetworking_model; + if !net_model.is_empty() + && net_model != "macvtap" + && net_model != "none" + && net_model != "tcfilter" + { + return Err(eother!( + "Invalid internetworking_model `{}` in configuration file", + net_model + )); + } + + let vfio_mode = &conf.runtime.vfio_mode; + if !vfio_mode.is_empty() && vfio_mode != "vfio" && vfio_mode != "guest-kernel" { + return Err(eother!( + "Invalid vfio_mode `{}` in configuration file", + vfio_mode + )); + } + + for bind in conf.runtime.sandbox_bind_mounts.iter() { + validate_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?; + } + + Ok(()) + } +} + +impl Runtime { + /// Check whether experiment `feature` is enabled or not. + pub fn is_experiment_enabled(&self, feature: &str) -> bool { + self.experimental.contains(&feature.to_string()) + } +} + +#[cfg(not(feature = "enable-vendor"))] +mod vendor { + use super::*; + + /// Vendor customization runtime configuration. + #[derive(Debug, Default, Deserialize, Serialize)] + pub struct RuntimeVendor {} + + impl ConfigOps for RuntimeVendor {} +} + +#[cfg(feature = "enable-vendor")] +#[path = "runtime_vendor.rs"] +mod vendor; + +pub use vendor::RuntimeVendor; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_invalid_config() { + let content = r#" +[runtime] +enable_debug = 10 +"#; + TomlConfig::load(content).unwrap_err(); + + let content = r#" +[runtime] +enable_debug = true +internetworking_model = "test" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap_err(); + + let content = r#" +[runtime] +enable_debug = true +internetworking_model = "macvtap,none" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap_err(); + + let content = r#" +[runtime] +enable_debug = true +vfio_mode = "none" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap_err(); + + let content = r#" +[runtime] +enable_debug = true +vfio_mode = "vfio,guest-kernel" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap_err(); + + let content = r#" +[runtime] +enable_debug = true +vfio_mode = "guest_kernel" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap_err(); + } + + #[test] + fn test_config() { + let content = r#" +[runtime] +enable_debug = true +experimental = ["a", "b"] +internetworking_model = "macvtap" +disable_new_netns = true +sandbox_bind_mounts = [] +sandbox_cgroup_only = true +enable_tracing = true +jaeger_endpoint = "localhost:1234" +jaeger_user = "user" +jaeger_password = "pw" +enable_pprof = true +disable_guest_seccomp = true +vfio_mode = "vfio" +field_should_be_ignored = true +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap(); + assert!(config.runtime.debug); + assert_eq!(config.runtime.experimental.len(), 2); + assert_eq!(&config.runtime.experimental[0], "a"); + assert_eq!(&config.runtime.experimental[1], "b"); + assert_eq!(&config.runtime.internetworking_model, "macvtap"); + assert!(config.runtime.disable_new_netns); + assert_eq!(config.runtime.sandbox_bind_mounts.len(), 0); + assert!(config.runtime.sandbox_cgroup_only); + assert!(config.runtime.enable_tracing); + assert!(config.runtime.is_experiment_enabled("a")); + assert!(config.runtime.is_experiment_enabled("b")); + assert!(!config.runtime.is_experiment_enabled("c")); + } +} diff --git a/src/libs/kata-types/src/config/runtime_vendor.rs b/src/libs/kata-types/src/config/runtime_vendor.rs new file mode 100644 index 000000000000..5981c7457029 --- /dev/null +++ b/src/libs/kata-types/src/config/runtime_vendor.rs @@ -0,0 +1,88 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! A sample for vendor to customize the runtime implementation. + +use super::*; +use crate::{eother, sl}; +use slog::Level; +/// Vendor customization runtime configuration. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct RuntimeVendor { + /// Log level + #[serde(default)] + pub log_level: u32, + + /// Prefix for log messages + #[serde(default)] + pub log_prefix: String, +} + +impl ConfigOps for RuntimeVendor { + fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { + if conf.runtime.vendor.log_level > Level::Debug as u32 { + conf.runtime.debug = true; + } + + Ok(()) + } + + /// Validate the configuration information. + fn validate(conf: &TomlConfig) -> Result<()> { + if conf.runtime.vendor.log_level > 10 { + warn!( + sl!(), + "log level {} in configuration file is invalid", conf.runtime.vendor.log_level + ); + return Err(eother!( + "log level {} in configuration file is invalid", + conf.runtime.vendor.log_level + )); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_invalid_vendor_config() { + let content = r#" +[runtime] +debug = false +log_level = 20 +log_prefix = "test" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap_err(); + + let content = r#" +[runtime] +debug = false +log_level = "test" +log_prefix = "test" +"#; + TomlConfig::load(content).unwrap_err(); + } + + #[test] + fn test_vendor_config() { + let content = r#" +[runtime] +debug = false +log_level = 10 +log_prefix = "test" +log_fmt = "nouse" +"#; + let config: TomlConfig = TomlConfig::load(content).unwrap(); + config.validate().unwrap(); + assert!(config.runtime.debug); + assert_eq!(config.runtime.vendor.log_level, 10); + assert_eq!(&config.runtime.vendor.log_prefix, "test"); + } +} diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs index 5fde5061509c..c05efd320792 100644 --- a/src/libs/kata-types/src/lib.rs +++ b/src/libs/kata-types/src/lib.rs @@ -5,11 +5,18 @@ //! Constants and Data Types shared by Kata Containers components. -#[deny(missing_docs)] +#![deny(missing_docs)] +#[macro_use] +extern crate slog; +#[macro_use] +extern crate serde; /// Constants and data types annotations. pub mod annotations; +/// Kata configuration information from configuration file. +pub mod config; + /// Constants and data types related to container. pub mod container; @@ -18,3 +25,56 @@ pub mod k8s; /// Constants and data types related to mount point. pub mod mount; + +/// Convenience macro to obtain the scoped logger +#[macro_export] +macro_rules! sl { + () => { + slog_scope::logger() + }; +} + +/// Helper to create std::io::Error(std::io::ErrorKind::Other) +#[macro_export] +macro_rules! eother { + () => (std::io::Error::new(std::io::ErrorKind::Other, "")); + ($fmt:expr) => ({ + std::io::Error::new(std::io::ErrorKind::Other, format!($fmt)) + }); + ($fmt:expr, $($arg:tt)*) => ({ + std::io::Error::new(std::io::ErrorKind::Other, format!($fmt, $($arg)*)) + }); +} + +/// Resolve a path to its final value. +#[macro_export] +macro_rules! resolve_path { + ($field:expr, $fmt:expr) => {{ + if !$field.is_empty() { + match Path::new(&$field).canonicalize() { + Err(e) => Err(eother!($fmt, &$field, e)), + Ok(path) => { + $field = path.to_string_lossy().to_string(); + Ok(()) + } + } + } else { + Ok(()) + } + }}; +} + +/// Validate a path. +#[macro_export] +macro_rules! validate_path { + ($field:expr, $fmt:expr) => {{ + if !$field.is_empty() { + Path::new(&$field) + .canonicalize() + .map_err(|e| eother!($fmt, &$field, e)) + .map(|_| ()) + } else { + Ok(()) + } + }}; +} From 69f10afb7130755a0b9a473ab52cf458deb9ecd4 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Sun, 26 Dec 2021 21:32:58 +0800 Subject: [PATCH 0011/1953] libs/types: support load Kata hypervisor configuration from file Add structures to load Kata hypevisor configuration from configuration files. Also define a mechanisms to: 1) for hypervisors to handle the configuration info. 2) for vendor to extend the Kata configuration structure. Signed-off-by: Liu Jiang Signed-off-by: Zhongtao Hu --- src/libs/Cargo.lock | 54 + src/libs/kata-types/Cargo.toml | 3 + src/libs/kata-types/src/config/default.rs | 37 + .../src/config/hypervisor/dragonball.rs | 176 +++ .../kata-types/src/config/hypervisor/mod.rs | 1052 +++++++++++++++++ .../kata-types/src/config/hypervisor/qemu.rs | 135 +++ .../src/config/hypervisor/vendor.rs | 14 + src/libs/kata-types/src/config/mod.rs | 72 +- src/libs/kata-types/tests/test-config.rs | 41 + .../tests/texture/configuration-qemu.toml | 78 ++ 10 files changed, 1661 insertions(+), 1 deletion(-) create mode 100644 src/libs/kata-types/src/config/hypervisor/dragonball.rs create mode 100644 src/libs/kata-types/src/config/hypervisor/mod.rs create mode 100644 src/libs/kata-types/src/config/hypervisor/qemu.rs create mode 100644 src/libs/kata-types/src/config/hypervisor/vendor.rs create mode 100644 src/libs/kata-types/tests/test-config.rs create mode 100644 src/libs/kata-types/tests/texture/configuration-qemu.toml diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index c9888c433d76..7a7ad017a49e 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" version = "1.0.57" @@ -225,6 +234,12 @@ dependencies = [ "slab", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -240,6 +255,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "indexmap" version = "1.8.1" @@ -287,8 +311,11 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" name = "kata-types" version = "0.1.0" dependencies = [ + "glob", "lazy_static", + "num_cpus", "oci", + "regex", "serde", "slog", "slog-scope", @@ -427,6 +454,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "oci" version = "0.1.0" @@ -584,6 +621,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + [[package]] name = "remove_dir_all" version = "0.5.3" diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 5c6035e36a85..ecae6f01845b 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -11,7 +11,10 @@ license = "Apache-2.0" edition = "2018" [dependencies] +glob = "0.3.0" lazy_static = "1.4.0" +num_cpus = "1.13.1" +regex = "1.5.4" serde = { version = "1.0.100", features = ["derive"] } slog = "2.5.2" slog-scope = "4.4.0" diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 4c501c06f598..722156206bf1 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -21,3 +21,40 @@ lazy_static! { } pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter"; + +pub const DEFAULT_BLOCK_DEVICE_TYPE: &str = "virtio-blk"; +pub const DEFAULT_VHOST_USER_STORE_PATH: &str = "/var/run/vhost-user"; +pub const DEFAULT_BLOCK_NVDIMM_MEM_OFFSET: u64 = 0; + +pub const DEFAULT_SHARED_FS_TYPE: &str = "virtio-9p"; +pub const DEFAULT_VIRTIO_FS_CACHE_MODE: &str = "none"; +pub const DEFAULT_VIRTIO_FS_DAX_SIZE_MB: u32 = 1024; +pub const DEFAULT_SHARED_9PFS_SIZE: u32 = 128 * 1024; +pub const MIN_SHARED_9PFS_SIZE: u32 = 4 * 1024; +pub const MAX_SHARED_9PFS_SIZE: u32 = 8 * 1024 * 1024; + +pub const DEFAULT_GUEST_HOOK_PATH: &str = "/opt"; + +pub const DEFAULT_GUEST_VCPUS: u32 = 1; + +// Default configuration for Dragonball +pub const DEFAULT_DB_GUEST_KENREL_IMAGE: &str = "vmlinuz"; +pub const DEFAULT_DB_GUEST_KENREL_PARAMS: &str = ""; +pub const DEFAULT_DB_ENTROPY_SOURCE: &str = "/dev/urandom"; +pub const DEFAULT_DB_MEMORY_SIZE: u32 = 128; +pub const DEFAULT_DB_MEMORY_SLOTS: u32 = 128; +pub const MAX_DB_VCPUS: u32 = 256; + +// Default configuration for qemu +pub const DEFAULT_QEMU_BINARY_PATH: &str = "qemu"; +pub const DEFAULT_QEMU_CONTROL_PATH: &str = ""; +pub const DEFAULT_QEMU_MACHINE_TYPE: &str = "q35"; +pub const DEFAULT_QEMU_ENTROPY_SOURCE: &str = "/dev/urandom"; +pub const DEFAULT_QEMU_GUEST_KENREL_IMAGE: &str = "vmlinuz"; +pub const DEFAULT_QEMU_GUEST_KENREL_PARAMS: &str = ""; +pub const DEFAULT_QEMU_FIRMWARE_PATH: &str = ""; +pub const DEFAULT_QEMU_MEMORY_SIZE: u32 = 128; +pub const DEFAULT_QEMU_MEMORY_SLOTS: u32 = 128; +pub const DEFAULT_QEMU_PCI_BRIDGES: u32 = 2; +pub const MAX_QEMU_PCI_BRIDGES: u32 = 5; +pub const MAX_QEMU_VCPUS: u32 = 256; diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs new file mode 100644 index 000000000000..8bde9cec7635 --- /dev/null +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -0,0 +1,176 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::Result; +use std::path::Path; +use std::sync::Arc; + +use super::{default, register_hypervisor_plugin}; +use crate::config::hypervisor::{ + VIRTIO_BLK, VIRTIO_BLK_MMIO, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM, +}; +use crate::config::{ConfigPlugin, TomlConfig}; +use crate::{eother, resolve_path, validate_path}; + +/// Hypervisor name for qemu, used to index `TomlConfig::hypervisor`. +pub const HYPERVISOR_NAME_DRAGONBALL: &str = "dragonball"; + +/// Configuration information for dragonball. +#[derive(Default, Debug)] +pub struct DragonballConfig {} + +impl DragonballConfig { + /// Create a new instance of `DragonballConfig`. + pub fn new() -> Self { + DragonballConfig {} + } + + /// Register the dragonball plugin. + pub fn register(self) { + let plugin = Arc::new(self); + register_hypervisor_plugin(HYPERVISOR_NAME_DRAGONBALL, plugin); + } +} + +impl ConfigPlugin for DragonballConfig { + fn name(&self) -> &str { + HYPERVISOR_NAME_DRAGONBALL + } + + /// Adjust the configuration information after loading from configuration file. + fn adjust_configuration(&self, conf: &mut TomlConfig) -> Result<()> { + if let Some(db) = conf.hypervisor.get_mut(HYPERVISOR_NAME_DRAGONBALL) { + resolve_path!(db.jailer_path, "Dragonball jailer path {} is invalid: {}")?; + + if db.boot_info.kernel.is_empty() { + db.boot_info.kernel = default::DEFAULT_DB_GUEST_KENREL_IMAGE.to_string(); + } + if db.boot_info.kernel_params.is_empty() { + db.boot_info.kernel_params = default::DEFAULT_DB_GUEST_KENREL_PARAMS.to_string(); + } + + if db.cpu_info.default_maxvcpus > default::MAX_DB_VCPUS { + db.cpu_info.default_maxvcpus = default::MAX_DB_VCPUS; + } + + if db.machine_info.entropy_source.is_empty() { + db.machine_info.entropy_source = default::DEFAULT_DB_ENTROPY_SOURCE.to_string(); + } + + if db.memory_info.default_memory == 0 { + db.memory_info.default_memory = default::DEFAULT_DB_MEMORY_SIZE; + } + if db.memory_info.memory_slots == 0 { + db.memory_info.memory_slots = default::DEFAULT_DB_MEMORY_SLOTS; + } + } + Ok(()) + } + + /// Validate the configuration information. + fn validate(&self, conf: &TomlConfig) -> Result<()> { + if let Some(db) = conf.hypervisor.get(HYPERVISOR_NAME_DRAGONBALL) { + if !db.path.is_empty() { + return Err(eother!("Path for dragonball hypervisor should be empty")); + } + if !db.valid_hypervisor_paths.is_empty() { + return Err(eother!( + "Valid hypervisor path for dragonball hypervisor should be empty" + )); + } + if !db.ctlpath.is_empty() { + return Err(eother!("CtlPath for dragonball hypervisor should be empty")); + } + if !db.valid_ctlpaths.is_empty() { + return Err(eother!("CtlPath for dragonball hypervisor should be empty")); + } + validate_path!(db.jailer_path, "Dragonball jailer path {} is invalid: {}")?; + if db.enable_iothreads { + return Err(eother!("Dragonball hypervisor doesn't support IO threads.")); + } + + if !db.blockdev_info.disable_block_device_use + && db.blockdev_info.block_device_driver != VIRTIO_BLK + && db.blockdev_info.block_device_driver != VIRTIO_BLK_MMIO + && db.blockdev_info.block_device_driver != VIRTIO_PMEM + { + return Err(eother!( + "{} is unsupported block device type.", + db.blockdev_info.block_device_driver + )); + } + + if db.boot_info.kernel.is_empty() { + return Err(eother!( + "Guest kernel image for dragonball hypervisor is empty" + )); + } + if db.boot_info.image.is_empty() { + return Err(eother!( + "Guest boot image for dragonball hypervisor is empty" + )); + } + if !db.boot_info.initrd.is_empty() { + return Err(eother!("Initrd for dragonball hypervisor should be empty")); + } + if !db.boot_info.firmware.is_empty() { + return Err(eother!( + "Firmware for dragonball hypervisor should be empty" + )); + } + + if (db.cpu_info.default_vcpus > 0 + && db.cpu_info.default_vcpus as u32 > default::MAX_DB_VCPUS) + || db.cpu_info.default_maxvcpus > default::MAX_DB_VCPUS + { + return Err(eother!( + "Dragonball hypervisor can not support {} vCPUs", + db.cpu_info.default_maxvcpus + )); + } + + if db.device_info.enable_iommu || db.device_info.enable_iommu_platform { + return Err(eother!("Dragonball hypervisor does not support vIOMMU")); + } + if db.device_info.hotplug_vfio_on_root_bus + || db.device_info.default_bridges > 0 + || db.device_info.pcie_root_port > 0 + { + return Err(eother!( + "Dragonball hypervisor does not support PCI hotplug options" + )); + } + + if !db.machine_info.machine_type.is_empty() { + return Err(eother!( + "Dragonball hypervisor does not support machine_type" + )); + } + if !db.machine_info.pflashes.is_empty() { + return Err(eother!("Dragonball hypervisor does not support pflashes")); + } + + if db.memory_info.enable_guest_swap { + return Err(eother!( + "Dragonball hypervisor doesn't support enable_guest_swap" + )); + } + + if db.security_info.rootless { + return Err(eother!( + "Dragonball hypervisor does not support rootless mode" + )); + } + + if let Some(v) = db.shared_fs.shared_fs.as_ref() { + if v != VIRTIO_FS && v != VIRTIO_FS_INLINE { + return Err(eother!("Dragonball hypervisor doesn't support {}", v)); + } + } + } + + Ok(()) + } +} diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs new file mode 100644 index 000000000000..23c74c971a60 --- /dev/null +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -0,0 +1,1052 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Configuration information for hypervisors. +//! +//! The configuration information for hypervisors is complex, and different hypervisor requires +//! different configuration information. To make it flexible and extensible, we build a multi-layer +//! architecture to manipulate hypervisor configuration information. +//! - the vendor layer. The `HypervisorVendor` structure provides hook points for vendors to +//! customize the configuration for its deployment. +//! - the hypervisor plugin layer. The hypervisor plugin layer provides hook points for different +//! hypervisors to manipulate the configuration information. +//! - the hypervisor common layer. This layer handles generic logic for all types of hypervisors. +//! +//! These three layers are applied in order. So changes made by the vendor layer will be visible +//! to the hypervisor plugin layer and the common layer. And changes made by the plugin layer will +//! only be visible to the common layer. +//! +//! Ideally the hypervisor configuration information should be split into hypervisor specific +//! part and common part. But the Kata 2.0 has adopted a policy to build a superset for all +//! hypervisors, so let's contain it... + +use std::collections::HashMap; +use std::io::Result; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use lazy_static::lazy_static; +use regex::RegexSet; + +use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; +use crate::{eother, resolve_path, validate_path}; + +mod dragonball; +pub use self::dragonball::{DragonballConfig, HYPERVISOR_NAME_DRAGONBALL}; + +mod qemu; +pub use self::qemu::{QemuConfig, HYPERVISOR_NAME_QEMU}; + +const VIRTIO_BLK: &str = "virtio-blk"; +const VIRTIO_BLK_MMIO: &str = "virtio-mmio"; +const VIRTIO_BLK_CCW: &str = "virtio-blk-ccw"; +const VIRTIO_SCSI: &str = "virtio-scsi"; +const VIRTIO_PMEM: &str = "nvdimm"; +const VIRTIO_9P: &str = "virtio-9p"; +const VIRTIO_FS: &str = "virtio-fs"; +const VIRTIO_FS_INLINE: &str = "inline-virtio-fs"; + +lazy_static! { + static ref HYPERVISOR_PLUGINS: Mutex>> = + Mutex::new(HashMap::new()); +} + +/// Register a hypervisor plugin with `name`. +pub fn register_hypervisor_plugin(name: &str, plugin: Arc) { + let mut hypervisors = HYPERVISOR_PLUGINS.lock().unwrap(); + hypervisors.insert(name.to_string(), plugin); +} + +/// Get the hypervisor plugin with `name`. +pub fn get_hypervisor_plugin(name: &str) -> Option> { + let hypervisors = HYPERVISOR_PLUGINS.lock().unwrap(); + hypervisors.get(name).cloned() +} + +/// Configuration information for block device. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct BlockDeviceInfo { + /// Disable block device from being used for a container's rootfs. + /// + /// In case of a storage driver like devicemapper where a container's root file system is + /// backed by a block device, the block device is passed directly to the hypervisor for + /// performance reasons. This flag prevents the block device from being passed to the + /// hypervisor, shared fs is used instead to pass the rootfs. + #[serde(default)] + pub disable_block_device_use: bool, + + /// Block storage driver to be used for the hypervisor in case the container rootfs is backed + /// by a block device. This is virtio-scsi, virtio-blk or nvdimm. + #[serde(default)] + pub block_device_driver: String, + + /// Specifies cache-related options will be set to block devices or not. + #[serde(default)] + pub block_device_cache_set: bool, + + /// Specifies cache-related options for block devices. + /// + /// Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. + #[serde(default)] + pub block_device_cache_direct: bool, + + /// Specifies cache-related options for block devices. + /// Denotes whether flush requests for the device are ignored. + #[serde(default)] + pub block_device_cache_noflush: bool, + + /// If false and nvdimm is supported, use nvdimm device to plug guest image. + #[serde(default)] + pub disable_image_nvdimm: bool, + + /// The size in MiB will be plused to max memory of hypervisor. + /// + /// It is the memory address space for the NVDIMM devie. If set block storage driver + /// (block_device_driver) to "nvdimm", should set memory_offset to the size of block device. + #[serde(default)] + pub memory_offset: u64, + + /// Enable vhost-user storage device, default false + /// + /// Enabling this will result in some Linux reserved block type major range 240-254 being + /// chosen to represent vhost-user devices. + #[serde(default)] + pub enable_vhost_user_store: bool, + + /// The base directory specifically used for vhost-user devices. + /// + /// Its sub-path "block" is used for block devices; "block/sockets" is where we expect + /// vhost-user sockets to live; "block/devices" is where simulated block device nodes for + /// vhost-user devices to live. + #[serde(default)] + pub vhost_user_store_path: String, + + /// List of valid annotations values for the vhost user store path. + /// + /// The default if not set is empty (all annotations rejected.) + #[serde(default)] + pub valid_vhost_user_store_paths: Vec, +} + +impl BlockDeviceInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + if self.disable_block_device_use { + self.block_device_driver = "".to_string(); + self.enable_vhost_user_store = false; + self.memory_offset = 0; + return Ok(()); + } + + if self.block_device_driver.is_empty() { + self.block_device_driver = default::DEFAULT_BLOCK_DEVICE_TYPE.to_string(); + } + if self.memory_offset == 0 { + self.memory_offset = default::DEFAULT_BLOCK_NVDIMM_MEM_OFFSET; + } + if !self.enable_vhost_user_store { + self.vhost_user_store_path = String::new(); + } else if self.vhost_user_store_path.is_empty() { + self.vhost_user_store_path = default::DEFAULT_VHOST_USER_STORE_PATH.to_string(); + } + resolve_path!( + self.vhost_user_store_path, + "Invalid vhost-user-store-path {}: {}" + )?; + + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + if self.disable_block_device_use { + return Ok(()); + } + if self.block_device_driver != VIRTIO_BLK + && self.block_device_driver != VIRTIO_BLK_CCW + && self.block_device_driver != VIRTIO_BLK_MMIO + && self.block_device_driver != VIRTIO_SCSI + && self.block_device_driver != VIRTIO_PMEM + { + return Err(eother!( + "{} is unsupported block device type.", + self.block_device_driver + )); + } + validate_path!( + self.vhost_user_store_path, + "Invalid vhost-user-store-path {}: {}" + )?; + + Ok(()) + } + + /// Validate path of vhost-user storage backend. + pub fn validate_vhost_user_store_path>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_vhost_user_store_paths, path) + } +} + +/// Guest kernel boot information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct BootInfo { + /// Path to guest kernel file on host + #[serde(default)] + pub kernel: String, + /// Guest kernel commandline. + #[serde(default)] + pub kernel_params: String, + /// Path to initrd file on host + #[serde(default)] + pub initrd: String, + /// Path to root device on host + #[serde(default)] + pub image: String, + /// Path to the firmware. + /// + /// If you want that qemu uses the default firmware leave this option empty. + #[serde(default)] + pub firmware: String, +} + +impl BootInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + resolve_path!(self.kernel, "guest kernel image file {} is invalid: {}")?; + resolve_path!(self.image, "guest boot image file {} is invalid: {}")?; + resolve_path!(self.initrd, "guest initrd image file {} is invalid: {}")?; + resolve_path!(self.firmware, "firmware image file {} is invalid: {}")?; + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + validate_path!(self.kernel, "guest kernel image file {} is invalid: {}")?; + validate_path!(self.image, "guest boot image file {} is invalid: {}")?; + validate_path!(self.initrd, "guest initrd image file {} is invalid: {}")?; + validate_path!(self.firmware, "firmware image file {} is invalid: {}")?; + if !self.image.is_empty() && !self.initrd.is_empty() { + return Err(eother!("Can not configure both initrd and image for boot")); + } + Ok(()) + } +} + +/// Virtual CPU configuration information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct CpuInfo { + /// CPU features, comma-separated list of cpu features to pass to the cpu. + /// For example, `cpu_features = "pmu=off,vmx=off" + #[serde(default)] + pub cpu_features: String, + + /// Default number of vCPUs per SB/VM: + /// - unspecified or 0 --> will be set to @DEFVCPUS@ + /// - < 0 --> will be set to the actual number of physical cores + /// > 0 <= number of physical cores --> will be set to the specified number + /// > number of physical cores --> will be set to the actual number of physical cores + #[serde(default)] + pub default_vcpus: i32, + + /// Default maximum number of vCPUs per SB/VM: + /// - unspecified or == 0 --> will be set to the actual number of physical cores or + /// to the maximum number of vCPUs supported by KVM + /// if that number is exceeded + /// - > 0 <= number of physical cores --> will be set to the specified number + /// - > number of physical cores --> will be set to the actual number of physical cores or + /// to the maximum number of vCPUs supported by KVM + /// if that number is exceeded + /// + /// WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used + /// when the actual number of physical cores is greater than it. + /// + /// WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU + /// the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 + /// vCPUs can be added to a SB/VM, but the memory footprint will be big. Another example, with + /// `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number + /// of vCPUs supported by the SB/VM. In general, we recommend that you do not edit this + /// variable, unless you know what are you doing. + /// + /// NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. + #[serde(default)] + pub default_maxvcpus: u32, +} + +impl CpuInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + let features: Vec<&str> = self.cpu_features.split(',').map(|v| v.trim()).collect(); + self.cpu_features = features.join(","); + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + Ok(()) + } + + /// Get default number of guest vCPUs. + pub fn get_default_vcpus(&self) -> u32 { + let cpus = num_cpus::get() as u32; + if self.default_vcpus < 0 || self.default_vcpus as u32 > cpus { + cpus + } else if self.default_vcpus == 0 { + default::DEFAULT_GUEST_VCPUS + } else { + self.default_vcpus as u32 + } + } + + /// Get default maximal number of guest vCPUs. + pub fn get_default_max_vcpus(&self) -> u32 { + let cpus = num_cpus::get() as u32; + if self.default_maxvcpus == 0 || self.default_maxvcpus > cpus { + cpus + } else { + self.default_maxvcpus + } + } +} + +/// Configuration information for shared filesystem, such virtio-9p and virtio-fs. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct DebugInfo { + /// This option changes the default hypervisor and kernel parameters to enable debug output + /// where available. + #[serde(default)] + pub enable_debug: bool, + + /// Enable dumping information about guest page structures if true. + #[serde(default)] + pub guest_memory_dump_paging: bool, + + /// Set where to save the guest memory dump file. + /// + /// If set, when GUEST_PANICKED event occurred, guest memory will be dumped to host filesystem + /// under guest_memory_dump_path. This directory will be created automatically if it does not + /// exist. The dumped file(also called vmcore) can be processed with crash or gdb. + /// + /// # WARNING: + /// Dump guest’s memory can take very long depending on the amount of guest memory and use + /// much disk space. + #[serde(default)] + pub guest_memory_dump_path: String, +} + +impl DebugInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + Ok(()) + } +} + +/// Virtual machine device configuration information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct DeviceInfo { + /// Bridges can be used to hot plug devices. + /// + /// Limitations: + /// - Currently only pci bridges are supported + /// - Until 30 devices per bridge can be hot plugged. + /// - Until 5 PCI bridges can be cold plugged per VM. + /// + /// This limitation could be a bug in qemu or in the kernel + /// Default number of bridges per SB/VM: + /// - unspecified or 0 --> will be set to @DEFBRIDGES@ + /// - > 1 <= 5 --> will be set to the specified number + /// - > 5 --> will be set to 5 + #[serde(default)] + pub default_bridges: u32, + + /// VFIO devices are hotplugged on a bridge by default. + /// + /// Enable hotplugging on root bus. This may be required for devices with a large PCI bar, + /// as this is a current limitation with hotplugging on a bridge. + #[serde(default)] + pub hotplug_vfio_on_root_bus: bool, + + /// Before hot plugging a PCIe device, you need to add a pcie_root_port device. + /// + /// Use this parameter when using some large PCI bar devices, such as Nvidia GPU. + /// The value means the number of pcie_root_port. + /// This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35" + #[serde(default)] + pub pcie_root_port: u32, + + /// Enable vIOMMU, default false + /// + /// Enabling this will result in the VM having a vIOMMU device. This will also add the + /// following options to the kernel's command line: intel_iommu=on,iommu=pt + #[serde(default)] + pub enable_iommu: bool, + + /// Enable IOMMU_PLATFORM, default false + /// + /// Enabling this will result in the VM device having iommu_platform=on set + #[serde(default)] + pub enable_iommu_platform: bool, +} + +impl DeviceInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + if self.default_bridges > 5 { + self.default_bridges = 5; + } + + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + if self.default_bridges > 5 { + return Err(eother!( + "The configured PCI bridges {} is too big", + self.default_bridges + )); + } + Ok(()) + } +} + +/// Configuration information for virtual machine. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct MachineInfo { + /// Virtual machine model/type. + #[serde(default)] + pub machine_type: String, + + /// Machine accelerators. + /// Comma-separated list of machine accelerators to pass to the hypervisor. + /// For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` + #[serde(default)] + pub machine_accelerators: String, + + /// Add flash image file to VM. + /// + /// The arguments of it should be in format of ["/path/to/flash0.img", "/path/to/flash1.img"]. + #[serde(default)] + pub pflashes: Vec, + + /// Default entropy source. + /// The path to a host source of entropy (including a real hardware RNG). + /// `/dev/urandom` and `/dev/random` are two main options. Be aware that `/dev/random` is a + /// blocking source of entropy. If the host runs out of entropy, the VMs boot time will + /// increase leading to get startup timeouts. The source of entropy `/dev/urandom` is + /// non-blocking and provides a generally acceptable source of entropy. It should work well + /// for pretty much all practical purposes. + #[serde(default)] + pub entropy_source: String, + + /// List of valid annotations values for entropy_source. + /// The default if not set is empty (all annotations rejected.) + #[serde(default)] + pub valid_entropy_sources: Vec, +} + +impl MachineInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + let accelerators: Vec<&str> = self + .machine_accelerators + .split(',') + .map(|v| v.trim()) + .collect(); + self.machine_accelerators = accelerators.join(","); + + for pflash in self.pflashes.iter_mut() { + resolve_path!(*pflash, "Flash image file {} is invalide: {}")?; + } + resolve_path!(self.entropy_source, "Entropy source {} is invalid: {}")?; + + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + for pflash in self.pflashes.iter() { + validate_path!(*pflash, "Flash image file {} is invalide: {}")?; + } + validate_path!(self.entropy_source, "Entropy source {} is invalid: {}")?; + Ok(()) + } + + /// Validate path of entropy source. + pub fn validate_entropy_source>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_entropy_sources, path) + } +} + +/// Virtual machine memory configuration information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct MemoryInfo { + /// Default memory size in MiB for SB/VM. + #[serde(default)] + pub default_memory: u32, + + /// Default memory slots per SB/VM. + /// + /// This is will determine the times that memory will be hotadded to sandbox/VM. + #[serde(default)] + pub memory_slots: u32, + + /// Enable file based guest memory support. + /// + /// The default is an empty string which will disable this feature. In the case of virtio-fs, + /// this is enabled automatically and '/dev/shm' is used as the backing folder. This option + /// will be ignored if VM templating is enabled. + #[serde(default)] + pub file_mem_backend: String, + + /// List of valid annotations values for the file_mem_backend annotation + /// + /// The default if not set is empty (all annotations rejected.) + #[serde(default)] + pub valid_file_mem_backends: Vec, + + /// Enable pre allocation of VM RAM, default false + /// + /// Enabling this will result in lower container density as all of the memory will be allocated + /// and locked. This is useful when you want to reserve all the memory upfront or in the cases + /// where you want memory latencies to be very predictable + #[serde(default)] + pub enable_mem_prealloc: bool, + + /// Enable huge pages for VM RAM, default false + /// + /// Enabling this will result in the VM memory being allocated using huge pages. This is useful + /// when you want to use vhost-user network stacks within the container. This will automatically + /// result in memory pre allocation. + #[serde(default)] + pub enable_hugepages: bool, + + /// Specifies virtio-mem will be enabled or not. + /// + /// Please note that this option should be used with the command + /// "echo 1 > /proc/sys/vm/overcommit_memory". + #[serde(default)] + pub enable_virtio_mem: bool, + + /// Enable swap of vm memory. Default false. + /// + /// The behaviour is undefined if mem_prealloc is also set to true + #[serde(default)] + pub enable_swap: bool, + + /// Enable swap in the guest. Default false. + /// + /// When enable_guest_swap is enabled, insert a raw file to the guest as the swap device if the + /// swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness") + /// is bigger than 0. + /// + /// The size of the swap device should be swap_in_bytes (set by annotation + /// "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes. + /// If swap_in_bytes is not set, the size should be memory_limit_in_bytes. + /// If swap_in_bytes and memory_limit_in_bytes is not set, the size should be default_memory. + #[serde(default)] + pub enable_guest_swap: bool, +} + +impl MemoryInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + resolve_path!( + self.file_mem_backend, + "Memory backend file {} is invalid: {}" + )?; + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + validate_path!( + self.file_mem_backend, + "Memory backend file {} is invalid: {}" + )?; + if self.default_memory == 0 { + return Err(eother!("Configured memory size for guest vm is zero")); + } + if self.memory_slots == 0 { + return Err(eother!("Configured memory slots for guest vm is zero")); + } + + Ok(()) + } + + /// Validate path of memory backend files. + pub fn validate_memory_backend_path>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_file_mem_backends, path) + } +} + +/// Configuration information for virtual machine. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct NetworkInfo { + /// If vhost-net backend for virtio-net is not desired, set to true. + /// + /// Default is false, which trades off security (vhost-net runs ring0) for network I/O + /// performance. + #[serde(default)] + pub disable_vhost_net: bool, + + /// Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). + /// + /// In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. + /// Default 0-sized value means unlimited rate. + #[serde(default)] + pub rx_rate_limiter_max_rate: u64, + + /// Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). + /// + /// In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional + /// Block) to discipline traffic. + /// Default 0-sized value means unlimited rate. + #[serde(default)] + pub tx_rate_limiter_max_rate: u64, +} + +impl NetworkInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + Ok(()) + } +} + +/// Configuration information for virtual machine. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct SecurityInfo { + /// Enable running QEMU VMM as a non-root user. + /// + /// By default QEMU VMM run as root. When this is set to true, QEMU VMM process runs as + /// a non-root random user. See documentation for the limitations of this mode. + #[serde(default)] + pub rootless: bool, + + /// Disable seccomp. + #[serde(default)] + pub disable_seccomp: bool, + + /// Enable confidential guest support. + /// + /// Toggling that setting may trigger different hardware features, ranging from memory + /// encryption to both memory and CPU-state encryption and integrity.The Kata Containers + /// runtime dynamically detects the available feature set and aims at enabling the largest + /// possible one. + #[serde(default)] + pub confidential_guest: bool, + + /// Path to OCI hook binaries in the *guest rootfs*. + /// + /// This does not affect host-side hooks which must instead be added to the OCI spec passed to + /// the runtime. + /// + /// You can create a rootfs with hooks by customizing the osbuilder scripts: + /// https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder + /// + /// Hooks must be stored in a subdirectory of guest_hook_path according to their hook type, + /// i.e. "guest_hook_path/{prestart,poststart,poststop}". The agent will scan these directories + /// for executable files and add them, in lexicographical order, to the lifecycle of the guest + /// container. + /// + /// Hooks are executed in the runtime namespace of the guest. See the official documentation: + /// https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks + /// + /// Warnings will be logged if any error is encountered while scanning for hooks, but it will + /// not abort container execution. + #[serde(default)] + pub guest_hook_path: String, + + /// List of valid annotation names for the hypervisor. + /// + /// Each member of the list is a regular expression, which is the base name of the annotation, + /// e.g. "path" for io.katacontainers.config.hypervisor.path" + #[serde(default)] + pub enable_annotations: Vec, +} + +impl SecurityInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + if self.guest_hook_path.is_empty() { + self.guest_hook_path = default::DEFAULT_GUEST_HOOK_PATH.to_string(); + } + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + Ok(()) + } + + /// Check whether annotation key is enabled or not. + pub fn is_annotation_enabled(&self, path: &str) -> bool { + if !path.starts_with("io.katacontainers.config.hypervisor.") { + return false; + } + let pos = "io.katacontainers.config.hypervisor.".len(); + let key = &path[pos..]; + if let Ok(set) = RegexSet::new(&self.enable_annotations) { + return set.is_match(key); + } + + false + } +} + +/// Configuration information for shared filesystem, such virtio-9p and virtio-fs. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct SharedFsInfo { + /// Shared file system type: + /// - virtio-fs (default) + /// - virtio-9p` + pub shared_fs: Option, + + /// Path to vhost-user-fs daemon. + #[serde(default)] + pub virtio_fs_daemon: String, + + /// List of valid annotations values for the virtiofs daemon + /// The default if not set is empty (all annotations rejected.) + #[serde(default)] + pub valid_virtio_fs_daemon_paths: Vec, + + /// Extra args for virtiofsd daemon + /// + /// Format example: + /// ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] + /// + /// see `virtiofsd -h` for possible options. + #[serde(default)] + pub virtio_fs_extra_args: Vec, + + /// Cache mode: + /// - none: Metadata, data, and pathname lookup are not cached in guest. They are always + /// fetched from host and any changes are immediately pushed to host. + /// - auto: Metadata and pathname lookup cache expires after a configured amount of time + /// (default is 1 second). Data is cached while the file is open (close to open consistency). + /// - always: Metadata, data, and pathname lookup are cached in guest and never expire. + #[serde(default)] + pub virtio_fs_cache: String, + + /// Default size of DAX cache in MiB + #[serde(default)] + pub virtio_fs_cache_size: u32, + + /// Enable virtio-fs DAX window if true. + #[serde(default)] + pub virtio_fs_is_dax: bool, + + /// This is the msize used for 9p shares. It is the number of bytes used for 9p packet payload. + #[serde(default)] + pub msize_9p: u32, +} + +impl SharedFsInfo { + /// Adjust the configuration information after loading from configuration file. + pub fn adjust_configuration(&mut self) -> Result<()> { + if self.shared_fs.as_deref() == Some("") { + self.shared_fs = Some(default::DEFAULT_SHARED_FS_TYPE.to_string()); + } + match self.shared_fs.as_deref() { + Some(VIRTIO_FS) => self.adjust_virtio_fs(false)?, + Some(VIRTIO_FS_INLINE) => self.adjust_virtio_fs(true)?, + Some(VIRTIO_9P) => { + if self.msize_9p == 0 { + self.msize_9p = default::DEFAULT_SHARED_9PFS_SIZE; + } + } + _ => {} + } + + Ok(()) + } + + /// Validate the configuration information. + pub fn validate(&self) -> Result<()> { + match self.shared_fs.as_deref() { + None => Ok(()), + Some(VIRTIO_FS) => self.validate_virtio_fs(false), + Some(VIRTIO_FS_INLINE) => self.validate_virtio_fs(true), + Some(VIRTIO_9P) => { + if self.msize_9p < default::MIN_SHARED_9PFS_SIZE + || self.msize_9p > default::MAX_SHARED_9PFS_SIZE + { + return Err(eother!( + "Invalid 9p configuration msize 0x{:x}", + self.msize_9p + )); + } + Ok(()) + } + Some(v) => Err(eother!("Invalid shared_fs type {}", v)), + } + } + + /// Validate path of virtio-fs daemon, especially for annotations. + pub fn validate_virtiofs_daemon_path>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_virtio_fs_daemon_paths, path) + } + + fn adjust_virtio_fs(&mut self, _inline: bool) -> Result<()> { + resolve_path!( + self.virtio_fs_daemon, + "Virtio-fs daemon path {} is invalid: {}" + )?; + if self.virtio_fs_cache.is_empty() { + self.virtio_fs_cache = default::DEFAULT_VIRTIO_FS_CACHE_MODE.to_string(); + } + if self.virtio_fs_is_dax && self.virtio_fs_cache_size == 0 { + self.virtio_fs_cache_size = default::DEFAULT_VIRTIO_FS_DAX_SIZE_MB; + } + if !self.virtio_fs_is_dax && self.virtio_fs_cache_size != 0 { + self.virtio_fs_is_dax = true; + } + Ok(()) + } + + fn validate_virtio_fs(&self, inline: bool) -> Result<()> { + if inline && !self.virtio_fs_daemon.is_empty() { + return Err(eother!( + "Executable path for inline-virtio-fs is not empty: {}", + &self.virtio_fs_daemon + )); + } + validate_path!( + self.virtio_fs_daemon, + "Virtio-fs daemon path {} is invalid: {}" + )?; + + if self.virtio_fs_cache != "none" + && self.virtio_fs_cache != "auto" + && self.virtio_fs_cache != "always" + { + return Err(eother!( + "Invalid virtio-fs cache mode: {}", + &self.virtio_fs_cache + )); + } + if self.virtio_fs_is_dax && self.virtio_fs_cache_size == 0 { + return Err(eother!( + "Invalid virtio-fs DAX window size: {}", + &self.virtio_fs_cache_size + )); + } + Ok(()) + } +} + +/// Common configuration information for hypervisors. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Hypervisor { + /// Path to the hypervisor executable. + #[serde(default)] + pub path: String, + /// List of valid annotations values for the hypervisor. + /// + /// Each member of the list is a path pattern as described by glob(3). The default if not set + /// is empty (all annotations rejected.) + #[serde(default)] + pub valid_hypervisor_paths: Vec, + + /// Hypervisor control executable path. + #[serde(default)] + pub ctlpath: String, + /// List of valid annotations values for the hypervisor control executable. + /// + /// Each member of the list is a path pattern as described by glob(3). The default if not set + /// is empty (all annotations rejected.) + #[serde(default)] + pub valid_ctlpaths: Vec, + + /// Control channel path. + #[serde(default)] + pub jailer_path: String, + /// List of valid annotations values for the hypervisor jailer path. + /// + /// Each member of the list is a path pattern as described by glob(3). The default if not set + /// is empty (all annotations rejected.) + #[serde(default)] + pub valid_jailer_paths: Vec, + + /// Disable the customizations done in the runtime when it detects that it is running on top + /// a VMM. This will result in the runtime behaving as it would when running on bare metal. + #[serde(default)] + pub disable_nesting_checks: bool, + + /// Enable iothreads (data-plane) to be used. This causes IO to be handled in a separate IO + /// thread. This is currently only implemented for SCSI. + #[serde(default)] + pub enable_iothreads: bool, + + /// Block device configuration information. + #[serde(default, flatten)] + pub blockdev_info: BlockDeviceInfo, + + /// Guest system boot information. + #[serde(default, flatten)] + pub boot_info: BootInfo, + + /// Guest virtual CPU configuration information. + #[serde(default, flatten)] + pub cpu_info: CpuInfo, + + /// Debug configuration information. + #[serde(default, flatten)] + pub debug_info: DebugInfo, + + /// Device configuration information. + #[serde(default, flatten)] + pub device_info: DeviceInfo, + + /// Virtual machine configuration information. + #[serde(default, flatten)] + pub machine_info: MachineInfo, + + /// Virtual machine memory configuration information. + #[serde(default, flatten)] + pub memory_info: MemoryInfo, + + /// Network configuration information. + #[serde(default, flatten)] + pub network_info: NetworkInfo, + + /// Security configuration information. + #[serde(default, flatten)] + pub security_info: SecurityInfo, + + /// Shared file system configuration information. + #[serde(default, flatten)] + pub shared_fs: SharedFsInfo, + + /// Vendor customized runtime configuration. + #[serde(default, flatten)] + pub vendor: HypervisorVendor, +} + +impl Hypervisor { + /// Validate path of hypervisor executable. + pub fn validate_hypervisor_path>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_hypervisor_paths, path) + } + + /// Validate path of hypervisor control executable. + pub fn validate_hypervisor_ctlpath>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_ctlpaths, path) + } + + /// Validate path of jailer executable. + pub fn validate_jailer_path>(&self, path: P) -> Result<()> { + validate_path_pattern(&self.valid_jailer_paths, path) + } +} + +impl ConfigOps for Hypervisor { + fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { + HypervisorVendor::adjust_configuration(conf)?; + + let hypervisors: Vec = conf.hypervisor.keys().cloned().collect(); + for hypervisor in hypervisors.iter() { + if let Some(plugin) = get_hypervisor_plugin(hypervisor) { + plugin.adjust_configuration(conf)?; + + // Safe to unwrap() because `hypervisor` is a valid key in the hash map. + let hv = conf.hypervisor.get_mut(hypervisor).unwrap(); + hv.blockdev_info.adjust_configuration()?; + hv.boot_info.adjust_configuration()?; + hv.cpu_info.adjust_configuration()?; + hv.debug_info.adjust_configuration()?; + hv.device_info.adjust_configuration()?; + hv.machine_info.adjust_configuration()?; + hv.memory_info.adjust_configuration()?; + hv.network_info.adjust_configuration()?; + hv.security_info.adjust_configuration()?; + hv.shared_fs.adjust_configuration()?; + } else { + return Err(eother!("Can not find plugin for hypervisor {}", hypervisor)); + } + } + + Ok(()) + } + + fn validate(conf: &TomlConfig) -> Result<()> { + HypervisorVendor::validate(conf)?; + + let hypervisors: Vec = conf.hypervisor.keys().cloned().collect(); + for hypervisor in hypervisors.iter() { + if let Some(plugin) = get_hypervisor_plugin(hypervisor) { + plugin.validate(conf)?; + + // Safe to unwrap() because `hypervisor` is a valid key in the hash map. + let hv = conf.hypervisor.get(hypervisor).unwrap(); + hv.blockdev_info.validate()?; + hv.boot_info.validate()?; + hv.cpu_info.validate()?; + hv.debug_info.validate()?; + hv.device_info.validate()?; + hv.machine_info.validate()?; + hv.memory_info.validate()?; + hv.network_info.validate()?; + hv.security_info.validate()?; + hv.shared_fs.validate()?; + validate_path!(hv.path, "Hypervisor binary path `{}` is invalid: {}")?; + validate_path!( + hv.ctlpath, + "Hypervisor control executable `{}` is invalid: {}" + )?; + validate_path!(hv.jailer_path, "Hypervisor jailer path `{}` is invalid: {}")?; + } else { + return Err(eother!("Can not find plugin for hypervisor {}", hypervisor)); + } + } + + Ok(()) + } +} + +#[cfg(not(feature = "enable-vendor"))] +mod vendor { + use super::*; + + /// Vendor customization runtime configuration. + #[derive(Debug, Default, Deserialize, Serialize)] + pub struct HypervisorVendor {} + + impl ConfigOps for HypervisorVendor {} +} + +#[cfg(feature = "enable-vendor")] +#[path = "vendor.rs"] +mod vendor; + +use crate::config::validate_path_pattern; +pub use vendor::HypervisorVendor; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_register_plugin() { + let db = DragonballConfig::new(); + db.register(); + + let db = Arc::new(DragonballConfig::new()); + register_hypervisor_plugin("dragonball", db); + + assert!(get_hypervisor_plugin("dragonball").is_some()); + assert!(get_hypervisor_plugin("dragonball2").is_none()); + } +} diff --git a/src/libs/kata-types/src/config/hypervisor/qemu.rs b/src/libs/kata-types/src/config/hypervisor/qemu.rs new file mode 100644 index 000000000000..87b4a11bef2f --- /dev/null +++ b/src/libs/kata-types/src/config/hypervisor/qemu.rs @@ -0,0 +1,135 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::Result; +use std::path::Path; +use std::sync::Arc; + +use super::{default, register_hypervisor_plugin}; +use crate::config::hypervisor::VIRTIO_BLK_MMIO; +use crate::config::{ConfigPlugin, TomlConfig}; +use crate::{eother, resolve_path, validate_path}; + +/// Hypervisor name for qemu, used to index `TomlConfig::hypervisor`. +pub const HYPERVISOR_NAME_QEMU: &str = "qemu"; + +/// Configuration information for qemu. +#[derive(Default, Debug)] +pub struct QemuConfig {} + +impl QemuConfig { + /// Create a new instance of `QemuConfig`. + pub fn new() -> Self { + QemuConfig {} + } + + /// Register the qemu plugin. + pub fn register(self) { + let plugin = Arc::new(self); + register_hypervisor_plugin(HYPERVISOR_NAME_QEMU, plugin); + } +} + +impl ConfigPlugin for QemuConfig { + fn name(&self) -> &str { + HYPERVISOR_NAME_QEMU + } + + /// Adjust the configuration information after loading from configuration file. + fn adjust_configuration(&self, conf: &mut TomlConfig) -> Result<()> { + if let Some(qemu) = conf.hypervisor.get_mut(HYPERVISOR_NAME_QEMU) { + if qemu.path.is_empty() { + qemu.path = default::DEFAULT_QEMU_BINARY_PATH.to_string(); + } + resolve_path!(qemu.path, "Qemu binary path `{}` is invalid: {}")?; + if qemu.ctlpath.is_empty() { + qemu.ctlpath = default::DEFAULT_QEMU_CONTROL_PATH.to_string(); + } + resolve_path!(qemu.ctlpath, "Qemu ctlpath `{}` is invalid: {}")?; + + if qemu.boot_info.kernel.is_empty() { + qemu.boot_info.kernel = default::DEFAULT_QEMU_GUEST_KENREL_IMAGE.to_string(); + } + if qemu.boot_info.kernel_params.is_empty() { + qemu.boot_info.kernel_params = + default::DEFAULT_QEMU_GUEST_KENREL_PARAMS.to_string(); + } + if qemu.boot_info.firmware.is_empty() { + qemu.boot_info.firmware = default::DEFAULT_QEMU_FIRMWARE_PATH.to_string(); + } + + if qemu.device_info.default_bridges == 0 { + qemu.device_info.default_bridges = default::DEFAULT_QEMU_PCI_BRIDGES; + if qemu.device_info.default_bridges > default::MAX_QEMU_PCI_BRIDGES { + qemu.device_info.default_bridges = default::MAX_QEMU_PCI_BRIDGES; + } + } + + if qemu.machine_info.machine_type.is_empty() { + qemu.machine_info.machine_type = default::DEFAULT_QEMU_MACHINE_TYPE.to_string(); + } + if qemu.machine_info.entropy_source.is_empty() { + qemu.machine_info.entropy_source = default::DEFAULT_QEMU_ENTROPY_SOURCE.to_string(); + } + + if qemu.memory_info.default_memory == 0 { + qemu.memory_info.default_memory = default::DEFAULT_QEMU_MEMORY_SIZE; + } + if qemu.memory_info.memory_slots == 0 { + qemu.memory_info.memory_slots = default::DEFAULT_QEMU_MEMORY_SLOTS; + } + } + + Ok(()) + } + + /// Validate the configuration information. + fn validate(&self, conf: &TomlConfig) -> Result<()> { + if let Some(qemu) = conf.hypervisor.get(HYPERVISOR_NAME_QEMU) { + validate_path!(qemu.path, "Qemu binary path `{}` is invalid: {}")?; + validate_path!(qemu.ctlpath, "Qemu control path `{}` is invalid: {}")?; + if !qemu.jailer_path.is_empty() { + return Err(eother!("Path for Qemu jailer should be empty")); + } + if !qemu.valid_jailer_paths.is_empty() { + return Err(eother!("Valid Qemu jailer path list should be empty")); + } + + if !qemu.blockdev_info.disable_block_device_use + && qemu.blockdev_info.block_device_driver == VIRTIO_BLK_MMIO + { + return Err(eother!("Qemu doesn't support virtio-blk-mmio")); + } + + if qemu.boot_info.kernel.is_empty() { + return Err(eother!("Guest kernel image for qemu is empty")); + } + if qemu.boot_info.image.is_empty() && qemu.boot_info.initrd.is_empty() { + return Err(eother!( + "Both guest boot image and initrd for qemu are empty" + )); + } + + if (qemu.cpu_info.default_vcpus > 0 + && qemu.cpu_info.default_vcpus as u32 > default::MAX_QEMU_VCPUS) + || qemu.cpu_info.default_maxvcpus > default::MAX_QEMU_VCPUS + { + return Err(eother!( + "Qemu hypervisor can not support {} vCPUs", + qemu.cpu_info.default_maxvcpus + )); + } + + if qemu.device_info.default_bridges > default::MAX_QEMU_PCI_BRIDGES { + return Err(eother!( + "Qemu hypervisor can not support {} PCI bridges", + qemu.device_info.default_bridges + )); + } + } + + Ok(()) + } +} diff --git a/src/libs/kata-types/src/config/hypervisor/vendor.rs b/src/libs/kata-types/src/config/hypervisor/vendor.rs new file mode 100644 index 000000000000..9b51d1016564 --- /dev/null +++ b/src/libs/kata-types/src/config/hypervisor/vendor.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! A sample for vendor to customize the hypervisor implementation. + +use super::*; + +/// Vendor customization runtime configuration. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct HypervisorVendor {} + +impl ConfigOps for HypervisorVendor {} diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 5d13263016a6..70fd81a5d8d0 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -4,19 +4,38 @@ // SPDX-License-Identifier: Apache-2.0 // +use std::collections::HashMap; use std::fs; use std::io::{self, Result}; use std::path::{Path, PathBuf}; -use crate::sl; +use crate::{eother, sl}; /// Default configuration values. pub mod default; +mod hypervisor; +pub use self::hypervisor::{ + BootInfo, DragonballConfig, Hypervisor, QemuConfig, HYPERVISOR_NAME_DRAGONBALL, + HYPERVISOR_NAME_QEMU, +}; + mod runtime; pub use self::runtime::{Runtime, RuntimeVendor}; /// Trait to manipulate global Kata configuration information. +pub trait ConfigPlugin: Send + Sync { + /// Get the plugin name. + fn name(&self) -> &str; + + /// Adjust the configuration information after loading from configuration file. + fn adjust_configuration(&self, _conf: &mut TomlConfig) -> Result<()>; + + /// Validate the configuration information. + fn validate(&self, _conf: &TomlConfig) -> Result<()>; +} + +/// Trait to manipulate Kata configuration information. pub trait ConfigOps { /// Adjust the configuration information after loading from configuration file. fn adjust_configuration(_conf: &mut TomlConfig) -> Result<()> { @@ -45,6 +64,9 @@ pub trait ConfigObjectOps { /// Kata configuration information. #[derive(Debug, Default, Deserialize, Serialize)] pub struct TomlConfig { + /// Configuration information for hypervisors. + #[serde(default)] + pub hypervisor: HashMap, /// Kata runtime configuration information. #[serde(default)] pub runtime: Runtime, @@ -99,6 +121,7 @@ impl TomlConfig { pub fn load(content: &str) -> Result { let mut config: TomlConfig = toml::from_str(content)?; + Hypervisor::adjust_configuration(&mut config)?; Runtime::adjust_configuration(&mut config)?; info!(sl!(), "get kata config: {:?}", config); @@ -107,6 +130,7 @@ impl TomlConfig { /// Validate Kata configuration information. pub fn validate(&self) -> Result<()> { + Hypervisor::validate(self)?; Runtime::validate(self)?; Ok(()) @@ -123,3 +147,49 @@ impl TomlConfig { Err(io::Error::from(io::ErrorKind::NotFound)) } } + +/// Validate the `path` matches one of the pattern in `patterns`. +/// +/// Each member in `patterns` is a path pattern as described by glob(3) +pub fn validate_path_pattern>(patterns: &[String], path: P) -> Result<()> { + let path = path + .as_ref() + .to_str() + .ok_or_else(|| eother!("Invalid path {}", path.as_ref().to_string_lossy()))?; + + for p in patterns.iter() { + if let Ok(glob) = glob::Pattern::new(p) { + if glob.matches(path) { + return Ok(()); + } + } + } + + Err(eother!("Path {} is not permitted", path)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_validate_path_pattern() { + let patterns = []; + validate_path_pattern(&patterns, "/bin/ls").unwrap_err(); + + let patterns = ["/bin".to_string()]; + validate_path_pattern(&patterns, "/bin/ls").unwrap_err(); + + let patterns = ["/bin/*/ls".to_string()]; + validate_path_pattern(&patterns, "/bin/ls").unwrap_err(); + + let patterns = ["/bin/*".to_string()]; + validate_path_pattern(&patterns, "/bin/ls").unwrap(); + + let patterns = ["/*".to_string()]; + validate_path_pattern(&patterns, "/bin/ls").unwrap(); + + let patterns = ["/usr/share".to_string(), "/bin/*".to_string()]; + validate_path_pattern(&patterns, "/bin/ls").unwrap(); + } +} diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test-config.rs new file mode 100644 index 000000000000..e88a250469dd --- /dev/null +++ b/src/libs/kata-types/tests/test-config.rs @@ -0,0 +1,41 @@ +use kata_types::config::{QemuConfig, TomlConfig, HYPERVISOR_NAME_QEMU}; +use std::fs; +use std::path::Path; + +#[test] +fn test_load_qemu_config() { + let plugin = QemuConfig::new(); + plugin.register(); + + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-qemu.toml"); + let content = fs::read_to_string(&path).unwrap(); + let config = TomlConfig::load(&content).unwrap(); + + let qemu = config.hypervisor.get(HYPERVISOR_NAME_QEMU).unwrap(); + assert_eq!(qemu.path, "/usr/bin/ls"); + assert_eq!(qemu.valid_hypervisor_paths.len(), 2); + assert_eq!(qemu.valid_hypervisor_paths[0], "/usr/bin/qemu*"); + assert_eq!(qemu.valid_hypervisor_paths[1], "/opt/qemu?"); + qemu.validate_hypervisor_path("/usr/bin/qemu0").unwrap(); + qemu.validate_hypervisor_path("/usr/bin/qemu1").unwrap(); + qemu.validate_hypervisor_path("/usr/bin/qemu2222").unwrap(); + qemu.validate_hypervisor_path("/opt/qemu3").unwrap(); + qemu.validate_hypervisor_path("/opt/qemu").unwrap_err(); + qemu.validate_hypervisor_path("/opt/qemu33").unwrap_err(); + assert_eq!(qemu.ctlpath, "/usr/bin/ls"); + assert_eq!(qemu.valid_ctlpaths.len(), 0); + assert!(qemu.jailer_path.is_empty()); + assert_eq!(qemu.valid_jailer_paths.len(), 0); + assert_eq!(qemu.disable_nesting_checks, true); + assert_eq!(qemu.enable_iothreads, true); + + assert_eq!(qemu.boot_info.image, "/usr/bin/echo"); + assert_eq!(qemu.boot_info.kernel, "/usr/bin/id"); + assert_eq!(qemu.boot_info.kernel_params, "ro"); + assert_eq!(qemu.boot_info.firmware, "/etc/hostname"); + + assert_eq!(qemu.cpu_info.cpu_features, "pmu=off,vmx=off"); + assert_eq!(qemu.cpu_info.default_vcpus, 2); + assert_eq!(qemu.cpu_info.default_maxvcpus, 64); +} diff --git a/src/libs/kata-types/tests/texture/configuration-qemu.toml b/src/libs/kata-types/tests/texture/configuration-qemu.toml new file mode 100644 index 000000000000..d30e0858a8d0 --- /dev/null +++ b/src/libs/kata-types/tests/texture/configuration-qemu.toml @@ -0,0 +1,78 @@ +[hypervisor.qemu] +path = "/usr/bin/ls" +valid_hypervisor_paths = ["/usr/bin/qemu*", "/opt/qemu?"] +ctlpath = "/usr/bin/ls" +disable_nesting_checks = true +enable_iothreads = true + +kernel = "/usr/bin/../bin/id" +image = "/usr/bin/./echo" +kernel_params = "ro" +firmware = "/etc/hostname" + +cpu_features="pmu=off,vmx=off" +default_vcpus = 2 +default_maxvcpus = 64 + +machine_type = "q35" +confidential_guest = true +rootless = true +enable_annotations = ["path", "ctlpath"] +machine_accelerators="noapic" +default_bridges = 2 +default_memory = 128 +memory_slots = 128 +memory_offset = 0x100000 +enable_virtio_mem = true +disable_block_device_use = false +shared_fs = "virtio-fs" +virtio_fs_daemon = "/usr/bin/id" +valid_virtio_fs_daemon_paths = ["/usr/local/bin/virtiofsd*"] +virtio_fs_cache_size = 512 +virtio_fs_extra_args = ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +virtio_fs_cache = "always" +block_device_driver = "virtio-blk" +block_device_cache_set = true +block_device_cache_direct = true +block_device_cache_noflush = true +enable_mem_prealloc = true +enable_hugepages = true +enable_vhost_user_store = true +vhost_user_store_path = "/tmp" +valid_vhost_user_store_paths = ["/var/kata/vhost-user-store*", "/tmp/kata?"] +enable_iommu = true +enable_iommu_platform = true +file_mem_backend = "/dev/shm" +valid_file_mem_backends = ["/dev/shm"] +enable_swap = true +pflashes = ["/proc/mounts"] +enable_debug = true +msize_9p = 16384 +disable_image_nvdimm = true +hotplug_vfio_on_root_bus = true +pcie_root_port = 2 +disable_vhost_net = true +entropy_source= "/dev/urandom" +valid_entropy_sources = ["/dev/urandom", "/dev/random"] +guest_hook_path = "/usr/share/oci/hooks" +rx_rate_limiter_max_rate = 10000 +tx_rate_limiter_max_rate = 10000 +guest_memory_dump_path="/var/crash/kata" +guest_memory_dump_paging = true +enable_guest_swap = true + +[runtime] +enable_debug = true +internetworking_model="macvtap" +disable_guest_seccomp=true +enable_tracing = true +jaeger_endpoint = "localhost:1234" +jaeger_user = "user" +jaeger_password = "pw" +disable_new_netns = true +sandbox_cgroup_only=true +sandbox_bind_mounts=["/proc/self"] +vfio_mode="vfio" +experimental=["a", "b"] +enable_pprof = true + From 387ffa914e81c3cab4b4c57e03599fe6fb01b2de Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 30 Dec 2021 09:55:09 +0800 Subject: [PATCH 0012/1953] libs/types: support load Kata agent configuration from file Add structures to load Kata agent configuration from configuration files. Also define a mechanism for vendor to extend the Kata configuration structure. Signed-off-by: Liu Jiang --- src/libs/kata-types/src/config/agent.rs | 80 +++++++++++++++++++ .../kata-types/src/config/agent_vendor.rs | 12 +++ src/libs/kata-types/src/config/default.rs | 2 + src/libs/kata-types/src/config/mod.rs | 8 ++ src/libs/kata-types/src/config/runtime.rs | 2 +- 5 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 src/libs/kata-types/src/config/agent.rs create mode 100644 src/libs/kata-types/src/config/agent_vendor.rs diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs new file mode 100644 index 000000000000..69babf86c652 --- /dev/null +++ b/src/libs/kata-types/src/config/agent.rs @@ -0,0 +1,80 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::Result; + +use crate::config::{ConfigOps, TomlConfig}; + +pub use vendor::AgentVendor; + +/// Kata agent configuration information. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Agent { + /// If enabled, the agent will log additional debug messages to the system log. + #[serde(default, rename = "enable_debug")] + pub debug: bool, + + /// Enable agent tracing. + /// + /// If enabled, the agent will generate OpenTelemetry trace spans. + /// # Notes: + /// - If the runtime also has tracing enabled, the agent spans will be associated with the + /// appropriate runtime parent span. + /// - If enabled, the runtime will wait for the container to shutdown, increasing the container + /// shutdown time slightly. + #[serde(default)] + pub enable_tracing: bool, + + /// Enable debug console. + /// If enabled, user can connect guest OS running inside hypervisor through + /// "kata-runtime exec " command + #[serde(default)] + pub debug_console_enabled: bool, + + /// Agent connection dialing timeout value in seconds + #[serde(default)] + pub dial_timeout: u32, + + /// Comma separated list of kernel modules and their parameters. + /// + /// These modules will be loaded in the guest kernel using modprobe(8). + /// The following example can be used to load two kernel modules with parameters: + /// - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] + /// The first word is considered as the module name and the rest as its parameters. + /// Container will not be started when: + /// - A kernel module is specified and the modprobe command is not installed in the guest + /// or it fails loading the module. + /// - The module is not available in the guest or it doesn't met the guest kernel + /// requirements, like architecture and version. + #[serde(default)] + pub kernel_modules: Vec, +} + +impl ConfigOps for Agent { + fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { + AgentVendor::adjust_configuration(conf)?; + Ok(()) + } + + fn validate(conf: &TomlConfig) -> Result<()> { + AgentVendor::validate(conf)?; + Ok(()) + } +} + +#[cfg(not(feature = "enable-vendor"))] +mod vendor { + use super::*; + + /// Vendor customization agent configuration. + #[derive(Debug, Default, Deserialize, Serialize)] + pub struct AgentVendor {} + + impl ConfigOps for AgentVendor {} +} + +#[cfg(feature = "enable-vendor")] +#[path = "agent_vendor.rs"] +mod vendor; diff --git a/src/libs/kata-types/src/config/agent_vendor.rs b/src/libs/kata-types/src/config/agent_vendor.rs new file mode 100644 index 000000000000..62ce710d01de --- /dev/null +++ b/src/libs/kata-types/src/config/agent_vendor.rs @@ -0,0 +1,12 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use super::*; + +/// Vendor customization agent configuration. +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct AgentVendor {} + +impl ConfigOps for AgentVendor {} diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 722156206bf1..a32975621e15 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -20,6 +20,8 @@ lazy_static! { ]; } +pub const DEFAULT_AGENT_NAME: &str = "kata"; + pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter"; pub const DEFAULT_BLOCK_DEVICE_TYPE: &str = "virtio-blk"; diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 70fd81a5d8d0..12f42794d85d 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -14,6 +14,9 @@ use crate::{eother, sl}; /// Default configuration values. pub mod default; +mod agent; +pub use self::agent::{Agent, AgentVendor}; + mod hypervisor; pub use self::hypervisor::{ BootInfo, DragonballConfig, Hypervisor, QemuConfig, HYPERVISOR_NAME_DRAGONBALL, @@ -64,6 +67,9 @@ pub trait ConfigObjectOps { /// Kata configuration information. #[derive(Debug, Default, Deserialize, Serialize)] pub struct TomlConfig { + /// Configuration information for agents. + #[serde(default)] + pub agent: HashMap, /// Configuration information for hypervisors. #[serde(default)] pub hypervisor: HashMap, @@ -123,6 +129,7 @@ impl TomlConfig { Hypervisor::adjust_configuration(&mut config)?; Runtime::adjust_configuration(&mut config)?; + Agent::adjust_configuration(&mut config)?; info!(sl!(), "get kata config: {:?}", config); Ok(config) @@ -132,6 +139,7 @@ impl TomlConfig { pub fn validate(&self) -> Result<()> { Hypervisor::validate(self)?; Runtime::validate(self)?; + Agent::validate(self)?; Ok(()) } diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 9784662f98f2..b4bd2466d7ad 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2021 Alibaba Cloud // // SPDX-License-Identifier: Apache-2.0 // From e19d04719fa3fcc865c7f540ed3bc8da05897649 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 30 Dec 2021 16:29:03 +0800 Subject: [PATCH 0013/1953] libs/types: implement KataConfig to wrap TomlConfig The TomlConfig structure is a parsed form of Kata configuration file, but it's a little inconveneient to access those configuration information directly. So introduce a wrapper KataConfig to easily access those configuration information. Two singletons of KataConfig is provided: - KATA_DEFAULT_CONFIG: the original version directly loaded from Kata configuration file. - KATA_ACTIVE_CONFIG: the active version is the KATA_DEFAULT_CONFIG patched by annotations. So the recommended to way to use these two singletons: - Load TomlConfig from configuration file and set it as the default one. - Clone the default one and patch it with values from annotations. - Use the default one for permission checks, such as to check for allowed annotation keys/values. - The patched version may be set as the active one or passed to clients. - The clients directly accesses information from the active/passed one, and do not need to check annotation for override. Signed-off-by: Liu Jiang --- src/libs/kata-types/src/config/mod.rs | 91 +++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 12f42794d85d..cfb6316e3379 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -8,6 +8,9 @@ use std::collections::HashMap; use std::fs; use std::io::{self, Result}; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use lazy_static::lazy_static; use crate::{eother, sl}; @@ -176,6 +179,94 @@ pub fn validate_path_pattern>(patterns: &[String], path: P) -> Re Err(eother!("Path {} is not permitted", path)) } +/// Kata configuration information. +pub struct KataConfig { + config: TomlConfig, + agent: String, + hypervisor: String, +} + +impl KataConfig { + /// Set the default Kata configuration object. + /// + /// The default Kata configuration information is loaded from system configuration file. + pub fn set_default_config(config: TomlConfig, hypervisor: &str, agent: &str) { + let kata = KataConfig { + config, + agent: agent.to_string(), + hypervisor: hypervisor.to_string(), + }; + *KATA_DEFAULT_CONFIG.lock().unwrap() = Arc::new(kata); + } + + /// Get the default Kata configuration object. + /// + /// The default Kata configuration information is loaded from system configuration file. + pub fn get_default_config() -> Arc { + KATA_DEFAULT_CONFIG.lock().unwrap().clone() + } + + /// Set the active Kata configuration object. + /// + /// The active Kata configuration information is default configuration information patched + /// with tunable configuration information from annotations. + pub fn set_active_config(config: TomlConfig, hypervisor: &str, agent: &str) { + let kata = KataConfig { + config, + agent: agent.to_string(), + hypervisor: hypervisor.to_string(), + }; + *KATA_ACTIVE_CONFIG.lock().unwrap() = Arc::new(kata); + } + + /// Get the active Kata configuration object. + /// + /// The active Kata configuration information is default configuration information patched + /// with tunable configuration information from annotations. + pub fn get_active_config() -> Arc { + KATA_ACTIVE_CONFIG.lock().unwrap().clone() + } + + /// Get the agent configuration in use. + pub fn get_agent(&self) -> Option<&Agent> { + if !self.agent.is_empty() { + self.config.agent.get(&self.agent) + } else { + None + } + } + + /// Get the hypervisor configuration in use. + pub fn get_hypervisor(&self) -> Option<&Hypervisor> { + if !self.hypervisor.is_empty() { + self.config.hypervisor.get(&self.hypervisor) + } else { + None + } + } +} + +lazy_static! { + static ref KATA_DEFAULT_CONFIG: Mutex> = { + let config = TomlConfig::load("").unwrap(); + let kata = KataConfig { + config, + agent: String::new(), + hypervisor: String::new(), + }; + Mutex::new(Arc::new(kata)) + }; + static ref KATA_ACTIVE_CONFIG: Mutex> = { + let config = TomlConfig::load("").unwrap(); + let kata = KataConfig { + config, + agent: String::new(), + hypervisor: String::new(), + }; + Mutex::new(Arc::new(kata)) + }; +} + #[cfg(test)] mod tests { use super::*; From 8cdd70f6c2ca5400f3f8ddbc699005a86059c7d2 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Sun, 26 Dec 2021 01:28:15 +0800 Subject: [PATCH 0014/1953] libs/types: change method to update config by annotation Some annotations are used to override hypervisor configurations, and you know it's dangerous. We must be careful when overriding hypervisor configuration by annotations, to avoid security flaws. There are two existing mechanisms to prevent attacks by annotations: 1) config.hypervisor.enable_annotations defines the allowed annotation keys for config.hypervisor. 2) config.hyperisor.xxxx_paths defines allowd values for specific keys. The access methods for config.hypervisor.xxx enforces the permisstion checks for above rules. To update conifg, traverse the annotation hashmap,check if the key is enabled in hypervisor or not. If it is enabled. For path related annotation, check whether it is valid or not before updating conifg. For cpu and memory related annotation, check whether it is more than or less than the limitation for DB and qemu beforing updating config. If it is not enabled, there will be three possibilities, agent related annotation, runtime related annotation and hypervisor related annotation but not enabled. The function will handle agent and runtime annotation first, then the option left will be the invlaid hypervisor, err message will be returned. add more edge cases tests for updating config clean up unused functions, delete unused files and fix warnings Fixes: #3523 Signed-off-by: Zhongtao Hu Signed-off-by: Liu Jiang --- src/libs/Cargo.lock | 13 +- src/libs/kata-types/Cargo.toml | 1 + .../kata-types/src/annotations/dockershim.rs | 10 + src/libs/kata-types/src/annotations/mod.rs | 764 +++++++++++++++++- .../kata-types/src/annotations/thirdparty.rs | 14 + src/libs/kata-types/src/config/agent.rs | 3 + src/libs/kata-types/src/config/default.rs | 10 +- .../src/config/hypervisor/dragonball.rs | 9 + .../kata-types/src/config/hypervisor/mod.rs | 23 +- .../kata-types/src/config/hypervisor/qemu.rs | 11 + src/libs/kata-types/src/config/mod.rs | 19 +- src/libs/kata-types/src/config/runtime.rs | 1 - src/libs/kata-types/tests/test-config.rs | 491 ++++++++++- .../tests/texture/configuration-anno-0.toml | 88 ++ ...on-qemu.toml => configuration-anno-1.toml} | 27 +- 15 files changed, 1408 insertions(+), 76 deletions(-) create mode 100644 src/libs/kata-types/src/annotations/thirdparty.rs create mode 100644 src/libs/kata-types/tests/texture/configuration-anno-0.toml rename src/libs/kata-types/tests/texture/{configuration-qemu.toml => configuration-anno-1.toml} (75%) diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 7a7ad017a49e..341abf83b46d 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -317,6 +317,7 @@ dependencies = [ "oci", "regex", "serde", + "serde_json", "slog", "slog-scope", "thiserror", @@ -663,18 +664,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a" +checksum = "2cf9235533494ea2ddcdb794665461814781c53f19d87b76e571a1c35acbad2b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537" +checksum = "8dcde03d87d4c973c04be249e7d8f0b35db1c848c487bd43032808e59dd8328d" dependencies = [ "proc-macro2", "quote", @@ -718,9 +719,9 @@ dependencies = [ [[package]] name = "slog-json" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9b96fb6b5e80e371423b4aca6656eb537661ce8f82c2697e619f8ca85d043" +checksum = "0f7f7a952ce80fca9da17bf0a53895d11f8aa1ba063668ca53fc72e7869329e9" dependencies = [ "chrono", "serde", diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index ecae6f01845b..786615e55311 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -18,6 +18,7 @@ regex = "1.5.4" serde = { version = "1.0.100", features = ["derive"] } slog = "2.5.2" slog-scope = "4.4.0" +serde_json = "1.0.73" thiserror = "1.0" toml = "0.5.8" diff --git a/src/libs/kata-types/src/annotations/dockershim.rs b/src/libs/kata-types/src/annotations/dockershim.rs index 1558983d9a1b..df1279dc5ac1 100644 --- a/src/libs/kata-types/src/annotations/dockershim.rs +++ b/src/libs/kata-types/src/annotations/dockershim.rs @@ -6,8 +6,18 @@ #![allow(missing_docs)] +//! Copied from k8s.io/pkg/kubelet/dockershim/docker_service.go, used to identify whether a docker +//! container is a sandbox or a regular container, will be removed after defining those as public +//! fields in dockershim. + +/// ContainerTypeLabelKey is the container type (podsandbox or container) of key. pub const CONTAINER_TYPE_LABEL_KEY: &str = "io.kubernetes.docker.type"; + +/// ContainerTypeLabelSandbox represents a sandbox sandbox container. pub const SANDBOX: &str = "podsandbox"; + +/// ContainerTypeLabelContainer represents a container running within a sandbox. pub const CONTAINER: &str = "container"; +/// SandboxIDLabelKey is the sandbox ID annotation. pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.sandbox.id"; diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index f1cd1432046a..43f23e727a82 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -1,9 +1,20 @@ -// Copyright (c) 2019 Alibaba Cloud +// Copyright (c) 2019-2021 Alibaba Cloud // Copyright (c) 2019 Ant Group // // SPDX-License-Identifier: Apache-2.0 // +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, BufReader, Result}; +use std::u32; + +use serde::Deserialize; + +use crate::config::hypervisor::get_hypervisor_plugin; +use crate::config::TomlConfig; +use crate::sl; + /// CRI-containerd specific annotations. pub mod cri_containerd; @@ -12,3 +23,754 @@ pub mod crio; /// Dockershim specific annotations. pub mod dockershim; + +/// Third-party annotations. +pub mod thirdparty; + +// Common section +/// Prefix for Kata specific annotations +pub const KATA_ANNO_PREFIX: &str = "io.katacontainers."; +/// Prefix for Kata configuration annotations +pub const KATA_ANNO_CONF_PREFIX: &str = "io.katacontainers.config."; +/// Prefix for Kata container annotations +pub const KATA_ANNO_CONTAINER_PREFIX: &str = "io.katacontainers.container."; +/// The annotation key to fetch runtime configuration file. +pub const SANDBOX_CONFIG_PATH_KEY: &str = "io.katacontainers.config_path"; + +// OCI section +/// The annotation key to fetch the OCI configuration file path. +pub const BUNDLE_PATH_KEY: &str = "io.katacontainers.pkg.oci.bundle_path"; +/// The annotation key to fetch container type. +pub const CONTAINER_TYPE_KEY: &str = "io.katacontainers.pkg.oci.container_type"; + +// Container resource related annotations +/// Prefix for Kata container resource related annotations. +pub const KATA_ANNO_CONTAINER_RESOURCE_PREFIX: &str = "io.katacontainers.container.resource"; +/// A container annotation to specify the Resources.Memory.Swappiness. +pub const KATA_ANNO_CONTAINER_RESOURCE_SWAPPINESS: &str = + "io.katacontainers.container.resource.swappiness"; +/// A container annotation to specify the Resources.Memory.Swap. +pub const KATA_ANNO_CONTAINER_RESOURCE_SWAP_IN_BYTES: &str = + "io.katacontainers.container.resource.swap_in_bytes"; + +// Agent related annotations +/// Prefix for Agent configurations. +pub const KATA_ANNO_CONF_AGENT_PREFIX: &str = "io.katacontainers.config.agent."; +/// KernelModules is the annotation key for passing the list of kernel modules and their parameters +/// that will be loaded in the guest kernel. +/// +/// Semicolon separated list of kernel modules and their parameters. These modules will be loaded +/// in the guest kernel using modprobe(8). +/// The following example can be used to load two kernel modules with parameters +/// +/// annotations: +/// io.katacontainers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" +/// +/// The first word is considered as the module name and the rest as its parameters. +pub const KATA_ANNO_CONF_KERNEL_MODULES: &str = "io.katacontainers.config.agent.kernel_modules"; +/// A sandbox annotation to enable tracing for the agent. +pub const KATA_ANNO_CONF_AGENT_TRACE: &str = "io.katacontainers.config.agent.enable_tracing"; +/// An annotation to specify the size of the pipes created for containers. +pub const KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE: &str = + "io.katacontainers.config.agent.container_pipe_size"; +/// An annotation key to specify the size of the pipes created for containers. +pub const CONTAINER_PIPE_SIZE_KERNEL_PARAM: &str = "agent.container_pipe_size"; + +// Hypervisor related annotations +/// Prefix for Hypervisor configurations. +pub const KATA_ANNO_CONF_HYPERVISOR_PREFIX: &str = "io.katacontainers.config.hypervisor."; +/// A sandbox annotation for passing a per container path pointing at the hypervisor that will run +/// the container VM. +pub const KATA_ANNO_CONF_HYPERVISOR_PATH: &str = "io.katacontainers.config.hypervisor.path"; +/// A sandbox annotation for passing a container hypervisor binary SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_HASH: &str = "io.katacontainers.config.hypervisor.path_hash"; +/// A sandbox annotation for passing a per container path pointing at the hypervisor control binary +/// that will run the container VM. +pub const KATA_ANNO_CONF_HYPERVISOR_CTLPATH: &str = "io.katacontainers.config.hypervisor.ctlpath"; +/// A sandbox annotation for passing a container hypervisor control binary SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_CTLHASH: &str = + "io.katacontainers.config.hypervisor.hypervisorctl_hash"; +/// A sandbox annotation for passing a per container path pointing at the jailer that will constrain +/// the container VM. +pub const KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH: &str = + "io.katacontainers.config.hypervisor.jailer_path"; +/// A sandbox annotation for passing a jailer binary SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_JAILER_HASH: &str = + "io.katacontainers.config.hypervisor.jailer_hash"; +/// A sandbox annotation to enable IO to be processed in a separate thread. +/// Supported currently for virtio-scsi driver. +pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS: &str = + "io.katacontainers.config.hypervisor.enable_iothreads"; +/// The hash type used for assets verification +pub const KATA_ANNO_CONF_HYPERVISOR_ASSET_HASH_TYPE: &str = + "io.katacontainers.config.hypervisor.asset_hash_type"; +/// SHA512 is the SHA-512 (64) hash algorithm +pub const SHA512: &str = "sha512"; + +// Hypervisor Block Device related annotations +/// Specify the driver to be used for block device either VirtioSCSI or VirtioBlock +pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER: &str = + "io.katacontainers.config.hypervisor.block_device_driver"; +/// A sandbox annotation that disallows a block device from being used. +pub const KATA_ANNO_CONF_HYPERVISOR_DISABLE_BLOCK_DEVICE_USE: &str = + "io.katacontainers.config.hypervisor.disable_block_device_use"; +/// A sandbox annotation that specifies cache-related options will be set to block devices or not. +pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_SET: &str = + "io.katacontainers.config.hypervisor.block_device_cache_set"; +/// A sandbox annotation that specifies cache-related options for block devices. +/// Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. +pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_DIRECT: &str = + "io.katacontainers.config.hypervisor.block_device_cache_direct"; +/// A sandbox annotation that specifies cache-related options for block devices. +/// Denotes whether flush requests for the device are ignored. +pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH: &str = + "io.katacontainers.config.hypervisor.block_device_cache_noflush"; +/// A sandbox annotation to specify use of nvdimm device for guest rootfs image. +pub const KATA_ANNO_CONF_HYPERVISOR_DISABLE_IMAGE_NVDIMM: &str = + "io.katacontainers.config.hypervisor.disable_image_nvdimm"; +/// A sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_MEMORY_OFFSET: &str = + "io.katacontainers.config.hypervisor.memory_offset"; +/// A sandbox annotation to specify if vhost-user-blk/scsi is abailable on the host +pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_VHOSTUSER_STORE: &str = + "io.katacontainers.config.hypervisor.enable_vhost_user_store"; +/// A sandbox annotation to specify the directory path where vhost-user devices related folders, +/// sockets and device nodes should be. +pub const KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH: &str = + "io.katacontainers.config.hypervisor.vhost_user_store_path"; + +// Hypervisor Guest Boot related annotations +/// A sandbox annotation for passing a per container path pointing at the kernel needed to boot +/// the container VM. +pub const KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH: &str = + "io.katacontainers.config.hypervisor.kernel"; +/// A sandbox annotation for passing a container kernel image SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_KERNEL_HASH: &str = + "io.katacontainers.config.hypervisor.kernel_hash"; +/// A sandbox annotation for passing a per container path pointing at the guest image that will run +/// in the container VM. +/// A sandbox annotation for passing additional guest kernel parameters. +pub const KATA_ANNO_CONF_HYPERVISOR_KERNEL_PARAMS: &str = + "io.katacontainers.config.hypervisor.kernel_params"; +/// A sandbox annotation for passing a container guest image path. +pub const KATA_ANNO_CONF_HYPERVISOR_IMAGE_PATH: &str = "io.katacontainers.config.hypervisor.image"; +/// A sandbox annotation for passing a container guest image SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_IMAGE_HASH: &str = + "io.katacontainers.config.hypervisor.image_hash"; +/// A sandbox annotation for passing a per container path pointing at the initrd that will run +/// in the container VM. +pub const KATA_ANNO_CONF_HYPERVISOR_INITRD_PATH: &str = + "io.katacontainers.config.hypervisor.initrd"; +/// A sandbox annotation for passing a container guest initrd SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_INITRD_HASH: &str = + "io.katacontainers.config.hypervisor.initrd_hash"; +/// A sandbox annotation for passing a per container path pointing at the guest firmware that will +/// run the container VM. +pub const KATA_ANNO_CONF_HYPERVISOR_FIRMWARE_PATH: &str = + "io.katacontainers.config.hypervisor.firmware"; +/// A sandbox annotation for passing a container guest firmware SHA-512 hash value. +pub const KATA_ANNO_CONF_HYPERVISOR_FIRMWARE_HASH: &str = + "io.katacontainers.config.hypervisor.firmware_hash"; + +// Hypervisor CPU related annotations +/// A sandbox annotation to specify cpu specific features. +pub const KATA_ANNO_CONF_HYPERVISOR_CPU_FEATURES: &str = + "io.katacontainers.config.hypervisor.cpu_features"; +/// A sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS: &str = + "io.katacontainers.config.hypervisor.default_vcpus"; +/// A sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MAX_VCPUS: &str = + "io.katacontainers.config.hypervisor.default_max_vcpus"; + +// Hypervisor Device related annotations +/// A sandbox annotation used to indicate if devices need to be hotplugged on the root bus instead +/// of a bridge. +pub const KATA_ANNO_CONF_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS: &str = + "io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus"; +/// PCIeRootPort is used to indicate the number of PCIe Root Port devices +pub const KATA_ANNO_CONF_HYPERVISOR_PCIE_ROOT_PORT: &str = + "io.katacontainers.config.hypervisor.pcie_root_port"; +/// A sandbox annotation to specify if the VM should have a vIOMMU device. +pub const KATA_ANNO_CONF_HYPERVISOR_IOMMU: &str = + "io.katacontainers.config.hypervisor.enable_iommu"; +/// Enable Hypervisor Devices IOMMU_PLATFORM +pub const KATA_ANNO_CONF_HYPERVISOR_IOMMU_PLATFORM: &str = + "io.katacontainers.config.hypervisor.enable_iommu_platform"; + +// Hypervisor Machine related annotations +/// A sandbox annotation to specify the type of machine being emulated by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_MACHINE_TYPE: &str = + "io.katacontainers.config.hypervisor.machine_type"; +/// A sandbox annotation to specify machine specific accelerators for the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_MACHINE_ACCELERATORS: &str = + "io.katacontainers.config.hypervisor.machine_accelerators"; +/// EntropySource is a sandbox annotation to specify the path to a host source of +/// entropy (/dev/random, /dev/urandom or real hardware RNG device) +pub const KATA_ANNO_CONF_HYPERVISOR_ENTROPY_SOURCE: &str = + "io.katacontainers.config.hypervisor.entropy_source"; + +// Hypervisor Memory related annotations +/// A sandbox annotation for the memory assigned for a VM by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY: &str = + "io.katacontainers.config.hypervisor.default_memory"; +/// A sandbox annotation to specify the memory slots assigned to the VM by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS: &str = + "io.katacontainers.config.hypervisor.memory_slots"; +/// A sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. +pub const KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC: &str = + "io.katacontainers.config.hypervisor.enable_mem_prealloc"; +/// A sandbox annotation to specify if the memory should be pre-allocated from huge pages. +pub const KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES: &str = + "io.katacontainers.config.hypervisor.enable_hugepages"; +/// A sandbox annotation to soecify file based memory backend root directory. +pub const KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str = + "io.katacontainers.config.hypervisor.file_mem_backend"; +/// A sandbox annotation that is used to enable/disable virtio-mem. +pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM: &str = + "io.katacontainers.config.hypervisor.enable_virtio_mem"; +/// A sandbox annotation to enable swap of vm memory. +pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP: &str = + "io.katacontainers.config.hypervisor.enable_swap"; +/// A sandbox annotation to enable swap in the guest. +pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP: &str = + "io.katacontainers.config.hypervisor.enable_guest_swap"; + +// Hypervisor Network related annotations +/// A sandbox annotation to specify if vhost-net is not available on the host. +pub const KATA_ANNO_CONF_HYPERVISOR_DISABLE_VHOST_NET: &str = + "io.katacontainers.config.hypervisor.disable_vhost_net"; +/// A sandbox annotation that specifies max rate on network I/O inbound bandwidth. +pub const KATA_ANNO_CONF_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE: &str = + "io.katacontainers.config.hypervisor.rx_rate_limiter_max_rate"; +/// A sandbox annotation that specifies max rate on network I/O outbound bandwidth. +pub const KATA_ANNO_CONF_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE: &str = + "io.katacontainers.config.hypervisor.tx_rate_limiter_max_rate"; + +// Hypervisor Security related annotations +/// A sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks. +pub const KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH: &str = + "io.katacontainers.config.hypervisor.guest_hook_path"; +/// A sandbox annotation to enable rootless hypervisor (only supported in QEMU currently). +pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR: &str = + "io.katacontainers.config.hypervisor.rootless"; + +// Hypervisor Shared File System related annotations +/// A sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs. +pub const KATA_ANNO_CONF_HYPERVISOR_SHARED_FS: &str = + "io.katacontainers.config.hypervisor.shared_fs"; +/// A sandbox annotations to specify virtio-fs vhost-user daemon path. +pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON: &str = + "io.katacontainers.config.hypervisor.virtio_fs_daemon"; +/// A sandbox annotation to specify the cache mode for fs version cache or "none". +pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE: &str = + "io.katacontainers.config.hypervisor.virtio_fs_cache"; +/// A sandbox annotation to specify the DAX cache size in MiB. +pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE_SIZE: &str = + "io.katacontainers.config.hypervisor.virtio_fs_cache_size"; +/// A sandbox annotation to pass options to virtiofsd daemon. +pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS: &str = + "io.katacontainers.config.hypervisor.virtio_fs_extra_args"; +/// A sandbox annotation to specify as the msize for 9p shares. +pub const KATA_ANNO_CONF_HYPERVISOR_MSIZE_9P: &str = "io.katacontainers.config.hypervisor.msize_9p"; + +// Runtime related annotations +/// Prefix for Runtime configurations. +pub const KATA_ANNO_CONF_RUNTIME_PREFIX: &str = "io.katacontainers.config.runtime."; +/// A sandbox annotation that determines if seccomp should be applied inside guest. +pub const KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP: &str = + "io.katacontainers.config.runtime.disable_guest_seccomp"; +/// A sandbox annotation that determines if pprof enabled. +pub const KATA_ANNO_CONF_ENABLE_PPROF: &str = "io.katacontainers.config.runtime.enable_pprof"; +/// A sandbox annotation that determines if experimental features enabled. +pub const KATA_ANNO_CONF_EXPERIMENTAL: &str = "io.katacontainers.config.runtime.experimental"; +/// A sandbox annotaion that determines how the VM should be connected to the the container network +/// interface. +pub const KATA_ANNO_CONF_INTER_NETWORK_MODEL: &str = + "io.katacontainers.config.runtime.internetworking_model"; +/// SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup. +pub const KATA_ANNO_CONF_SANDBOX_CGROUP_ONLY: &str = + "io.katacontainers.config.runtime.sandbox_cgroup_only"; +/// A sandbox annotation that determines if create a netns for hypervisor process. +pub const KATA_ANNO_CONF_DISABLE_NEW_NETNS: &str = + "io.katacontainers.config.runtime.disable_new_netns"; +/// A sandbox annotation to specify how attached VFIO devices should be treated. +pub const KATA_ANNO_CONF_VFIO_MODE: &str = "io.katacontainers.config.runtime.vfio_mode"; + +/// A helper structure to query configuration information by check annotations. +#[derive(Debug, Default, Deserialize)] +pub struct Annotation { + annotations: HashMap, +} + +impl From> for Annotation { + fn from(annotations: HashMap) -> Self { + Annotation { annotations } + } +} + +impl Annotation { + /// Create a new instance of [`Annotation`]. + pub fn new(annotations: HashMap) -> Annotation { + Annotation { annotations } + } + + /// Deserialize an object from a json string. + pub fn deserialize(path: &str) -> Result + where + for<'a> T: Deserialize<'a>, + { + let f = BufReader::new(File::open(path)?); + Ok(serde_json::from_reader(f)?) + } + + /// Get an immutable reference to the annotation hashmap. + pub fn get_annotation(&self) -> &HashMap { + &self.annotations + } + + /// Get a mutable reference to the annotation hashmap. + pub fn get_annotation_mut(&mut self) -> &mut HashMap { + &mut self.annotations + } + + /// Get the value of annotation with `key` as string. + pub fn get(&self, key: &str) -> Option { + let v = self.annotations.get(key)?; + let value = v.trim(); + + if !value.is_empty() { + Some(String::from(value)) + } else { + None + } + } + + /// Get the value of annotation with `key` as bool. + pub fn get_bool(&self, key: &str) -> Option { + if let Some(value) = self.get(key) { + let value = value.trim(); + if value.parse::().is_err() { + warn!(sl!(), "failed to parse bool value from {}", value); + } else { + return Some(value.parse::().unwrap()); + } + } + + None + } + + /// Get the value of annotation with `key` as u32. + pub fn get_u32(&self, key: &str) -> Option { + let s = self.get(key)?; + match s.parse::() { + Ok(nums) => { + if nums > 0 { + Some(nums) + } else { + None + } + } + + Err(e) => { + warn!( + sl!(), + "failed to parse u32 value from {}, error: {:?}", s, e + ); + None + } + } + } + + /// Get the value of annotation with `key` as i32. + pub fn get_i32(&self, key: &str) -> Option { + let s = self.get(key)?; + s.parse::() + .map_or(None, |x| if x < 0 { None } else { Some(x) }) + } + + /// Get the value of annotation with `key` as u64. + pub fn get_u64(&self, key: &str) -> Option { + let s = self.get(key)?; + match s.parse::() { + Ok(nums) => { + if nums > 0 { + Some(nums) + } else { + None + } + } + + Err(e) => { + warn!( + sl!(), + "failed to parse u64 value from {}, error: {:?}", s, e + ); + None + } + } + } +} + +// Miscellaneous annotations. +impl Annotation { + /// Get the annotation of sandbox configuration file path. + pub fn get_sandbox_config_path(&self) -> Option { + self.get(SANDBOX_CONFIG_PATH_KEY) + } + + /// Get the annotation of bundle path. + pub fn get_bundle_path(&self) -> Option { + self.get(BUNDLE_PATH_KEY) + } + + /// Get the annotation of container type. + pub fn get_container_type(&self) -> Option { + self.get(CONTAINER_TYPE_KEY) + } + + /// Get the annotation to specify the Resources.Memory.Swappiness. + pub fn get_container_resource_swappiness(&self) -> Option { + let v = self.get_u32(KATA_ANNO_CONTAINER_RESOURCE_SWAPPINESS)?; + if v > 100 { + None + } else { + Some(v) + } + } + + /// Get the annotation to specify the Resources.Memory.Swap. + pub fn get_container_resource_swap_in_bytes(&self) -> Option { + self.get(KATA_ANNO_CONTAINER_RESOURCE_SWAP_IN_BYTES) + } +} + +impl Annotation { + /// update config info by annotation + pub fn update_config_by_annotation( + &self, + config: &mut TomlConfig, + hypervisor_name: &str, + agent_name: &str, + ) -> Result<()> { + if config.hypervisor.get_mut(hypervisor_name).is_none() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("hypervisor {} not found", hypervisor_name), + )); + } + + if config.agent.get_mut(agent_name).is_none() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("agent {} not found", agent_name), + )); + } + let mut hv = config.hypervisor.get_mut(hypervisor_name).unwrap(); + let mut ag = config.agent.get_mut(agent_name).unwrap(); + for (key, value) in &self.annotations { + if hv.security_info.is_annotation_enabled(key) { + match key.as_str() { + // update hypervisor config + // Hypervisor related annotations + KATA_ANNO_CONF_HYPERVISOR_PATH => { + hv.validate_hypervisor_path(value)?; + hv.path = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_CTLPATH => { + hv.validate_hypervisor_ctlpath(value)?; + hv.ctlpath = value.to_string(); + } + + KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH => { + hv.validate_jailer_path(value)?; + hv.jailer_path = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS => { + hv.enable_iothreads = self.get_bool(key).unwrap_or_default(); + } + // Hypervisor Block Device related annotations + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER => { + hv.blockdev_info.block_device_driver = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_DISABLE_BLOCK_DEVICE_USE => { + hv.blockdev_info.disable_block_device_use = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_SET => { + hv.blockdev_info.block_device_cache_set = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_DIRECT => { + hv.blockdev_info.block_device_cache_direct = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH => { + hv.blockdev_info.block_device_cache_noflush = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_DISABLE_IMAGE_NVDIMM => { + hv.blockdev_info.disable_image_nvdimm = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_MEMORY_OFFSET => { + hv.blockdev_info.memory_offset = self.get_u64(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_ENABLE_VHOSTUSER_STORE => { + hv.blockdev_info.enable_vhost_user_store = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH => { + hv.blockdev_info.validate_vhost_user_store_path(value)?; + hv.blockdev_info.vhost_user_store_path = value.to_string(); + } + // Hypervisor Guest Boot related annotations + KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH => { + hv.boot_info.validate_boot_path(value)?; + hv.boot_info.kernel = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_KERNEL_PARAMS => { + hv.boot_info.kernel_params = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_IMAGE_PATH => { + hv.boot_info.validate_boot_path(value)?; + hv.boot_info.image = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_INITRD_PATH => { + hv.boot_info.validate_boot_path(value)?; + hv.boot_info.initrd = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_FIRMWARE_PATH => { + hv.boot_info.validate_boot_path(value)?; + hv.boot_info.firmware = value.to_string(); + } + // Hypervisor CPU related annotations + KATA_ANNO_CONF_HYPERVISOR_CPU_FEATURES => { + hv.cpu_info.cpu_features = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS => { + let num_cpus = self.get_i32(key).unwrap_or_default(); + if num_cpus + > get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_max_cpus() as i32 + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Vcpus specified in annotation {} is more than maximum limitation {}", + num_cpus, + get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_max_cpus() + ), + )); + } else { + hv.cpu_info.default_vcpus = num_cpus; + } + } + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MAX_VCPUS => { + hv.cpu_info.default_maxvcpus = self.get_u32(key).unwrap_or_default(); + } + // Hypervisor Device related annotations + KATA_ANNO_CONF_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => { + hv.device_info.hotplug_vfio_on_root_bus = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_PCIE_ROOT_PORT => { + hv.device_info.pcie_root_port = self.get_u32(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_IOMMU => { + hv.device_info.enable_iommu = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_IOMMU_PLATFORM => { + hv.device_info.enable_iommu_platform = + self.get_bool(key).unwrap_or_default(); + } + // Hypervisor Machine related annotations + KATA_ANNO_CONF_HYPERVISOR_MACHINE_TYPE => { + hv.machine_info.machine_type = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_MACHINE_ACCELERATORS => { + hv.machine_info.machine_accelerators = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_ENTROPY_SOURCE => { + hv.machine_info.validate_entropy_source(value)?; + hv.machine_info.entropy_source = value.to_string(); + } + // Hypervisor Memory related annotations + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY => { + let mem = self.get_u32(key).unwrap_or_default(); + if mem + < get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_min_memory() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Memory specified in annotation {} is less than minmum required {}", + mem, + get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_min_memory() + ), + )); + } else { + hv.memory_info.default_memory = mem; + } + } + KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS => match self.get_u32(key) { + Some(v) => { + hv.memory_info.memory_slots = v; + } + None => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{}in annotation is less than zero", key), + )); + } + }, + + KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC => { + hv.memory_info.enable_mem_prealloc = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES => { + hv.memory_info.enable_hugepages = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { + hv.memory_info.validate_memory_backend_path(value)?; + hv.memory_info.file_mem_backend = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM => { + hv.memory_info.enable_virtio_mem = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP => { + hv.memory_info.enable_swap = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP => { + hv.memory_info.enable_guest_swap = self.get_bool(key).unwrap_or_default(); + } + // Hypervisor Network related annotations + KATA_ANNO_CONF_HYPERVISOR_DISABLE_VHOST_NET => { + hv.network_info.disable_vhost_net = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { + Some(v) => { + hv.network_info.rx_rate_limiter_max_rate = v; + } + None => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{} in annotation is less than zero", key), + )); + } + }, + KATA_ANNO_CONF_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { + Some(v) => { + hv.network_info.tx_rate_limiter_max_rate = v; + } + None => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{} in annotation is less than zero", key), + )); + } + }, + // Hypervisor Security related annotations + KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH => { + hv.security_info.validate_path(value)?; + hv.security_info.guest_hook_path = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR => { + hv.security_info.rootless = self.get_bool(key).unwrap_or_default(); + } + // Hypervisor Shared File System related annotations + KATA_ANNO_CONF_HYPERVISOR_SHARED_FS => { + hv.shared_fs.shared_fs = self.get(key); + } + + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON => { + hv.shared_fs.validate_virtiofs_daemon_path(value)?; + hv.shared_fs.virtio_fs_daemon = value.to_string(); + } + + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE => { + hv.shared_fs.virtio_fs_cache = value.to_string(); + } + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => { + hv.shared_fs.virtio_fs_cache_size = self.get_u32(key).unwrap_or_default(); + } + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS => { + let args: Vec = + value.to_string().split(',').map(str::to_string).collect(); + for arg in args { + hv.shared_fs.virtio_fs_extra_args.push(arg.to_string()); + } + } + KATA_ANNO_CONF_HYPERVISOR_MSIZE_9P => { + hv.shared_fs.msize_9p = self.get_u32(key).unwrap_or_default(); + } + + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Invalid Annotation Type {}", key), + )); + } + } + } else { + match key.as_str() { + //update agent config + KATA_ANNO_CONF_KERNEL_MODULES => { + let kernel_mod: Vec = + value.to_string().split(';').map(str::to_string).collect(); + for modules in kernel_mod { + ag.kernel_modules.push(modules.to_string()); + } + } + KATA_ANNO_CONF_AGENT_TRACE => { + ag.enable_tracing = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE => { + ag.container_pipe_size = self.get_u32(key).unwrap_or_default(); + } + //update runtume config + KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP => { + config.runtime.disable_guest_seccomp = + self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_ENABLE_PPROF => { + config.runtime.enable_pprof = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_EXPERIMENTAL => { + let args: Vec = + value.to_string().split(',').map(str::to_string).collect(); + for arg in args { + config.runtime.experimental.push(arg.to_string()); + } + } + KATA_ANNO_CONF_INTER_NETWORK_MODEL => { + config.runtime.internetworking_model = value.to_string(); + } + KATA_ANNO_CONF_SANDBOX_CGROUP_ONLY => { + config.runtime.disable_new_netns = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_DISABLE_NEW_NETNS => { + config.runtime.disable_new_netns = self.get_bool(key).unwrap_or_default(); + } + KATA_ANNO_CONF_VFIO_MODE => { + config.runtime.vfio_mode = value.to_string(); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Annotation {} not enabled", key), + )); + } + } + } + } + Ok(()) + } +} diff --git a/src/libs/kata-types/src/annotations/thirdparty.rs b/src/libs/kata-types/src/annotations/thirdparty.rs new file mode 100644 index 000000000000..4ea1548b9fd9 --- /dev/null +++ b/src/libs/kata-types/src/annotations/thirdparty.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Third-party annotations - annotations defined by other projects or k8s plugins but that can +//! change Kata Containers behaviour. + +/// Annotation to enable SGX. +/// +/// Hardware-based isolation and memory encryption. +// Supported suffixes are: Ki | Mi | Gi | Ti | Pi | Ei . For example: 4Mi +// For more information about supported suffixes see https://physics.nist.gov/cuu/Units/binary.html +pub const SGXEPC: &str = "sgx.intel.com/epc"; diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 69babf86c652..9f4f0abbb4e8 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -50,6 +50,9 @@ pub struct Agent { /// requirements, like architecture and version. #[serde(default)] pub kernel_modules: Vec, + + /// contianer pipe size + pub container_pipe_size: u32, } impl ConfigOps for Agent { diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index a32975621e15..829a1d7ddc99 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -9,17 +9,12 @@ use lazy_static::lazy_static; lazy_static! { - /// Default configuration file paths. + /// Default configuration file paths, vendor may extend the list pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![ - "/etc/kata-containers2/configuration.toml", - "/usr/share/defaults/kata-containers2/configuration.toml", - "/etc/kata-containers/configuration_v2.toml", - "/usr/share/defaults/kata-containers/configuration_v2.toml", "/etc/kata-containers/configuration.toml", "/usr/share/defaults/kata-containers/configuration.toml", ]; } - pub const DEFAULT_AGENT_NAME: &str = "kata"; pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter"; @@ -46,7 +41,7 @@ pub const DEFAULT_DB_ENTROPY_SOURCE: &str = "/dev/urandom"; pub const DEFAULT_DB_MEMORY_SIZE: u32 = 128; pub const DEFAULT_DB_MEMORY_SLOTS: u32 = 128; pub const MAX_DB_VCPUS: u32 = 256; - +pub const MIN_DB_MEMORY_SIZE: u32 = 64; // Default configuration for qemu pub const DEFAULT_QEMU_BINARY_PATH: &str = "qemu"; pub const DEFAULT_QEMU_CONTROL_PATH: &str = ""; @@ -60,3 +55,4 @@ pub const DEFAULT_QEMU_MEMORY_SLOTS: u32 = 128; pub const DEFAULT_QEMU_PCI_BRIDGES: u32 = 2; pub const MAX_QEMU_PCI_BRIDGES: u32 = 5; pub const MAX_QEMU_VCPUS: u32 = 256; +pub const MIN_QEMU_MEMORY_SIZE: u32 = 64; diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs index 8bde9cec7635..c0b553c223d6 100644 --- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -6,8 +6,11 @@ use std::io::Result; use std::path::Path; use std::sync::Arc; +use std::u32; use super::{default, register_hypervisor_plugin}; +use crate::config::default::MAX_DB_VCPUS; +use crate::config::default::MIN_DB_MEMORY_SIZE; use crate::config::hypervisor::{ VIRTIO_BLK, VIRTIO_BLK_MMIO, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM, }; @@ -35,6 +38,12 @@ impl DragonballConfig { } impl ConfigPlugin for DragonballConfig { + fn get_max_cpus(&self) -> u32 { + MAX_DB_VCPUS + } + fn get_min_memory(&self) -> u32 { + MIN_DB_MEMORY_SIZE + } fn name(&self) -> &str { HYPERVISOR_NAME_DRAGONBALL } diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 23c74c971a60..4e083f9ba255 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -31,6 +31,7 @@ use lazy_static::lazy_static; use regex::RegexSet; use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; +use crate::annotations::KATA_ANNO_CONF_HYPERVISOR_PREFIX; use crate::{eother, resolve_path, validate_path}; mod dragonball; @@ -232,6 +233,12 @@ impl BootInfo { } Ok(()) } + + /// Validate guest kernel image annotaion + pub fn validate_boot_path(&self, path: &str) -> Result<()> { + validate_path!(path, "path {} is invalid{}")?; + Ok(()) + } } /// Virtual CPU configuration information. @@ -692,17 +699,22 @@ impl SecurityInfo { /// Check whether annotation key is enabled or not. pub fn is_annotation_enabled(&self, path: &str) -> bool { - if !path.starts_with("io.katacontainers.config.hypervisor.") { + if !path.starts_with(KATA_ANNO_CONF_HYPERVISOR_PREFIX) { return false; } - let pos = "io.katacontainers.config.hypervisor.".len(); + let pos = KATA_ANNO_CONF_HYPERVISOR_PREFIX.len(); let key = &path[pos..]; if let Ok(set) = RegexSet::new(&self.enable_annotations) { return set.is_match(key); } - false } + + /// Validate path + pub fn validate_path(&self, path: &str) -> Result<()> { + validate_path!(path, "path {} is invalid{}")?; + Ok(()) + } } /// Configuration information for shared filesystem, such virtio-9p and virtio-fs. @@ -955,12 +967,10 @@ impl Hypervisor { impl ConfigOps for Hypervisor { fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { HypervisorVendor::adjust_configuration(conf)?; - let hypervisors: Vec = conf.hypervisor.keys().cloned().collect(); for hypervisor in hypervisors.iter() { if let Some(plugin) = get_hypervisor_plugin(hypervisor) { plugin.adjust_configuration(conf)?; - // Safe to unwrap() because `hypervisor` is a valid key in the hash map. let hv = conf.hypervisor.get_mut(hypervisor).unwrap(); hv.blockdev_info.adjust_configuration()?; @@ -1031,9 +1041,8 @@ mod vendor { #[path = "vendor.rs"] mod vendor; +pub use self::vendor::HypervisorVendor; use crate::config::validate_path_pattern; -pub use vendor::HypervisorVendor; - #[cfg(test)] mod tests { use super::*; diff --git a/src/libs/kata-types/src/config/hypervisor/qemu.rs b/src/libs/kata-types/src/config/hypervisor/qemu.rs index 87b4a11bef2f..6c59f54e9545 100644 --- a/src/libs/kata-types/src/config/hypervisor/qemu.rs +++ b/src/libs/kata-types/src/config/hypervisor/qemu.rs @@ -8,6 +8,10 @@ use std::path::Path; use std::sync::Arc; use super::{default, register_hypervisor_plugin}; + +use crate::config::default::MAX_QEMU_VCPUS; +use crate::config::default::MIN_QEMU_MEMORY_SIZE; + use crate::config::hypervisor::VIRTIO_BLK_MMIO; use crate::config::{ConfigPlugin, TomlConfig}; use crate::{eother, resolve_path, validate_path}; @@ -33,6 +37,13 @@ impl QemuConfig { } impl ConfigPlugin for QemuConfig { + fn get_max_cpus(&self) -> u32 { + MAX_QEMU_VCPUS + } + + fn get_min_memory(&self) -> u32 { + MIN_QEMU_MEMORY_SIZE + } fn name(&self) -> &str { HYPERVISOR_NAME_QEMU } diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index cfb6316e3379..c2380c299346 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -9,6 +9,7 @@ use std::fs; use std::io::{self, Result}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; +use std::u32; use lazy_static::lazy_static; @@ -18,9 +19,9 @@ use crate::{eother, sl}; pub mod default; mod agent; -pub use self::agent::{Agent, AgentVendor}; +pub mod hypervisor; -mod hypervisor; +use self::agent::Agent; pub use self::hypervisor::{ BootInfo, DragonballConfig, Hypervisor, QemuConfig, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_QEMU, @@ -39,6 +40,12 @@ pub trait ConfigPlugin: Send + Sync { /// Validate the configuration information. fn validate(&self, _conf: &TomlConfig) -> Result<()>; + + /// Get the minmum memory for hypervisor + fn get_min_memory(&self) -> u32; + + /// Get the max defualt cpus + fn get_max_cpus(&self) -> u32; } /// Trait to manipulate Kata configuration information. @@ -129,12 +136,10 @@ impl TomlConfig { /// Load Kata configuration information from string. pub fn load(content: &str) -> Result { let mut config: TomlConfig = toml::from_str(content)?; - Hypervisor::adjust_configuration(&mut config)?; Runtime::adjust_configuration(&mut config)?; Agent::adjust_configuration(&mut config)?; info!(sl!(), "get kata config: {:?}", config); - Ok(config) } @@ -167,7 +172,6 @@ pub fn validate_path_pattern>(patterns: &[String], path: P) -> Re .as_ref() .to_str() .ok_or_else(|| eother!("Invalid path {}", path.as_ref().to_string_lossy()))?; - for p in patterns.iter() { if let Ok(glob) = glob::Pattern::new(p) { if glob.matches(path) { @@ -226,6 +230,10 @@ impl KataConfig { pub fn get_active_config() -> Arc { KATA_ACTIVE_CONFIG.lock().unwrap().clone() } + /// Get the config in use + pub fn get_config(&self) -> &TomlConfig { + &self.config + } /// Get the agent configuration in use. pub fn get_agent(&self) -> Option<&Agent> { @@ -254,6 +262,7 @@ lazy_static! { agent: String::new(), hypervisor: String::new(), }; + Mutex::new(Arc::new(kata)) }; static ref KATA_ACTIVE_CONFIG: Mutex> = { diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index b4bd2466d7ad..48e66858eba5 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -115,7 +115,6 @@ pub struct Runtime { impl ConfigOps for Runtime { fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { RuntimeVendor::adjust_configuration(conf)?; - if conf.runtime.internetworking_model.is_empty() { conf.runtime.internetworking_model = default::DEFAULT_INTERNETWORKING_MODEL.to_owned(); } diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test-config.rs index e88a250469dd..9a2061c7c7f2 100644 --- a/src/libs/kata-types/tests/test-config.rs +++ b/src/libs/kata-types/tests/test-config.rs @@ -1,41 +1,452 @@ -use kata_types::config::{QemuConfig, TomlConfig, HYPERVISOR_NAME_QEMU}; -use std::fs; -use std::path::Path; - -#[test] -fn test_load_qemu_config() { - let plugin = QemuConfig::new(); - plugin.register(); - - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-qemu.toml"); - let content = fs::read_to_string(&path).unwrap(); - let config = TomlConfig::load(&content).unwrap(); - - let qemu = config.hypervisor.get(HYPERVISOR_NAME_QEMU).unwrap(); - assert_eq!(qemu.path, "/usr/bin/ls"); - assert_eq!(qemu.valid_hypervisor_paths.len(), 2); - assert_eq!(qemu.valid_hypervisor_paths[0], "/usr/bin/qemu*"); - assert_eq!(qemu.valid_hypervisor_paths[1], "/opt/qemu?"); - qemu.validate_hypervisor_path("/usr/bin/qemu0").unwrap(); - qemu.validate_hypervisor_path("/usr/bin/qemu1").unwrap(); - qemu.validate_hypervisor_path("/usr/bin/qemu2222").unwrap(); - qemu.validate_hypervisor_path("/opt/qemu3").unwrap(); - qemu.validate_hypervisor_path("/opt/qemu").unwrap_err(); - qemu.validate_hypervisor_path("/opt/qemu33").unwrap_err(); - assert_eq!(qemu.ctlpath, "/usr/bin/ls"); - assert_eq!(qemu.valid_ctlpaths.len(), 0); - assert!(qemu.jailer_path.is_empty()); - assert_eq!(qemu.valid_jailer_paths.len(), 0); - assert_eq!(qemu.disable_nesting_checks, true); - assert_eq!(qemu.enable_iothreads, true); - - assert_eq!(qemu.boot_info.image, "/usr/bin/echo"); - assert_eq!(qemu.boot_info.kernel, "/usr/bin/id"); - assert_eq!(qemu.boot_info.kernel_params, "ro"); - assert_eq!(qemu.boot_info.firmware, "/etc/hostname"); - - assert_eq!(qemu.cpu_info.cpu_features, "pmu=off,vmx=off"); - assert_eq!(qemu.cpu_info.default_vcpus, 2); - assert_eq!(qemu.cpu_info.default_maxvcpus, 64); +#[cfg(test)] +mod tests { + use kata_types::annotations::{ + Annotation, KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE, KATA_ANNO_CONF_AGENT_TRACE, + KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP, KATA_ANNO_CONF_ENABLE_PPROF, + KATA_ANNO_CONF_EXPERIMENTAL, KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH, + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER, KATA_ANNO_CONF_HYPERVISOR_CTLPATH, + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS, + KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS, + KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, + KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES, + KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH, KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH, + KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS, + KATA_ANNO_CONF_HYPERVISOR_PATH, KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH, + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CONF_KERNEL_MODULES, + }; + use kata_types::config::KataConfig; + use kata_types::config::{QemuConfig, TomlConfig}; + use std::collections::HashMap; + use std::fs; + use std::path::Path; + #[test] + fn test_change_config_annotation() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + std::process::Command::new("mkdir") + .arg("./hypervisor_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./store_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./test_hypervisor_hook_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./jvm") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./test_file_backend_mem_root") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./test_jailer_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./test_kernel_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("mkdir") + .arg("./virtio_fs") + .output() + .expect("failed to execute process"); + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_KERNEL_MODULES.to_string(), + "j465 aaa=1;r33w".to_string(), + ); + anno_hash.insert(KATA_ANNO_CONF_AGENT_TRACE.to_string(), "false".to_string()); + anno_hash.insert( + KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE.to_string(), + "3".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_PATH.to_string(), + "./hypervisor_path".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER.to_string(), + "device".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH.to_string(), + "./store_path".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP.to_string(), + "true".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH.to_string(), + "./test_hypervisor_hook_path".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_CTLPATH.to_string(), + "./jvm".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS.to_string(), + "12".to_string(), + ); + anno_hash.insert(KATA_ANNO_CONF_ENABLE_PPROF.to_string(), "false".to_string()); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY.to_string(), + "100".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR.to_string(), + "./test_file_backend_mem_root".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH.to_string(), + "./test_jailer_path".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH.to_string(), + "./test_kernel_path".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS.to_string(), + "100".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS.to_string(), + "rr,dg,er".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM.to_string(), + "false".to_string(), + ); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON.to_string(), + "./virtio_fs".to_string(), + ); + anno_hash.insert(KATA_ANNO_CONF_EXPERIMENTAL.to_string(), "c,d,e".to_string()); + + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_ok()); + KataConfig::set_active_config(config, "qemu", "agnet0"); + if let Some(ag) = KataConfig::get_default_config().get_agent() { + assert_eq!( + ag.kernel_modules[0], + "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1" + ); + + assert_eq!(ag.kernel_modules[1], "i915_enabled_ppgtt=0"); + assert_eq!(ag.kernel_modules[2], "j465 aaa=1"); + assert_eq!(ag.kernel_modules[3], "r33w"); + assert!(!ag.enable_tracing); + assert_eq!(ag.container_pipe_size, 3); + } + if let Some(hv) = KataConfig::get_default_config().get_hypervisor() { + assert_eq!(hv.path, "./hypervisor_path".to_string()); + assert_eq!(hv.blockdev_info.block_device_driver, "device"); + assert!(!hv.blockdev_info.block_device_cache_noflush); + assert!(hv.blockdev_info.block_device_cache_set); + assert_eq!(hv.blockdev_info.vhost_user_store_path, "./store_path"); + assert_eq!( + hv.security_info.guest_hook_path, + "./test_hypervisor_hook_path" + ); + assert!(!hv.memory_info.enable_mem_prealloc); + assert_eq!(hv.ctlpath, "./jvm".to_string()); + assert_eq!(hv.cpu_info.default_vcpus, 12); + assert!(!hv.memory_info.enable_guest_swap); + assert_eq!(hv.memory_info.default_memory, 100); + assert!(!hv.enable_iothreads); + assert!(!hv.enable_iothreads); + assert!(!hv.memory_info.enable_swap); + assert_eq!( + hv.memory_info.file_mem_backend, + "./test_file_backend_mem_root" + ); + assert!(!hv.memory_info.enable_hugepages); + assert_eq!(hv.jailer_path, "./test_jailer_path".to_string()); + assert_eq!(hv.boot_info.kernel, "./test_kernel_path"); + assert_eq!(hv.memory_info.memory_slots, 100); + assert_eq!(hv.shared_fs.virtio_fs_extra_args[5], "rr"); + assert_eq!(hv.shared_fs.virtio_fs_extra_args[6], "dg"); + assert_eq!(hv.shared_fs.virtio_fs_extra_args[7], "er"); + assert!(!hv.memory_info.enable_virtio_mem); + assert_eq!(hv.shared_fs.virtio_fs_daemon, "./virtio_fs"); + } + + assert!( + KataConfig::get_active_config() + .get_config() + .runtime + .disable_guest_seccomp + ); + + assert!( + !KataConfig::get_active_config() + .get_config() + .runtime + .enable_pprof + ); + assert_eq!( + KataConfig::get_active_config() + .get_config() + .runtime + .experimental, + ["a", "b", "c", "d", "e"] + ); + std::process::Command::new("rmdir") + .arg("./hypervisor_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("rmdir") + .arg("./test_hypervisor_hook_path") + .output() + .expect("failed to execute process"); + + std::process::Command::new("rmdir") + .arg("./test_file_backend_mem_root") + .output() + .expect("failed to execute process"); + + std::process::Command::new("rmdir") + .arg("./test_jailer_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("rmdir") + .arg("./test_kernel_path") + .output() + .expect("failed to execute process"); + std::process::Command::new("rmdir") + .arg("./virtio_fs") + .output() + .expect("failed to execute process"); + std::process::Command::new("rmdir") + .arg("./jvm") + .output() + .expect("failed to execute process"); + std::process::Command::new("rmdir") + .arg("./store_path") + .output() + .expect("failed to execute process"); + } + + #[test] + fn test_fail_to_change_block_device_driver_because_not_enabled() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-1.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER.to_string(), + "fvfvfvfvf".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_enable_guest_swap_because_not_enabled() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-1.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), + "false".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_hypervisor_path_because_of_invalid_path() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_PATH.to_string(), + "/usr/bin/nle".to_string(), + ); + let anno = Annotation::new(anno_hash); + + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + let mut config = TomlConfig::load(&content).unwrap(); + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_kernel_path_because_of_invalid_path() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH.to_string(), + "/usr/bin/cdcd".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_memory_slots_because_of_less_than_zero() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let qemu = QemuConfig::new(); + qemu.register(); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS.to_string(), + "-1".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_default_memory_because_less_than_min_memory_size() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY.to_string(), + "10".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_default_vcpus_becuase_more_than_max_cpu_size() { + let path = env!("CARGO_MANIFEST_DIR"); + let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); + let content = fs::read_to_string(&path).unwrap(); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(config, "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS.to_string(), + "400".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } } diff --git a/src/libs/kata-types/tests/texture/configuration-anno-0.toml b/src/libs/kata-types/tests/texture/configuration-anno-0.toml new file mode 100644 index 000000000000..30b199260832 --- /dev/null +++ b/src/libs/kata-types/tests/texture/configuration-anno-0.toml @@ -0,0 +1,88 @@ +[hypervisor.qemu] +path = "/usr/bin/lsns" +valid_hypervisor_paths = ["/usr/bin/qemu*", "/opt/qemu?","/usr/bin/ls*","./hypervisor_path"] +valid_jailer_paths = ["/usr/lib/rust","./test_jailer_path"] +ctlpath = "/usr/bin/" +valid_ctlpaths = ["/usr/lib/jvm","usr/bin/qemu-io","./jvm"] +disable_nesting_checks = true +enable_iothreads = true +jailer_path = "/usr/local" +kernel = "/usr/bin/../bin/zcmp" +image = "/usr/bin/./tabs" +kernel_params = "ro" +firmware = "/etc/hostname" + +cpu_features="pmu=off,vmx=off" +default_vcpus = 2 +default_maxvcpus = 64 + +machine_type = "q35" +confidential_guest = true +rootless = true +enable_annotations = ["shared_fs","path", "ctlpath","jailer_path","enable_iothreads","default_memory","memory_slots","enable_mem_prealloc","enable_hugepages","file_mem_backend","enable_virtio_mem","enable_swap","enable_guest_swap","default_vcpus","virtio_fs_extra_args","block_device_driver","vhost_user_store_path","kernel","guest_hook_path","block_device_cache_noflush","virtio_fs_daemon"] +machine_accelerators="noapic" +default_bridges = 2 +default_memory = 128 +memory_slots = 128 +memory_offset = 0x100000 +enable_virtio_mem = true +disable_block_device_use = false +shared_fs = "virtio-fs" +virtio_fs_daemon = "/usr/bin/uptime" +valid_virtio_fs_daemon_paths = ["/usr/local/bin/virtiofsd*","./virtio_fs"] +virtio_fs_cache_size = 512 +virtio_fs_extra_args = ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] +virtio_fs_cache = "always" +block_device_driver = "virtio-blk" +block_device_cache_set = true +block_device_cache_direct = true +block_device_cache_noflush = true +enable_mem_prealloc = true +enable_hugepages = true +enable_vhost_user_store = true +vhost_user_store_path = "/tmp" +valid_vhost_user_store_paths = ["/var/kata/vhost-user-store*", "/tmp/kata?","/var/tmp","./store_path"] +enable_iommu = true +enable_iommu_platform = true +file_mem_backend = "/dev/shm" +valid_file_mem_backends = ["/dev/shm","/dev/snd","./test_file_backend_mem_root"] +enable_swap = true +pflashes = ["/proc/mounts"] +enable_debug = true +msize_9p = 16384 +disable_image_nvdimm = true +hotplug_vfio_on_root_bus = true +pcie_root_port = 2 +disable_vhost_net = true +entropy_source= "/dev/urandom" +valid_entropy_sources = ["/dev/urandom", "/dev/random"] +guest_hook_path = "/usr/share" +rx_rate_limiter_max_rate = 10000 +tx_rate_limiter_max_rate = 10000 +guest_memory_dump_path="/var/crash/kata" +guest_memory_dump_paging = true +enable_guest_swap = true + +[agent.agent0] +enable_tracing = true +debug_console_enabled = true +debug = true +dial_timeout = 1 +kernel_modules = ["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1","i915_enabled_ppgtt=0"] +container_pipe_size = 2 +[runtime] +enable_debug = true +internetworking_model="macvtap" +disable_guest_seccomp=false +enable_tracing = true +jaeger_endpoint = "localhost:1234" +jaeger_user = "user" +jaeger_password = "pw" +disable_new_netns = true +sandbox_cgroup_only=true +sandbox_bind_mounts=["/proc/self"] +vfio_mode="vfio" +experimental=["a", "b"] +enable_pprof = true + + diff --git a/src/libs/kata-types/tests/texture/configuration-qemu.toml b/src/libs/kata-types/tests/texture/configuration-anno-1.toml similarity index 75% rename from src/libs/kata-types/tests/texture/configuration-qemu.toml rename to src/libs/kata-types/tests/texture/configuration-anno-1.toml index d30e0858a8d0..9a411bdff5af 100644 --- a/src/libs/kata-types/tests/texture/configuration-qemu.toml +++ b/src/libs/kata-types/tests/texture/configuration-anno-1.toml @@ -1,12 +1,13 @@ [hypervisor.qemu] -path = "/usr/bin/ls" -valid_hypervisor_paths = ["/usr/bin/qemu*", "/opt/qemu?"] -ctlpath = "/usr/bin/ls" +path = "/usr/bin/lsns" +valid_hypervisor_paths = ["/usr/bin/qemu*", "/opt/qemu?","/usr/bin/lsns","./hypervisor_path"] +valid_jailer_paths = ["/usr/lib/rust"] +ctlpath = "/usr/bin" disable_nesting_checks = true enable_iothreads = true - -kernel = "/usr/bin/../bin/id" -image = "/usr/bin/./echo" +jailer_path = "/usr/local" +kernel = "/usr/bin/../bin/uptime" +image = "/usr/bin/./lessecho" kernel_params = "ro" firmware = "/etc/hostname" @@ -17,7 +18,7 @@ default_maxvcpus = 64 machine_type = "q35" confidential_guest = true rootless = true -enable_annotations = ["path", "ctlpath"] +enable_annotations = ["path", "ctlpath","jailer_path"] machine_accelerators="noapic" default_bridges = 2 default_memory = 128 @@ -26,7 +27,7 @@ memory_offset = 0x100000 enable_virtio_mem = true disable_block_device_use = false shared_fs = "virtio-fs" -virtio_fs_daemon = "/usr/bin/id" +virtio_fs_daemon = "/usr/bin/uptime" valid_virtio_fs_daemon_paths = ["/usr/local/bin/virtiofsd*"] virtio_fs_cache_size = 512 virtio_fs_extra_args = ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] @@ -61,10 +62,17 @@ guest_memory_dump_path="/var/crash/kata" guest_memory_dump_paging = true enable_guest_swap = true +[agent.agent0] +enable_tracing = true +debug_console_enabled = true +debug = true +dial_timeout = 1 +kernel_modules = ["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1","i915_enabled_ppgtt=0"] +container_pipe_size = 2 [runtime] enable_debug = true internetworking_model="macvtap" -disable_guest_seccomp=true +pdisable_guest_seccomp=true enable_tracing = true jaeger_endpoint = "localhost:1234" jaeger_user = "user" @@ -76,3 +84,4 @@ vfio_mode="vfio" experimental=["a", "b"] enable_pprof = true + From 97d8c6c0fa2f537340c8587dee6467a26f2bb583 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 24 Jan 2022 20:36:00 +0800 Subject: [PATCH 0015/1953] docs: modify move-issues-to-in-progress.yaml change issue backlog to runtime-rs Signed-off-by: Zhongtao Hu --- .github/workflows/move-issues-to-in-progress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/move-issues-to-in-progress.yaml b/.github/workflows/move-issues-to-in-progress.yaml index 0e15abaea3e5..5ab9beb98dae 100644 --- a/.github/workflows/move-issues-to-in-progress.yaml +++ b/.github/workflows/move-issues-to-in-progress.yaml @@ -59,7 +59,7 @@ jobs: exit 1 } - project_name="Issue backlog" + project_name="runtime-rs" project_type="org" project_column="In progress" From 626828696ddae26793c78d78033b5152f72b4016 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 25 Jan 2022 11:47:02 +0800 Subject: [PATCH 0016/1953] libs/types: add license for test-config.rs add SPDX license identifier: Apache-2.0 Signed-off-by: Zhongtao Hu --- src/libs/kata-types/tests/test-config.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test-config.rs index 9a2061c7c7f2..0f49f28d92f7 100644 --- a/src/libs/kata-types/tests/test-config.rs +++ b/src/libs/kata-types/tests/test-config.rs @@ -1,3 +1,7 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// #[cfg(test)] mod tests { use kata_types::annotations::{ From 8ffff40af4eaacdeb91be82fe3450cf26a61559f Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 26 Jan 2022 14:21:55 +0800 Subject: [PATCH 0017/1953] libs/types:Option type to handle empty tomlconfig loading from empty string is only used to identity that the config is not initialized yet, so Option is a better option Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/config/mod.rs | 20 ++++++++++++-------- src/libs/kata-types/tests/test-config.rs | 18 +++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index c2380c299346..5d3b2d6bce7b 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -185,7 +185,7 @@ pub fn validate_path_pattern>(patterns: &[String], path: P) -> Re /// Kata configuration information. pub struct KataConfig { - config: TomlConfig, + config: Option, agent: String, hypervisor: String, } @@ -194,7 +194,7 @@ impl KataConfig { /// Set the default Kata configuration object. /// /// The default Kata configuration information is loaded from system configuration file. - pub fn set_default_config(config: TomlConfig, hypervisor: &str, agent: &str) { + pub fn set_default_config(config: Option, hypervisor: &str, agent: &str) { let kata = KataConfig { config, agent: agent.to_string(), @@ -214,7 +214,7 @@ impl KataConfig { /// /// The active Kata configuration information is default configuration information patched /// with tunable configuration information from annotations. - pub fn set_active_config(config: TomlConfig, hypervisor: &str, agent: &str) { + pub fn set_active_config(config: Option, hypervisor: &str, agent: &str) { let kata = KataConfig { config, agent: agent.to_string(), @@ -232,13 +232,13 @@ impl KataConfig { } /// Get the config in use pub fn get_config(&self) -> &TomlConfig { - &self.config + self.config.as_ref().unwrap() } /// Get the agent configuration in use. pub fn get_agent(&self) -> Option<&Agent> { if !self.agent.is_empty() { - self.config.agent.get(&self.agent) + self.config.as_ref().unwrap().agent.get(&self.agent) } else { None } @@ -247,7 +247,11 @@ impl KataConfig { /// Get the hypervisor configuration in use. pub fn get_hypervisor(&self) -> Option<&Hypervisor> { if !self.hypervisor.is_empty() { - self.config.hypervisor.get(&self.hypervisor) + self.config + .as_ref() + .unwrap() + .hypervisor + .get(&self.hypervisor) } else { None } @@ -256,7 +260,7 @@ impl KataConfig { lazy_static! { static ref KATA_DEFAULT_CONFIG: Mutex> = { - let config = TomlConfig::load("").unwrap(); + let config = Some(TomlConfig::load("").unwrap()); let kata = KataConfig { config, agent: String::new(), @@ -266,7 +270,7 @@ lazy_static! { Mutex::new(Arc::new(kata)) }; static ref KATA_ACTIVE_CONFIG: Mutex> = { - let config = TomlConfig::load("").unwrap(); + let config = Some(TomlConfig::load("").unwrap()); let kata = KataConfig { config, agent: String::new(), diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test-config.rs index 0f49f28d92f7..9f01f13e94d9 100644 --- a/src/libs/kata-types/tests/test-config.rs +++ b/src/libs/kata-types/tests/test-config.rs @@ -34,7 +34,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); std::process::Command::new("mkdir") .arg("./hypervisor_path") @@ -175,7 +175,7 @@ mod tests { assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") .is_ok()); - KataConfig::set_active_config(config, "qemu", "agnet0"); + KataConfig::set_active_config(Some(config), "qemu", "agnet0"); if let Some(ag) = KataConfig::get_default_config().get_agent() { assert_eq!( ag.kernel_modules[0], @@ -287,7 +287,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); anno_hash.insert( @@ -312,7 +312,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); anno_hash.insert( @@ -337,7 +337,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); anno_hash.insert( @@ -365,7 +365,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); anno_hash.insert( @@ -386,7 +386,7 @@ mod tests { let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); let content = fs::read_to_string(&path).unwrap(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let qemu = QemuConfig::new(); qemu.register(); @@ -414,7 +414,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); anno_hash.insert( @@ -439,7 +439,7 @@ mod tests { qemu.register(); let config = TomlConfig::load(&content).unwrap(); - KataConfig::set_active_config(config, "qemu", "agent0"); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); anno_hash.insert( From 2599a06a56980730871589b70e6e7273136f926a Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 31 Jan 2022 09:56:12 +0800 Subject: [PATCH 0018/1953] libs/types:use include_str! in test file use include_str! to load toml file to string fmt Signed-off-by: Zhongtao Hu --- src/libs/kata-types/tests/test-config.rs | 29 ++++++------------------ 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test-config.rs index 9f01f13e94d9..6c5e25ab30e8 100644 --- a/src/libs/kata-types/tests/test-config.rs +++ b/src/libs/kata-types/tests/test-config.rs @@ -26,10 +26,7 @@ mod tests { use std::path::Path; #[test] fn test_change_config_annotation() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); - let content = fs::read_to_string(&path).unwrap(); - + let content = include_str!("texture/configuration-anno-0.toml"); let qemu = QemuConfig::new(); qemu.register(); @@ -279,9 +276,7 @@ mod tests { #[test] fn test_fail_to_change_block_device_driver_because_not_enabled() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-1.toml"); - let content = fs::read_to_string(&path).unwrap(); + let content = include_str!("texture/configuration-anno-1.toml"); let qemu = QemuConfig::new(); qemu.register(); @@ -304,9 +299,7 @@ mod tests { #[test] fn test_fail_to_change_enable_guest_swap_because_not_enabled() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-1.toml"); - let content = fs::read_to_string(&path).unwrap(); + let content = include_str!("texture/configuration-anno-1.toml"); let qemu = QemuConfig::new(); qemu.register(); @@ -329,9 +322,7 @@ mod tests { #[test] fn test_fail_to_change_hypervisor_path_because_of_invalid_path() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); - let content = fs::read_to_string(&path).unwrap(); + let content = include_str!("texture/configuration-anno-0.toml"); let qemu = QemuConfig::new(); qemu.register(); @@ -382,9 +373,7 @@ mod tests { #[test] fn test_fail_to_change_memory_slots_because_of_less_than_zero() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); - let content = fs::read_to_string(&path).unwrap(); + let content = include_str!("texture/configuration-anno-0.toml"); let config = TomlConfig::load(&content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); @@ -406,9 +395,7 @@ mod tests { #[test] fn test_fail_to_change_default_memory_because_less_than_min_memory_size() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); - let content = fs::read_to_string(&path).unwrap(); + let content = include_str!("texture/configuration-anno-0.toml"); let qemu = QemuConfig::new(); qemu.register(); @@ -431,9 +418,7 @@ mod tests { #[test] fn test_fail_to_change_default_vcpus_becuase_more_than_max_cpu_size() { - let path = env!("CARGO_MANIFEST_DIR"); - let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); - let content = fs::read_to_string(&path).unwrap(); + let content = include_str!("texture/configuration-anno-0.toml"); let qemu = QemuConfig::new(); qemu.register(); From 45e5780e7cf23a3a6bca30bc8648c0e617fd3c60 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 31 Jan 2022 10:14:28 +0800 Subject: [PATCH 0019/1953] libs/types: fixed spelling and grammer error fixed spelling and grammer error in some files Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/annotations/mod.rs | 10 +++++----- src/libs/kata-types/src/config/agent.rs | 2 +- src/libs/kata-types/src/config/default.rs | 8 ++++---- .../kata-types/src/config/hypervisor/dragonball.rs | 4 ++-- src/libs/kata-types/src/config/hypervisor/mod.rs | 10 +++++----- src/libs/kata-types/src/config/hypervisor/qemu.rs | 10 +++++----- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 43f23e727a82..27103ef621b5 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -325,12 +325,12 @@ impl Annotation { } /// Get an immutable reference to the annotation hashmap. - pub fn get_annotation(&self) -> &HashMap { + pub fn get_annotations(&self) -> &HashMap { &self.annotations } /// Get a mutable reference to the annotation hashmap. - pub fn get_annotation_mut(&mut self) -> &mut HashMap { + pub fn get_annotations_mut(&mut self) -> &mut HashMap { &mut self.annotations } @@ -609,7 +609,7 @@ impl Annotation { return Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "Memory specified in annotation {} is less than minmum required {}", + "Memory specified in annotation {} is less than minimum required {}", mem, get_hypervisor_plugin(hypervisor_name) .unwrap() @@ -627,7 +627,7 @@ impl Annotation { None => { return Err(io::Error::new( io::ErrorKind::InvalidData, - format!("{}in annotation is less than zero", key), + format!("{} in annotation is less than zero", key), )); } }, @@ -715,7 +715,7 @@ impl Annotation { _ => { return Err(io::Error::new( io::ErrorKind::InvalidInput, - format!("Invalid Annotation Type {}", key), + format!("Invalid annotation type {}", key), )); } } diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 9f4f0abbb4e8..82d3eb72ff86 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -51,7 +51,7 @@ pub struct Agent { #[serde(default)] pub kernel_modules: Vec, - /// contianer pipe size + /// container pipe size pub container_pipe_size: u32, } diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 829a1d7ddc99..b5b4029fbf65 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -35,8 +35,8 @@ pub const DEFAULT_GUEST_HOOK_PATH: &str = "/opt"; pub const DEFAULT_GUEST_VCPUS: u32 = 1; // Default configuration for Dragonball -pub const DEFAULT_DB_GUEST_KENREL_IMAGE: &str = "vmlinuz"; -pub const DEFAULT_DB_GUEST_KENREL_PARAMS: &str = ""; +pub const DEFAULT_DB_GUEST_KERNEL_IMAGE: &str = "vmlinuz"; +pub const DEFAULT_DB_GUEST_KERNEL_PARAMS: &str = ""; pub const DEFAULT_DB_ENTROPY_SOURCE: &str = "/dev/urandom"; pub const DEFAULT_DB_MEMORY_SIZE: u32 = 128; pub const DEFAULT_DB_MEMORY_SLOTS: u32 = 128; @@ -47,8 +47,8 @@ pub const DEFAULT_QEMU_BINARY_PATH: &str = "qemu"; pub const DEFAULT_QEMU_CONTROL_PATH: &str = ""; pub const DEFAULT_QEMU_MACHINE_TYPE: &str = "q35"; pub const DEFAULT_QEMU_ENTROPY_SOURCE: &str = "/dev/urandom"; -pub const DEFAULT_QEMU_GUEST_KENREL_IMAGE: &str = "vmlinuz"; -pub const DEFAULT_QEMU_GUEST_KENREL_PARAMS: &str = ""; +pub const DEFAULT_QEMU_GUEST_KERNEL_IMAGE: &str = "vmlinuz"; +pub const DEFAULT_QEMU_GUEST_KERNEL_PARAMS: &str = ""; pub const DEFAULT_QEMU_FIRMWARE_PATH: &str = ""; pub const DEFAULT_QEMU_MEMORY_SIZE: u32 = 128; pub const DEFAULT_QEMU_MEMORY_SLOTS: u32 = 128; diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs index c0b553c223d6..95158019fb2f 100644 --- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -54,10 +54,10 @@ impl ConfigPlugin for DragonballConfig { resolve_path!(db.jailer_path, "Dragonball jailer path {} is invalid: {}")?; if db.boot_info.kernel.is_empty() { - db.boot_info.kernel = default::DEFAULT_DB_GUEST_KENREL_IMAGE.to_string(); + db.boot_info.kernel = default::DEFAULT_DB_GUEST_KERNEL_IMAGE.to_string(); } if db.boot_info.kernel_params.is_empty() { - db.boot_info.kernel_params = default::DEFAULT_DB_GUEST_KENREL_PARAMS.to_string(); + db.boot_info.kernel_params = default::DEFAULT_DB_GUEST_KERNEL_PARAMS.to_string(); } if db.cpu_info.default_maxvcpus > default::MAX_DB_VCPUS { diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 4e083f9ba255..da8e206503bf 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -336,7 +336,7 @@ pub struct DebugInfo { /// exist. The dumped file(also called vmcore) can be processed with crash or gdb. /// /// # WARNING: - /// Dump guest’s memory can take very long depending on the amount of guest memory and use + /// Dump guest's memory can take very long depending on the amount of guest memory and use /// much disk space. #[serde(default)] pub guest_memory_dump_path: String, @@ -415,7 +415,7 @@ impl DeviceInfo { pub fn validate(&self) -> Result<()> { if self.default_bridges > 5 { return Err(eother!( - "The configured PCI bridges {} is too big", + "The configured PCI bridges {} are too many", self.default_bridges )); } @@ -479,7 +479,7 @@ impl MachineInfo { /// Validate the configuration information. pub fn validate(&self) -> Result<()> { for pflash in self.pflashes.iter() { - validate_path!(*pflash, "Flash image file {} is invalide: {}")?; + validate_path!(*pflash, "Flash image file {} is invalid: {}")?; } validate_path!(self.entropy_source, "Entropy source {} is invalid: {}")?; Ok(()) @@ -578,10 +578,10 @@ impl MemoryInfo { "Memory backend file {} is invalid: {}" )?; if self.default_memory == 0 { - return Err(eother!("Configured memory size for guest vm is zero")); + return Err(eother!("Configured memory size for guest VM is zero")); } if self.memory_slots == 0 { - return Err(eother!("Configured memory slots for guest vm is zero")); + return Err(eother!("Configured memory slots for guest VM are zero")); } Ok(()) diff --git a/src/libs/kata-types/src/config/hypervisor/qemu.rs b/src/libs/kata-types/src/config/hypervisor/qemu.rs index 6c59f54e9545..eb14779bd2e8 100644 --- a/src/libs/kata-types/src/config/hypervisor/qemu.rs +++ b/src/libs/kata-types/src/config/hypervisor/qemu.rs @@ -61,11 +61,11 @@ impl ConfigPlugin for QemuConfig { resolve_path!(qemu.ctlpath, "Qemu ctlpath `{}` is invalid: {}")?; if qemu.boot_info.kernel.is_empty() { - qemu.boot_info.kernel = default::DEFAULT_QEMU_GUEST_KENREL_IMAGE.to_string(); + qemu.boot_info.kernel = default::DEFAULT_QEMU_GUEST_KERNEL_IMAGE.to_string(); } if qemu.boot_info.kernel_params.is_empty() { qemu.boot_info.kernel_params = - default::DEFAULT_QEMU_GUEST_KENREL_PARAMS.to_string(); + default::DEFAULT_QEMU_GUEST_KERNEL_PARAMS.to_string(); } if qemu.boot_info.firmware.is_empty() { qemu.boot_info.firmware = default::DEFAULT_QEMU_FIRMWARE_PATH.to_string(); @@ -99,10 +99,10 @@ impl ConfigPlugin for QemuConfig { /// Validate the configuration information. fn validate(&self, conf: &TomlConfig) -> Result<()> { if let Some(qemu) = conf.hypervisor.get(HYPERVISOR_NAME_QEMU) { - validate_path!(qemu.path, "Qemu binary path `{}` is invalid: {}")?; - validate_path!(qemu.ctlpath, "Qemu control path `{}` is invalid: {}")?; + validate_path!(qemu.path, "QEMU binary path `{}` is invalid: {}")?; + validate_path!(qemu.ctlpath, "QEMU control path `{}` is invalid: {}")?; if !qemu.jailer_path.is_empty() { - return Err(eother!("Path for Qemu jailer should be empty")); + return Err(eother!("Path for QEMU jailer should be empty")); } if !qemu.valid_jailer_paths.is_empty() { return Err(eother!("Valid Qemu jailer path list should be empty")); From 6ae87d9d661e721ee3b3f249817b7f20b4f4d652 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 31 Jan 2022 11:25:13 +0800 Subject: [PATCH 0020/1953] libs/types: use contains to make code more readable use contains to when validate hypervisor block_device_driver Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/config/hypervisor/mod.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index da8e206503bf..4aa4ceba737c 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -165,12 +165,14 @@ impl BlockDeviceInfo { if self.disable_block_device_use { return Ok(()); } - if self.block_device_driver != VIRTIO_BLK - && self.block_device_driver != VIRTIO_BLK_CCW - && self.block_device_driver != VIRTIO_BLK_MMIO - && self.block_device_driver != VIRTIO_SCSI - && self.block_device_driver != VIRTIO_PMEM - { + let l = [ + VIRTIO_BLK, + VIRTIO_BLK_CCW, + VIRTIO_BLK_MMIO, + VIRTIO_SCSI, + VIRTIO_PMEM, + ]; + if !l.contains(&self.block_device_driver.as_str()) { return Err(eother!( "{} is unsupported block device type.", self.block_device_driver From 6cffd943be5585dad66b0c396463e2e61f08dc5f Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Fri, 4 Feb 2022 11:47:15 +0800 Subject: [PATCH 0021/1953] libs/types:return Result to handle parse error If there is a parse error when we are trying to get the annotations, we will return Result> to handle that. Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/annotations/mod.rs | 559 +++++++++++++-------- src/libs/kata-types/tests/test-config.rs | 46 ++ 2 files changed, 395 insertions(+), 210 deletions(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 27103ef621b5..8aa7c2319c92 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -13,7 +13,6 @@ use serde::Deserialize; use crate::config::hypervisor::get_hypervisor_plugin; use crate::config::TomlConfig; -use crate::sl; /// CRI-containerd specific annotations. pub mod cri_containerd; @@ -336,79 +335,71 @@ impl Annotation { /// Get the value of annotation with `key` as string. pub fn get(&self, key: &str) -> Option { - let v = self.annotations.get(key)?; - let value = v.trim(); - - if !value.is_empty() { - Some(String::from(value)) - } else { - None - } + self.annotations.get(key).map(|v| String::from(v.trim())) } /// Get the value of annotation with `key` as bool. - pub fn get_bool(&self, key: &str) -> Option { + pub fn get_bool(&self, key: &str) -> Result> { if let Some(value) = self.get(key) { - let value = value.trim(); - if value.parse::().is_err() { - warn!(sl!(), "failed to parse bool value from {}", value); - } else { - return Some(value.parse::().unwrap()); - } + return value + .parse::() + .map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid input {} for bool", key), + ) + }) + .map(Some); } - - None + Ok(None) } /// Get the value of annotation with `key` as u32. - pub fn get_u32(&self, key: &str) -> Option { - let s = self.get(key)?; - match s.parse::() { - Ok(nums) => { - if nums > 0 { - Some(nums) - } else { - None - } - } - - Err(e) => { - warn!( - sl!(), - "failed to parse u32 value from {}, error: {:?}", s, e - ); - None - } + pub fn get_u32(&self, key: &str) -> Result> { + if let Some(value) = self.get(key) { + return value + .parse::() + .map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid input {} for u32", key), + ) + }) + .map(Some); } + Ok(None) } /// Get the value of annotation with `key` as i32. - pub fn get_i32(&self, key: &str) -> Option { - let s = self.get(key)?; - s.parse::() - .map_or(None, |x| if x < 0 { None } else { Some(x) }) + pub fn get_i32(&self, key: &str) -> Result> { + if let Some(value) = self.get(key) { + return value + .parse::() + .map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid input {} for i32", key), + ) + }) + .map(Some); + } + Ok(None) } /// Get the value of annotation with `key` as u64. - pub fn get_u64(&self, key: &str) -> Option { - let s = self.get(key)?; - match s.parse::() { - Ok(nums) => { - if nums > 0 { - Some(nums) - } else { - None - } - } - - Err(e) => { - warn!( - sl!(), - "failed to parse u64 value from {}, error: {:?}", s, e - ); - None - } + pub fn get_u64(&self, key: &str) -> Result> { + if let Some(value) = self.get(key) { + return value + .parse::() + .map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid input {} for u64", key), + ) + }) + .map(Some); } + Ok(None) } } @@ -430,12 +421,19 @@ impl Annotation { } /// Get the annotation to specify the Resources.Memory.Swappiness. - pub fn get_container_resource_swappiness(&self) -> Option { - let v = self.get_u32(KATA_ANNO_CONTAINER_RESOURCE_SWAPPINESS)?; - if v > 100 { - None - } else { - Some(v) + pub fn get_container_resource_swappiness(&self) -> Result> { + match self.get_u32(KATA_ANNO_CONTAINER_RESOURCE_SWAPPINESS) { + Ok(r) => { + if r.unwrap_or_default() > 100 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{} greater than 100", r.unwrap_or_default()), + )); + } else { + Ok(r) + } + } + Err(e) => Err(e), } } @@ -486,40 +484,79 @@ impl Annotation { hv.validate_jailer_path(value)?; hv.jailer_path = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS => { - hv.enable_iothreads = self.get_bool(key).unwrap_or_default(); - } + KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS => match self.get_bool(key) { + Ok(r) => { + hv.enable_iothreads = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, // Hypervisor Block Device related annotations KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER => { hv.blockdev_info.block_device_driver = value.to_string(); } KATA_ANNO_CONF_HYPERVISOR_DISABLE_BLOCK_DEVICE_USE => { - hv.blockdev_info.disable_block_device_use = - self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_SET => { - hv.blockdev_info.block_device_cache_set = - self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_DIRECT => { - hv.blockdev_info.block_device_cache_direct = - self.get_bool(key).unwrap_or_default(); + match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.disable_block_device_use = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + } } + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_SET => match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.block_device_cache_set = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_DIRECT => match self.get_bool(key) + { + Ok(r) => { + hv.blockdev_info.block_device_cache_direct = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH => { - hv.blockdev_info.block_device_cache_noflush = - self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_DISABLE_IMAGE_NVDIMM => { - hv.blockdev_info.disable_image_nvdimm = - self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_MEMORY_OFFSET => { - hv.blockdev_info.memory_offset = self.get_u64(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_ENABLE_VHOSTUSER_STORE => { - hv.blockdev_info.enable_vhost_user_store = - self.get_bool(key).unwrap_or_default(); + match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.block_device_cache_noflush = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + } } + KATA_ANNO_CONF_HYPERVISOR_DISABLE_IMAGE_NVDIMM => match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.disable_image_nvdimm = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_MEMORY_OFFSET => match self.get_u64(key) { + Ok(r) => { + hv.blockdev_info.memory_offset = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_ENABLE_VHOSTUSER_STORE => match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.enable_vhost_user_store = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH => { hv.blockdev_info.validate_vhost_user_store_path(value)?; hv.blockdev_info.vhost_user_store_path = value.to_string(); @@ -548,45 +585,75 @@ impl Annotation { KATA_ANNO_CONF_HYPERVISOR_CPU_FEATURES => { hv.cpu_info.cpu_features = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS => { - let num_cpus = self.get_i32(key).unwrap_or_default(); - if num_cpus - > get_hypervisor_plugin(hypervisor_name) - .unwrap() - .get_max_cpus() as i32 - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "Vcpus specified in annotation {} is more than maximum limitation {}", - num_cpus, - get_hypervisor_plugin(hypervisor_name) - .unwrap() - .get_max_cpus() - ), - )); - } else { - hv.cpu_info.default_vcpus = num_cpus; + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS => match self.get_i32(key) { + Ok(num_cpus) => { + let num_cpus = num_cpus.unwrap_or_default(); + if num_cpus + > get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_max_cpus() as i32 + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Vcpus specified in annotation {} is more than maximum limitation {}", + num_cpus, + get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_max_cpus() + ), + )); + } else { + hv.cpu_info.default_vcpus = num_cpus; + } } - } - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MAX_VCPUS => { - hv.cpu_info.default_maxvcpus = self.get_u32(key).unwrap_or_default(); - } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MAX_VCPUS => match self.get_u32(key) { + Ok(r) => { + hv.cpu_info.default_maxvcpus = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, // Hypervisor Device related annotations KATA_ANNO_CONF_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => { - hv.device_info.hotplug_vfio_on_root_bus = - self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_PCIE_ROOT_PORT => { - hv.device_info.pcie_root_port = self.get_u32(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_IOMMU => { - hv.device_info.enable_iommu = self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_IOMMU_PLATFORM => { - hv.device_info.enable_iommu_platform = - self.get_bool(key).unwrap_or_default(); + match self.get_bool(key) { + Ok(r) => { + hv.device_info.hotplug_vfio_on_root_bus = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + } } + KATA_ANNO_CONF_HYPERVISOR_PCIE_ROOT_PORT => match self.get_u32(key) { + Ok(r) => { + hv.device_info.pcie_root_port = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_IOMMU => match self.get_bool(key) { + Ok(r) => { + hv.device_info.enable_iommu = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_IOMMU_PLATFORM => match self.get_bool(key) { + Ok(r) => { + hv.device_info.enable_iommu_platform = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, // Hypervisor Machine related annotations KATA_ANNO_CONF_HYPERVISOR_MACHINE_TYPE => { hv.machine_info.machine_type = value.to_string(); @@ -599,82 +666,108 @@ impl Annotation { hv.machine_info.entropy_source = value.to_string(); } // Hypervisor Memory related annotations - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY => { - let mem = self.get_u32(key).unwrap_or_default(); - if mem - < get_hypervisor_plugin(hypervisor_name) - .unwrap() - .get_min_memory() - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "Memory specified in annotation {} is less than minimum required {}", - mem, - get_hypervisor_plugin(hypervisor_name) - .unwrap() - .get_min_memory() - ), - )); - } else { - hv.memory_info.default_memory = mem; + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY => match self.get_u32(key) { + Ok(r) => { + let mem = r.unwrap_or_default(); + if mem + < get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_min_memory() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Memory specified in annotation {} is less than minimum required {}", + mem, + get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_min_memory() + ), + )); + } else { + hv.memory_info.default_memory = mem; + } } - } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS => match self.get_u32(key) { - Some(v) => { - hv.memory_info.memory_slots = v; + Ok(v) => { + hv.memory_info.memory_slots = v.unwrap_or_default(); } - None => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("{} in annotation is less than zero", key), - )); + Err(e) => { + return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC => { - hv.memory_info.enable_mem_prealloc = self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES => { - hv.memory_info.enable_hugepages = self.get_bool(key).unwrap_or_default(); - } + KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC => match self.get_bool(key) { + Ok(r) => { + hv.memory_info.enable_mem_prealloc = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES => match self.get_bool(key) { + Ok(r) => { + hv.memory_info.enable_hugepages = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { hv.memory_info.validate_memory_backend_path(value)?; hv.memory_info.file_mem_backend = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM => { - hv.memory_info.enable_virtio_mem = self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP => { - hv.memory_info.enable_swap = self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP => { - hv.memory_info.enable_guest_swap = self.get_bool(key).unwrap_or_default(); - } + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM => match self.get_bool(key) { + Ok(r) => { + hv.memory_info.enable_virtio_mem = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP => match self.get_bool(key) { + Ok(r) => { + hv.memory_info.enable_swap = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP => match self.get_bool(key) { + Ok(r) => { + hv.memory_info.enable_guest_swap = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, // Hypervisor Network related annotations - KATA_ANNO_CONF_HYPERVISOR_DISABLE_VHOST_NET => { - hv.network_info.disable_vhost_net = self.get_bool(key).unwrap_or_default(); - } + KATA_ANNO_CONF_HYPERVISOR_DISABLE_VHOST_NET => match self.get_bool(key) { + Ok(r) => { + hv.network_info.disable_vhost_net = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { - Some(v) => { - hv.network_info.rx_rate_limiter_max_rate = v; + Ok(r) => { + hv.network_info.rx_rate_limiter_max_rate = r.unwrap_or_default(); } - None => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("{} in annotation is less than zero", key), - )); + Err(e) => { + return Err(e); } }, KATA_ANNO_CONF_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { - Some(v) => { - hv.network_info.tx_rate_limiter_max_rate = v; + Ok(r) => { + hv.network_info.tx_rate_limiter_max_rate = r.unwrap_or_default(); } - None => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("{} in annotation is less than zero", key), - )); + Err(e) => { + return Err(e); } }, // Hypervisor Security related annotations @@ -683,7 +776,14 @@ impl Annotation { hv.security_info.guest_hook_path = value.to_string(); } KATA_ANNO_CONF_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR => { - hv.security_info.rootless = self.get_bool(key).unwrap_or_default(); + match self.get_bool(key) { + Ok(r) => { + hv.security_info.rootless = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + } } // Hypervisor Shared File System related annotations KATA_ANNO_CONF_HYPERVISOR_SHARED_FS => { @@ -698,9 +798,14 @@ impl Annotation { KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE => { hv.shared_fs.virtio_fs_cache = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => { - hv.shared_fs.virtio_fs_cache_size = self.get_u32(key).unwrap_or_default(); - } + KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => match self.get_u32(key) { + Ok(r) => { + hv.shared_fs.virtio_fs_cache_size = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS => { let args: Vec = value.to_string().split(',').map(str::to_string).collect(); @@ -708,9 +813,14 @@ impl Annotation { hv.shared_fs.virtio_fs_extra_args.push(arg.to_string()); } } - KATA_ANNO_CONF_HYPERVISOR_MSIZE_9P => { - hv.shared_fs.msize_9p = self.get_u32(key).unwrap_or_default(); - } + KATA_ANNO_CONF_HYPERVISOR_MSIZE_9P => match self.get_u32(key) { + Ok(v) => { + hv.shared_fs.msize_9p = v.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, _ => { return Err(io::Error::new( @@ -729,20 +839,39 @@ impl Annotation { ag.kernel_modules.push(modules.to_string()); } } - KATA_ANNO_CONF_AGENT_TRACE => { - ag.enable_tracing = self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE => { - ag.container_pipe_size = self.get_u32(key).unwrap_or_default(); - } + KATA_ANNO_CONF_AGENT_TRACE => match self.get_bool(key) { + Ok(r) => { + ag.enable_tracing = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE => match self.get_u32(key) { + Ok(v) => { + ag.container_pipe_size = v.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, //update runtume config - KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP => { - config.runtime.disable_guest_seccomp = - self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_ENABLE_PPROF => { - config.runtime.enable_pprof = self.get_bool(key).unwrap_or_default(); - } + KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP => match self.get_bool(key) { + Ok(r) => { + config.runtime.disable_guest_seccomp = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_ENABLE_PPROF => match self.get_bool(key) { + Ok(r) => { + config.runtime.enable_pprof = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_EXPERIMENTAL => { let args: Vec = value.to_string().split(',').map(str::to_string).collect(); @@ -753,12 +882,22 @@ impl Annotation { KATA_ANNO_CONF_INTER_NETWORK_MODEL => { config.runtime.internetworking_model = value.to_string(); } - KATA_ANNO_CONF_SANDBOX_CGROUP_ONLY => { - config.runtime.disable_new_netns = self.get_bool(key).unwrap_or_default(); - } - KATA_ANNO_CONF_DISABLE_NEW_NETNS => { - config.runtime.disable_new_netns = self.get_bool(key).unwrap_or_default(); - } + KATA_ANNO_CONF_SANDBOX_CGROUP_ONLY => match self.get_bool(key) { + Ok(r) => { + config.runtime.sandbox_cgroup_only = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CONF_DISABLE_NEW_NETNS => match self.get_bool(key) { + Ok(r) => { + config.runtime.disable_new_netns = r.unwrap_or_default(); + } + Err(e) => { + return Err(e); + } + }, KATA_ANNO_CONF_VFIO_MODE => { config.runtime.vfio_mode = value.to_string(); } diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test-config.rs index 6c5e25ab30e8..cbd134c3cd4c 100644 --- a/src/libs/kata-types/tests/test-config.rs +++ b/src/libs/kata-types/tests/test-config.rs @@ -438,4 +438,50 @@ mod tests { .update_config_by_annotation(&mut config, "qemu", "agent0") .is_err()); } + + #[test] + fn test_fail_to_change_enable_guest_swap_because_invalid_input() { + let content = include_str!("texture/configuration-anno-0.toml"); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), + "false1".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } + + #[test] + fn test_fail_to_change_default_vcpus_becuase_invalid_input() { + let content = include_str!("texture/configuration-anno-0.toml"); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(&content).unwrap(); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); + + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS.to_string(), + "ddc".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(&content).unwrap(); + + assert!(anno + .update_config_by_annotation(&mut config, "qemu", "agent0") + .is_err()); + } } From d96716b4d27cc54ab7aa549ec41a1abfcb299ce2 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Fri, 11 Feb 2022 17:10:23 +0800 Subject: [PATCH 0022/1953] libs/types:fix styles and implementation details 1. Some Nit problems are fixed 2. Make the code more readable 3. Modify some implementation details Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/annotations/mod.rs | 312 +++++++++--------- .../kata-types/src/annotations/thirdparty.rs | 2 - src/libs/kata-types/src/config/default.rs | 2 +- .../kata-types/src/config/hypervisor/mod.rs | 26 +- .../kata-types/src/config/hypervisor/qemu.rs | 3 - .../kata-types/src/config/runtime_vendor.rs | 4 - .../tests/{test-config.rs => test_config.rs} | 98 +++--- 7 files changed, 214 insertions(+), 233 deletions(-) rename src/libs/kata-types/tests/{test-config.rs => test_config.rs} (80%) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 8aa7c2319c92..ea4e5c03a0ef 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -30,11 +30,11 @@ pub mod thirdparty; /// Prefix for Kata specific annotations pub const KATA_ANNO_PREFIX: &str = "io.katacontainers."; /// Prefix for Kata configuration annotations -pub const KATA_ANNO_CONF_PREFIX: &str = "io.katacontainers.config."; +pub const KATA_ANNO_CFG_PREFIX: &str = "io.katacontainers.config."; /// Prefix for Kata container annotations pub const KATA_ANNO_CONTAINER_PREFIX: &str = "io.katacontainers.container."; /// The annotation key to fetch runtime configuration file. -pub const SANDBOX_CONFIG_PATH_KEY: &str = "io.katacontainers.config_path"; +pub const SANDBOX_CFG_PATH_KEY: &str = "io.katacontainers.config_path"; // OCI section /// The annotation key to fetch the OCI configuration file path. @@ -44,17 +44,17 @@ pub const CONTAINER_TYPE_KEY: &str = "io.katacontainers.pkg.oci.container_type"; // Container resource related annotations /// Prefix for Kata container resource related annotations. -pub const KATA_ANNO_CONTAINER_RESOURCE_PREFIX: &str = "io.katacontainers.container.resource"; +pub const KATA_ANNO_CONTAINER_RES_PREFIX: &str = "io.katacontainers.container.resource"; /// A container annotation to specify the Resources.Memory.Swappiness. -pub const KATA_ANNO_CONTAINER_RESOURCE_SWAPPINESS: &str = +pub const KATA_ANNO_CONTAINER_RES_SWAPPINESS: &str = "io.katacontainers.container.resource.swappiness"; /// A container annotation to specify the Resources.Memory.Swap. -pub const KATA_ANNO_CONTAINER_RESOURCE_SWAP_IN_BYTES: &str = +pub const KATA_ANNO_CONTAINER_RES_SWAP_IN_BYTES: &str = "io.katacontainers.container.resource.swap_in_bytes"; // Agent related annotations /// Prefix for Agent configurations. -pub const KATA_ANNO_CONF_AGENT_PREFIX: &str = "io.katacontainers.config.agent."; +pub const KATA_ANNO_CFG_AGENT_PREFIX: &str = "io.katacontainers.config.agent."; /// KernelModules is the annotation key for passing the list of kernel modules and their parameters /// that will be loaded in the guest kernel. /// @@ -66,235 +66,232 @@ pub const KATA_ANNO_CONF_AGENT_PREFIX: &str = "io.katacontainers.config.agent."; /// io.katacontainers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" /// /// The first word is considered as the module name and the rest as its parameters. -pub const KATA_ANNO_CONF_KERNEL_MODULES: &str = "io.katacontainers.config.agent.kernel_modules"; +pub const KATA_ANNO_CFG_KERNEL_MODULES: &str = "io.katacontainers.config.agent.kernel_modules"; /// A sandbox annotation to enable tracing for the agent. -pub const KATA_ANNO_CONF_AGENT_TRACE: &str = "io.katacontainers.config.agent.enable_tracing"; +pub const KATA_ANNO_CFG_AGENT_TRACE: &str = "io.katacontainers.config.agent.enable_tracing"; /// An annotation to specify the size of the pipes created for containers. -pub const KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE: &str = +pub const KATA_ANNO_CFG_AGENT_CONTAINER_PIPE_SIZE: &str = "io.katacontainers.config.agent.container_pipe_size"; /// An annotation key to specify the size of the pipes created for containers. pub const CONTAINER_PIPE_SIZE_KERNEL_PARAM: &str = "agent.container_pipe_size"; // Hypervisor related annotations /// Prefix for Hypervisor configurations. -pub const KATA_ANNO_CONF_HYPERVISOR_PREFIX: &str = "io.katacontainers.config.hypervisor."; +pub const KATA_ANNO_CFG_HYPERVISOR_PREFIX: &str = "io.katacontainers.config.hypervisor."; /// A sandbox annotation for passing a per container path pointing at the hypervisor that will run /// the container VM. -pub const KATA_ANNO_CONF_HYPERVISOR_PATH: &str = "io.katacontainers.config.hypervisor.path"; +pub const KATA_ANNO_CFG_HYPERVISOR_PATH: &str = "io.katacontainers.config.hypervisor.path"; /// A sandbox annotation for passing a container hypervisor binary SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_HASH: &str = "io.katacontainers.config.hypervisor.path_hash"; +pub const KATA_ANNO_CFG_HYPERVISOR_HASH: &str = "io.katacontainers.config.hypervisor.path_hash"; /// A sandbox annotation for passing a per container path pointing at the hypervisor control binary /// that will run the container VM. -pub const KATA_ANNO_CONF_HYPERVISOR_CTLPATH: &str = "io.katacontainers.config.hypervisor.ctlpath"; +pub const KATA_ANNO_CFG_HYPERVISOR_CTLPATH: &str = "io.katacontainers.config.hypervisor.ctlpath"; /// A sandbox annotation for passing a container hypervisor control binary SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_CTLHASH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_CTLHASH: &str = "io.katacontainers.config.hypervisor.hypervisorctl_hash"; /// A sandbox annotation for passing a per container path pointing at the jailer that will constrain /// the container VM. -pub const KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH: &str = "io.katacontainers.config.hypervisor.jailer_path"; /// A sandbox annotation for passing a jailer binary SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_JAILER_HASH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_JAILER_HASH: &str = "io.katacontainers.config.hypervisor.jailer_hash"; /// A sandbox annotation to enable IO to be processed in a separate thread. /// Supported currently for virtio-scsi driver. -pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS: &str = "io.katacontainers.config.hypervisor.enable_iothreads"; /// The hash type used for assets verification -pub const KATA_ANNO_CONF_HYPERVISOR_ASSET_HASH_TYPE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ASSET_HASH_TYPE: &str = "io.katacontainers.config.hypervisor.asset_hash_type"; /// SHA512 is the SHA-512 (64) hash algorithm pub const SHA512: &str = "sha512"; // Hypervisor Block Device related annotations /// Specify the driver to be used for block device either VirtioSCSI or VirtioBlock -pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER: &str = "io.katacontainers.config.hypervisor.block_device_driver"; /// A sandbox annotation that disallows a block device from being used. -pub const KATA_ANNO_CONF_HYPERVISOR_DISABLE_BLOCK_DEVICE_USE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_DISABLE_BLOCK_DEV_USE: &str = "io.katacontainers.config.hypervisor.disable_block_device_use"; /// A sandbox annotation that specifies cache-related options will be set to block devices or not. -pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_SET: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_SET: &str = "io.katacontainers.config.hypervisor.block_device_cache_set"; /// A sandbox annotation that specifies cache-related options for block devices. /// Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. -pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_DIRECT: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_DIRECT: &str = "io.katacontainers.config.hypervisor.block_device_cache_direct"; /// A sandbox annotation that specifies cache-related options for block devices. /// Denotes whether flush requests for the device are ignored. -pub const KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH: &str = "io.katacontainers.config.hypervisor.block_device_cache_noflush"; /// A sandbox annotation to specify use of nvdimm device for guest rootfs image. -pub const KATA_ANNO_CONF_HYPERVISOR_DISABLE_IMAGE_NVDIMM: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_DISABLE_IMAGE_NVDIMM: &str = "io.katacontainers.config.hypervisor.disable_image_nvdimm"; /// A sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_MEMORY_OFFSET: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_OFFSET: &str = "io.katacontainers.config.hypervisor.memory_offset"; /// A sandbox annotation to specify if vhost-user-blk/scsi is abailable on the host -pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_VHOSTUSER_STORE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_VHOSTUSER_STORE: &str = "io.katacontainers.config.hypervisor.enable_vhost_user_store"; /// A sandbox annotation to specify the directory path where vhost-user devices related folders, /// sockets and device nodes should be. -pub const KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH: &str = "io.katacontainers.config.hypervisor.vhost_user_store_path"; // Hypervisor Guest Boot related annotations /// A sandbox annotation for passing a per container path pointing at the kernel needed to boot /// the container VM. -pub const KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH: &str = - "io.katacontainers.config.hypervisor.kernel"; +pub const KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH: &str = "io.katacontainers.config.hypervisor.kernel"; /// A sandbox annotation for passing a container kernel image SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_KERNEL_HASH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_KERNEL_HASH: &str = "io.katacontainers.config.hypervisor.kernel_hash"; /// A sandbox annotation for passing a per container path pointing at the guest image that will run /// in the container VM. /// A sandbox annotation for passing additional guest kernel parameters. -pub const KATA_ANNO_CONF_HYPERVISOR_KERNEL_PARAMS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_KERNEL_PARAMS: &str = "io.katacontainers.config.hypervisor.kernel_params"; /// A sandbox annotation for passing a container guest image path. -pub const KATA_ANNO_CONF_HYPERVISOR_IMAGE_PATH: &str = "io.katacontainers.config.hypervisor.image"; +pub const KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH: &str = "io.katacontainers.config.hypervisor.image"; /// A sandbox annotation for passing a container guest image SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_IMAGE_HASH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_IMAGE_HASH: &str = "io.katacontainers.config.hypervisor.image_hash"; /// A sandbox annotation for passing a per container path pointing at the initrd that will run /// in the container VM. -pub const KATA_ANNO_CONF_HYPERVISOR_INITRD_PATH: &str = - "io.katacontainers.config.hypervisor.initrd"; +pub const KATA_ANNO_CFG_HYPERVISOR_INITRD_PATH: &str = "io.katacontainers.config.hypervisor.initrd"; /// A sandbox annotation for passing a container guest initrd SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_INITRD_HASH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_INITRD_HASH: &str = "io.katacontainers.config.hypervisor.initrd_hash"; /// A sandbox annotation for passing a per container path pointing at the guest firmware that will /// run the container VM. -pub const KATA_ANNO_CONF_HYPERVISOR_FIRMWARE_PATH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_FIRMWARE_PATH: &str = "io.katacontainers.config.hypervisor.firmware"; /// A sandbox annotation for passing a container guest firmware SHA-512 hash value. -pub const KATA_ANNO_CONF_HYPERVISOR_FIRMWARE_HASH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_FIRMWARE_HASH: &str = "io.katacontainers.config.hypervisor.firmware_hash"; // Hypervisor CPU related annotations /// A sandbox annotation to specify cpu specific features. -pub const KATA_ANNO_CONF_HYPERVISOR_CPU_FEATURES: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES: &str = "io.katacontainers.config.hypervisor.cpu_features"; /// A sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS: &str = "io.katacontainers.config.hypervisor.default_vcpus"; /// A sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MAX_VCPUS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MAX_VCPUS: &str = "io.katacontainers.config.hypervisor.default_max_vcpus"; // Hypervisor Device related annotations /// A sandbox annotation used to indicate if devices need to be hotplugged on the root bus instead /// of a bridge. -pub const KATA_ANNO_CONF_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS: &str = "io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus"; /// PCIeRootPort is used to indicate the number of PCIe Root Port devices -pub const KATA_ANNO_CONF_HYPERVISOR_PCIE_ROOT_PORT: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_PCIE_ROOT_PORT: &str = "io.katacontainers.config.hypervisor.pcie_root_port"; /// A sandbox annotation to specify if the VM should have a vIOMMU device. -pub const KATA_ANNO_CONF_HYPERVISOR_IOMMU: &str = - "io.katacontainers.config.hypervisor.enable_iommu"; +pub const KATA_ANNO_CFG_HYPERVISOR_IOMMU: &str = "io.katacontainers.config.hypervisor.enable_iommu"; /// Enable Hypervisor Devices IOMMU_PLATFORM -pub const KATA_ANNO_CONF_HYPERVISOR_IOMMU_PLATFORM: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_IOMMU_PLATFORM: &str = "io.katacontainers.config.hypervisor.enable_iommu_platform"; // Hypervisor Machine related annotations /// A sandbox annotation to specify the type of machine being emulated by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_MACHINE_TYPE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE: &str = "io.katacontainers.config.hypervisor.machine_type"; /// A sandbox annotation to specify machine specific accelerators for the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_MACHINE_ACCELERATORS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_MACHINE_ACCELERATORS: &str = "io.katacontainers.config.hypervisor.machine_accelerators"; /// EntropySource is a sandbox annotation to specify the path to a host source of /// entropy (/dev/random, /dev/urandom or real hardware RNG device) -pub const KATA_ANNO_CONF_HYPERVISOR_ENTROPY_SOURCE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENTROPY_SOURCE: &str = "io.katacontainers.config.hypervisor.entropy_source"; // Hypervisor Memory related annotations /// A sandbox annotation for the memory assigned for a VM by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY: &str = "io.katacontainers.config.hypervisor.default_memory"; /// A sandbox annotation to specify the memory slots assigned to the VM by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS: &str = "io.katacontainers.config.hypervisor.memory_slots"; /// A sandbox annotation that specifies the memory space used for nvdimm device by the hypervisor. -pub const KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str = "io.katacontainers.config.hypervisor.enable_mem_prealloc"; /// A sandbox annotation to specify if the memory should be pre-allocated from huge pages. -pub const KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES: &str = "io.katacontainers.config.hypervisor.enable_hugepages"; /// A sandbox annotation to soecify file based memory backend root directory. -pub const KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str = "io.katacontainers.config.hypervisor.file_mem_backend"; /// A sandbox annotation that is used to enable/disable virtio-mem. -pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM: &str = "io.katacontainers.config.hypervisor.enable_virtio_mem"; /// A sandbox annotation to enable swap of vm memory. -pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP: &str = "io.katacontainers.config.hypervisor.enable_swap"; /// A sandbox annotation to enable swap in the guest. -pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP: &str = "io.katacontainers.config.hypervisor.enable_guest_swap"; // Hypervisor Network related annotations /// A sandbox annotation to specify if vhost-net is not available on the host. -pub const KATA_ANNO_CONF_HYPERVISOR_DISABLE_VHOST_NET: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_DISABLE_VHOST_NET: &str = "io.katacontainers.config.hypervisor.disable_vhost_net"; /// A sandbox annotation that specifies max rate on network I/O inbound bandwidth. -pub const KATA_ANNO_CONF_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE: &str = "io.katacontainers.config.hypervisor.rx_rate_limiter_max_rate"; /// A sandbox annotation that specifies max rate on network I/O outbound bandwidth. -pub const KATA_ANNO_CONF_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE: &str = "io.katacontainers.config.hypervisor.tx_rate_limiter_max_rate"; // Hypervisor Security related annotations /// A sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks. -pub const KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH: &str = "io.katacontainers.config.hypervisor.guest_hook_path"; /// A sandbox annotation to enable rootless hypervisor (only supported in QEMU currently). -pub const KATA_ANNO_CONF_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR: &str = "io.katacontainers.config.hypervisor.rootless"; // Hypervisor Shared File System related annotations /// A sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs. -pub const KATA_ANNO_CONF_HYPERVISOR_SHARED_FS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_SHARED_FS: &str = "io.katacontainers.config.hypervisor.shared_fs"; /// A sandbox annotations to specify virtio-fs vhost-user daemon path. -pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON: &str = "io.katacontainers.config.hypervisor.virtio_fs_daemon"; /// A sandbox annotation to specify the cache mode for fs version cache or "none". -pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE: &str = "io.katacontainers.config.hypervisor.virtio_fs_cache"; /// A sandbox annotation to specify the DAX cache size in MiB. -pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE_SIZE: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE_SIZE: &str = "io.katacontainers.config.hypervisor.virtio_fs_cache_size"; /// A sandbox annotation to pass options to virtiofsd daemon. -pub const KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS: &str = "io.katacontainers.config.hypervisor.virtio_fs_extra_args"; /// A sandbox annotation to specify as the msize for 9p shares. -pub const KATA_ANNO_CONF_HYPERVISOR_MSIZE_9P: &str = "io.katacontainers.config.hypervisor.msize_9p"; +pub const KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P: &str = "io.katacontainers.config.hypervisor.msize_9p"; // Runtime related annotations /// Prefix for Runtime configurations. -pub const KATA_ANNO_CONF_RUNTIME_PREFIX: &str = "io.katacontainers.config.runtime."; +pub const KATA_ANNO_CFG_RUNTIME_PREFIX: &str = "io.katacontainers.config.runtime."; /// A sandbox annotation that determines if seccomp should be applied inside guest. -pub const KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP: &str = +pub const KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP: &str = "io.katacontainers.config.runtime.disable_guest_seccomp"; /// A sandbox annotation that determines if pprof enabled. -pub const KATA_ANNO_CONF_ENABLE_PPROF: &str = "io.katacontainers.config.runtime.enable_pprof"; +pub const KATA_ANNO_CFG_ENABLE_PPROF: &str = "io.katacontainers.config.runtime.enable_pprof"; /// A sandbox annotation that determines if experimental features enabled. -pub const KATA_ANNO_CONF_EXPERIMENTAL: &str = "io.katacontainers.config.runtime.experimental"; +pub const KATA_ANNO_CFG_EXPERIMENTAL: &str = "io.katacontainers.config.runtime.experimental"; /// A sandbox annotaion that determines how the VM should be connected to the the container network /// interface. -pub const KATA_ANNO_CONF_INTER_NETWORK_MODEL: &str = +pub const KATA_ANNO_CFG_INTER_NETWORK_MODEL: &str = "io.katacontainers.config.runtime.internetworking_model"; /// SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup. -pub const KATA_ANNO_CONF_SANDBOX_CGROUP_ONLY: &str = +pub const KATA_ANNO_CFG_SANDBOX_CGROUP_ONLY: &str = "io.katacontainers.config.runtime.sandbox_cgroup_only"; /// A sandbox annotation that determines if create a netns for hypervisor process. -pub const KATA_ANNO_CONF_DISABLE_NEW_NETNS: &str = +pub const KATA_ANNO_CFG_DISABLE_NEW_NETNS: &str = "io.katacontainers.config.runtime.disable_new_netns"; /// A sandbox annotation to specify how attached VFIO devices should be treated. -pub const KATA_ANNO_CONF_VFIO_MODE: &str = "io.katacontainers.config.runtime.vfio_mode"; +pub const KATA_ANNO_CFG_VFIO_MODE: &str = "io.katacontainers.config.runtime.vfio_mode"; /// A helper structure to query configuration information by check annotations. #[derive(Debug, Default, Deserialize)] @@ -407,7 +404,7 @@ impl Annotation { impl Annotation { /// Get the annotation of sandbox configuration file path. pub fn get_sandbox_config_path(&self) -> Option { - self.get(SANDBOX_CONFIG_PATH_KEY) + self.get(SANDBOX_CFG_PATH_KEY) } /// Get the annotation of bundle path. @@ -422,7 +419,7 @@ impl Annotation { /// Get the annotation to specify the Resources.Memory.Swappiness. pub fn get_container_resource_swappiness(&self) -> Result> { - match self.get_u32(KATA_ANNO_CONTAINER_RESOURCE_SWAPPINESS) { + match self.get_u32(KATA_ANNO_CONTAINER_RES_SWAPPINESS) { Ok(r) => { if r.unwrap_or_default() > 100 { return Err(io::Error::new( @@ -439,7 +436,7 @@ impl Annotation { /// Get the annotation to specify the Resources.Memory.Swap. pub fn get_container_resource_swap_in_bytes(&self) -> Option { - self.get(KATA_ANNO_CONTAINER_RESOURCE_SWAP_IN_BYTES) + self.get(KATA_ANNO_CONTAINER_RES_SWAP_IN_BYTES) } } @@ -471,20 +468,20 @@ impl Annotation { match key.as_str() { // update hypervisor config // Hypervisor related annotations - KATA_ANNO_CONF_HYPERVISOR_PATH => { + KATA_ANNO_CFG_HYPERVISOR_PATH => { hv.validate_hypervisor_path(value)?; hv.path = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_CTLPATH => { + KATA_ANNO_CFG_HYPERVISOR_CTLPATH => { hv.validate_hypervisor_ctlpath(value)?; hv.ctlpath = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH => { + KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH => { hv.validate_jailer_path(value)?; hv.jailer_path = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS => match self.get_bool(key) { Ok(r) => { hv.enable_iothreads = r.unwrap_or_default(); } @@ -493,20 +490,18 @@ impl Annotation { } }, // Hypervisor Block Device related annotations - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER => { + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER => { hv.blockdev_info.block_device_driver = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_DISABLE_BLOCK_DEVICE_USE => { - match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.disable_block_device_use = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); - } + KATA_ANNO_CFG_HYPERVISOR_DISABLE_BLOCK_DEV_USE => match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.disable_block_device_use = r.unwrap_or_default(); } - } - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_SET => match self.get_bool(key) { + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_SET => match self.get_bool(key) { Ok(r) => { hv.blockdev_info.block_device_cache_set = r.unwrap_or_default(); } @@ -514,8 +509,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_DIRECT => match self.get_bool(key) - { + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_DIRECT => match self.get_bool(key) { Ok(r) => { hv.blockdev_info.block_device_cache_direct = r.unwrap_or_default(); } @@ -523,17 +517,15 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH => { - match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.block_device_cache_noflush = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); - } + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH => match self.get_bool(key) { + Ok(r) => { + hv.blockdev_info.block_device_cache_noflush = r.unwrap_or_default(); } - } - KATA_ANNO_CONF_HYPERVISOR_DISABLE_IMAGE_NVDIMM => match self.get_bool(key) { + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CFG_HYPERVISOR_DISABLE_IMAGE_NVDIMM => match self.get_bool(key) { Ok(r) => { hv.blockdev_info.disable_image_nvdimm = r.unwrap_or_default(); } @@ -541,7 +533,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_MEMORY_OFFSET => match self.get_u64(key) { + KATA_ANNO_CFG_HYPERVISOR_MEMORY_OFFSET => match self.get_u64(key) { Ok(r) => { hv.blockdev_info.memory_offset = r.unwrap_or_default(); } @@ -549,7 +541,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_ENABLE_VHOSTUSER_STORE => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_VHOSTUSER_STORE => match self.get_bool(key) { Ok(r) => { hv.blockdev_info.enable_vhost_user_store = r.unwrap_or_default(); } @@ -557,35 +549,35 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH => { + KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH => { hv.blockdev_info.validate_vhost_user_store_path(value)?; hv.blockdev_info.vhost_user_store_path = value.to_string(); } // Hypervisor Guest Boot related annotations - KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH => { + KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH => { hv.boot_info.validate_boot_path(value)?; hv.boot_info.kernel = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_KERNEL_PARAMS => { + KATA_ANNO_CFG_HYPERVISOR_KERNEL_PARAMS => { hv.boot_info.kernel_params = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_IMAGE_PATH => { + KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH => { hv.boot_info.validate_boot_path(value)?; hv.boot_info.image = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_INITRD_PATH => { + KATA_ANNO_CFG_HYPERVISOR_INITRD_PATH => { hv.boot_info.validate_boot_path(value)?; hv.boot_info.initrd = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_FIRMWARE_PATH => { + KATA_ANNO_CFG_HYPERVISOR_FIRMWARE_PATH => { hv.boot_info.validate_boot_path(value)?; hv.boot_info.firmware = value.to_string(); } // Hypervisor CPU related annotations - KATA_ANNO_CONF_HYPERVISOR_CPU_FEATURES => { + KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES => { hv.cpu_info.cpu_features = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS => match self.get_i32(key) { + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS => match self.get_i32(key) { Ok(num_cpus) => { let num_cpus = num_cpus.unwrap_or_default(); if num_cpus @@ -611,7 +603,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MAX_VCPUS => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MAX_VCPUS => match self.get_u32(key) { Ok(r) => { hv.cpu_info.default_maxvcpus = r.unwrap_or_default(); } @@ -620,17 +612,15 @@ impl Annotation { } }, // Hypervisor Device related annotations - KATA_ANNO_CONF_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => { - match self.get_bool(key) { - Ok(r) => { - hv.device_info.hotplug_vfio_on_root_bus = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); - } + KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => match self.get_bool(key) { + Ok(r) => { + hv.device_info.hotplug_vfio_on_root_bus = r.unwrap_or_default(); } - } - KATA_ANNO_CONF_HYPERVISOR_PCIE_ROOT_PORT => match self.get_u32(key) { + Err(e) => { + return Err(e); + } + }, + KATA_ANNO_CFG_HYPERVISOR_PCIE_ROOT_PORT => match self.get_u32(key) { Ok(r) => { hv.device_info.pcie_root_port = r.unwrap_or_default(); } @@ -638,7 +628,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_IOMMU => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_IOMMU => match self.get_bool(key) { Ok(r) => { hv.device_info.enable_iommu = r.unwrap_or_default(); } @@ -646,7 +636,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_IOMMU_PLATFORM => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_IOMMU_PLATFORM => match self.get_bool(key) { Ok(r) => { hv.device_info.enable_iommu_platform = r.unwrap_or_default(); } @@ -655,18 +645,18 @@ impl Annotation { } }, // Hypervisor Machine related annotations - KATA_ANNO_CONF_HYPERVISOR_MACHINE_TYPE => { + KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE => { hv.machine_info.machine_type = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_MACHINE_ACCELERATORS => { + KATA_ANNO_CFG_HYPERVISOR_MACHINE_ACCELERATORS => { hv.machine_info.machine_accelerators = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_ENTROPY_SOURCE => { + KATA_ANNO_CFG_HYPERVISOR_ENTROPY_SOURCE => { hv.machine_info.validate_entropy_source(value)?; hv.machine_info.entropy_source = value.to_string(); } // Hypervisor Memory related annotations - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => match self.get_u32(key) { Ok(r) => { let mem = r.unwrap_or_default(); if mem @@ -692,7 +682,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS => match self.get_u32(key) { Ok(v) => { hv.memory_info.memory_slots = v.unwrap_or_default(); } @@ -701,7 +691,7 @@ impl Annotation { } }, - KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC => match self.get_bool(key) { Ok(r) => { hv.memory_info.enable_mem_prealloc = r.unwrap_or_default(); } @@ -709,7 +699,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_bool(key) { Ok(r) => { hv.memory_info.enable_hugepages = r.unwrap_or_default(); } @@ -717,11 +707,11 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { + KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { hv.memory_info.validate_memory_backend_path(value)?; hv.memory_info.file_mem_backend = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM => match self.get_bool(key) { Ok(r) => { hv.memory_info.enable_virtio_mem = r.unwrap_or_default(); } @@ -729,7 +719,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP => match self.get_bool(key) { Ok(r) => { hv.memory_info.enable_swap = r.unwrap_or_default(); } @@ -737,7 +727,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP => match self.get_bool(key) { Ok(r) => { hv.memory_info.enable_guest_swap = r.unwrap_or_default(); } @@ -746,7 +736,7 @@ impl Annotation { } }, // Hypervisor Network related annotations - KATA_ANNO_CONF_HYPERVISOR_DISABLE_VHOST_NET => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_DISABLE_VHOST_NET => match self.get_bool(key) { Ok(r) => { hv.network_info.disable_vhost_net = r.unwrap_or_default(); } @@ -754,7 +744,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { + KATA_ANNO_CFG_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { Ok(r) => { hv.network_info.rx_rate_limiter_max_rate = r.unwrap_or_default(); } @@ -762,7 +752,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { + KATA_ANNO_CFG_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { Ok(r) => { hv.network_info.tx_rate_limiter_max_rate = r.unwrap_or_default(); } @@ -771,11 +761,11 @@ impl Annotation { } }, // Hypervisor Security related annotations - KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH => { + KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH => { hv.security_info.validate_path(value)?; hv.security_info.guest_hook_path = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR => { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR => { match self.get_bool(key) { Ok(r) => { hv.security_info.rootless = r.unwrap_or_default(); @@ -786,19 +776,19 @@ impl Annotation { } } // Hypervisor Shared File System related annotations - KATA_ANNO_CONF_HYPERVISOR_SHARED_FS => { + KATA_ANNO_CFG_HYPERVISOR_SHARED_FS => { hv.shared_fs.shared_fs = self.get(key); } - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON => { + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON => { hv.shared_fs.validate_virtiofs_daemon_path(value)?; hv.shared_fs.virtio_fs_daemon = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE => { + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE => { hv.shared_fs.virtio_fs_cache = value.to_string(); } - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => match self.get_u32(key) { Ok(r) => { hv.shared_fs.virtio_fs_cache_size = r.unwrap_or_default(); } @@ -806,14 +796,14 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS => { + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS => { let args: Vec = value.to_string().split(',').map(str::to_string).collect(); for arg in args { hv.shared_fs.virtio_fs_extra_args.push(arg.to_string()); } } - KATA_ANNO_CONF_HYPERVISOR_MSIZE_9P => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P => match self.get_u32(key) { Ok(v) => { hv.shared_fs.msize_9p = v.unwrap_or_default(); } @@ -832,14 +822,14 @@ impl Annotation { } else { match key.as_str() { //update agent config - KATA_ANNO_CONF_KERNEL_MODULES => { + KATA_ANNO_CFG_KERNEL_MODULES => { let kernel_mod: Vec = value.to_string().split(';').map(str::to_string).collect(); for modules in kernel_mod { ag.kernel_modules.push(modules.to_string()); } } - KATA_ANNO_CONF_AGENT_TRACE => match self.get_bool(key) { + KATA_ANNO_CFG_AGENT_TRACE => match self.get_bool(key) { Ok(r) => { ag.enable_tracing = r.unwrap_or_default(); } @@ -847,7 +837,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE => match self.get_u32(key) { + KATA_ANNO_CFG_AGENT_CONTAINER_PIPE_SIZE => match self.get_u32(key) { Ok(v) => { ag.container_pipe_size = v.unwrap_or_default(); } @@ -856,7 +846,7 @@ impl Annotation { } }, //update runtume config - KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP => match self.get_bool(key) { + KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP => match self.get_bool(key) { Ok(r) => { config.runtime.disable_guest_seccomp = r.unwrap_or_default(); } @@ -864,7 +854,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_ENABLE_PPROF => match self.get_bool(key) { + KATA_ANNO_CFG_ENABLE_PPROF => match self.get_bool(key) { Ok(r) => { config.runtime.enable_pprof = r.unwrap_or_default(); } @@ -872,17 +862,17 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_EXPERIMENTAL => { + KATA_ANNO_CFG_EXPERIMENTAL => { let args: Vec = value.to_string().split(',').map(str::to_string).collect(); for arg in args { config.runtime.experimental.push(arg.to_string()); } } - KATA_ANNO_CONF_INTER_NETWORK_MODEL => { + KATA_ANNO_CFG_INTER_NETWORK_MODEL => { config.runtime.internetworking_model = value.to_string(); } - KATA_ANNO_CONF_SANDBOX_CGROUP_ONLY => match self.get_bool(key) { + KATA_ANNO_CFG_SANDBOX_CGROUP_ONLY => match self.get_bool(key) { Ok(r) => { config.runtime.sandbox_cgroup_only = r.unwrap_or_default(); } @@ -890,7 +880,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_DISABLE_NEW_NETNS => match self.get_bool(key) { + KATA_ANNO_CFG_DISABLE_NEW_NETNS => match self.get_bool(key) { Ok(r) => { config.runtime.disable_new_netns = r.unwrap_or_default(); } @@ -898,7 +888,7 @@ impl Annotation { return Err(e); } }, - KATA_ANNO_CONF_VFIO_MODE => { + KATA_ANNO_CFG_VFIO_MODE => { config.runtime.vfio_mode = value.to_string(); } _ => { diff --git a/src/libs/kata-types/src/annotations/thirdparty.rs b/src/libs/kata-types/src/annotations/thirdparty.rs index 4ea1548b9fd9..28a522d7c69b 100644 --- a/src/libs/kata-types/src/annotations/thirdparty.rs +++ b/src/libs/kata-types/src/annotations/thirdparty.rs @@ -9,6 +9,4 @@ /// Annotation to enable SGX. /// /// Hardware-based isolation and memory encryption. -// Supported suffixes are: Ki | Mi | Gi | Ti | Pi | Ei . For example: 4Mi -// For more information about supported suffixes see https://physics.nist.gov/cuu/Units/binary.html pub const SGXEPC: &str = "sgx.intel.com/epc"; diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index b5b4029fbf65..fd40e748be68 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -23,7 +23,7 @@ pub const DEFAULT_BLOCK_DEVICE_TYPE: &str = "virtio-blk"; pub const DEFAULT_VHOST_USER_STORE_PATH: &str = "/var/run/vhost-user"; pub const DEFAULT_BLOCK_NVDIMM_MEM_OFFSET: u64 = 0; -pub const DEFAULT_SHARED_FS_TYPE: &str = "virtio-9p"; +pub const DEFAULT_SHARED_FS_TYPE: &str = "virtio-fs"; pub const DEFAULT_VIRTIO_FS_CACHE_MODE: &str = "none"; pub const DEFAULT_VIRTIO_FS_DAX_SIZE_MB: u32 = 1024; pub const DEFAULT_SHARED_9PFS_SIZE: u32 = 128 * 1024; diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 4aa4ceba737c..e167a34bb6d7 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -31,7 +31,7 @@ use lazy_static::lazy_static; use regex::RegexSet; use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; -use crate::annotations::KATA_ANNO_CONF_HYPERVISOR_PREFIX; +use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX; use crate::{eother, resolve_path, validate_path}; mod dragonball; @@ -48,6 +48,7 @@ const VIRTIO_PMEM: &str = "nvdimm"; const VIRTIO_9P: &str = "virtio-9p"; const VIRTIO_FS: &str = "virtio-fs"; const VIRTIO_FS_INLINE: &str = "inline-virtio-fs"; +const MAX_BRIDGE_SIZE: u32 = 5; lazy_static! { static ref HYPERVISOR_PLUGINS: Mutex>> = @@ -169,8 +170,8 @@ impl BlockDeviceInfo { VIRTIO_BLK, VIRTIO_BLK_CCW, VIRTIO_BLK_MMIO, - VIRTIO_SCSI, VIRTIO_PMEM, + VIRTIO_SCSI, ]; if !l.contains(&self.block_device_driver.as_str()) { return Err(eother!( @@ -406,8 +407,8 @@ pub struct DeviceInfo { impl DeviceInfo { /// Adjust the configuration information after loading from configuration file. pub fn adjust_configuration(&mut self) -> Result<()> { - if self.default_bridges > 5 { - self.default_bridges = 5; + if self.default_bridges > MAX_BRIDGE_SIZE { + self.default_bridges = MAX_BRIDGE_SIZE; } Ok(()) @@ -415,7 +416,7 @@ impl DeviceInfo { /// Validate the configuration information. pub fn validate(&self) -> Result<()> { - if self.default_bridges > 5 { + if self.default_bridges > MAX_BRIDGE_SIZE { return Err(eother!( "The configured PCI bridges {} are too many", self.default_bridges @@ -701,10 +702,10 @@ impl SecurityInfo { /// Check whether annotation key is enabled or not. pub fn is_annotation_enabled(&self, path: &str) -> bool { - if !path.starts_with(KATA_ANNO_CONF_HYPERVISOR_PREFIX) { + if !path.starts_with(KATA_ANNO_CFG_HYPERVISOR_PREFIX) { return false; } - let pos = KATA_ANNO_CONF_HYPERVISOR_PREFIX.len(); + let pos = KATA_ANNO_CFG_HYPERVISOR_PREFIX.len(); let key = &path[pos..]; if let Ok(set) = RegexSet::new(&self.enable_annotations) { return set.is_match(key); @@ -798,8 +799,8 @@ impl SharedFsInfo { || self.msize_9p > default::MAX_SHARED_9PFS_SIZE { return Err(eother!( - "Invalid 9p configuration msize 0x{:x}", - self.msize_9p + "Invalid 9p configuration msize 0x{:x}, min value is 0x{:x}, max value is 0x{:x}", + self.msize_9p,default::MIN_SHARED_9PFS_SIZE, default::MAX_SHARED_9PFS_SIZE )); } Ok(()) @@ -842,10 +843,9 @@ impl SharedFsInfo { "Virtio-fs daemon path {} is invalid: {}" )?; - if self.virtio_fs_cache != "none" - && self.virtio_fs_cache != "auto" - && self.virtio_fs_cache != "always" - { + let l = ["none", "auto", "always"]; + + if !l.contains(&self.virtio_fs_cache.as_str()) { return Err(eother!( "Invalid virtio-fs cache mode: {}", &self.virtio_fs_cache diff --git a/src/libs/kata-types/src/config/hypervisor/qemu.rs b/src/libs/kata-types/src/config/hypervisor/qemu.rs index eb14779bd2e8..52fc4f47ddb8 100644 --- a/src/libs/kata-types/src/config/hypervisor/qemu.rs +++ b/src/libs/kata-types/src/config/hypervisor/qemu.rs @@ -73,9 +73,6 @@ impl ConfigPlugin for QemuConfig { if qemu.device_info.default_bridges == 0 { qemu.device_info.default_bridges = default::DEFAULT_QEMU_PCI_BRIDGES; - if qemu.device_info.default_bridges > default::MAX_QEMU_PCI_BRIDGES { - qemu.device_info.default_bridges = default::MAX_QEMU_PCI_BRIDGES; - } } if qemu.machine_info.machine_type.is_empty() { diff --git a/src/libs/kata-types/src/config/runtime_vendor.rs b/src/libs/kata-types/src/config/runtime_vendor.rs index 5981c7457029..a0529f5ea3e6 100644 --- a/src/libs/kata-types/src/config/runtime_vendor.rs +++ b/src/libs/kata-types/src/config/runtime_vendor.rs @@ -32,10 +32,6 @@ impl ConfigOps for RuntimeVendor { /// Validate the configuration information. fn validate(conf: &TomlConfig) -> Result<()> { if conf.runtime.vendor.log_level > 10 { - warn!( - sl!(), - "log level {} in configuration file is invalid", conf.runtime.vendor.log_level - ); return Err(eother!( "log level {} in configuration file is invalid", conf.runtime.vendor.log_level diff --git a/src/libs/kata-types/tests/test-config.rs b/src/libs/kata-types/tests/test_config.rs similarity index 80% rename from src/libs/kata-types/tests/test-config.rs rename to src/libs/kata-types/tests/test_config.rs index cbd134c3cd4c..6d4cc65ea990 100644 --- a/src/libs/kata-types/tests/test-config.rs +++ b/src/libs/kata-types/tests/test_config.rs @@ -5,19 +5,19 @@ #[cfg(test)] mod tests { use kata_types::annotations::{ - Annotation, KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE, KATA_ANNO_CONF_AGENT_TRACE, - KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP, KATA_ANNO_CONF_ENABLE_PPROF, - KATA_ANNO_CONF_EXPERIMENTAL, KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH, - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER, KATA_ANNO_CONF_HYPERVISOR_CTLPATH, - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS, - KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS, - KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, - KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES, - KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH, KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH, - KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS, - KATA_ANNO_CONF_HYPERVISOR_PATH, KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH, - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CONF_KERNEL_MODULES, + Annotation, KATA_ANNO_CFG_AGENT_CONTAINER_PIPE_SIZE, KATA_ANNO_CFG_AGENT_TRACE, + KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP, KATA_ANNO_CFG_ENABLE_PPROF, + KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH, + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH, + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS, + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, + KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, + KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES, + KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, + KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, + KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES, }; use kata_types::config::KataConfig; use kata_types::config::{QemuConfig, TomlConfig}; @@ -67,104 +67,104 @@ mod tests { .expect("failed to execute process"); let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_KERNEL_MODULES.to_string(), + KATA_ANNO_CFG_KERNEL_MODULES.to_string(), "j465 aaa=1;r33w".to_string(), ); - anno_hash.insert(KATA_ANNO_CONF_AGENT_TRACE.to_string(), "false".to_string()); + anno_hash.insert(KATA_ANNO_CFG_AGENT_TRACE.to_string(), "false".to_string()); anno_hash.insert( - KATA_ANNO_CONF_AGENT_CONTAINER_PIPE_SIZE.to_string(), + KATA_ANNO_CFG_AGENT_CONTAINER_PIPE_SIZE.to_string(), "3".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_PATH.to_string(), "./hypervisor_path".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER.to_string(), + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER.to_string(), "device".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_CACHE_NOFLUSH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_VHOSTUSER_STORE_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH.to_string(), "./store_path".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_DISABLE_GUEST_SECCOMP.to_string(), + KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP.to_string(), "true".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_GUEST_HOOK_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH.to_string(), "./test_hypervisor_hook_path".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_MEMORY_PREALLOC.to_string(), + KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_CTLPATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_CTLPATH.to_string(), "./jvm".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS.to_string(), "12".to_string(), ); - anno_hash.insert(KATA_ANNO_CONF_ENABLE_PPROF.to_string(), "false".to_string()); + anno_hash.insert(KATA_ANNO_CFG_ENABLE_PPROF.to_string(), "false".to_string()); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY.to_string(), + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY.to_string(), "100".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_ENABLE_IO_THREADS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_ENABLE_SWAP.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR.to_string(), + KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR.to_string(), "./test_file_backend_mem_root".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_HUGE_PAGES.to_string(), + KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_JAILER_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH.to_string(), "./test_jailer_path".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH.to_string(), "./test_kernel_path".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS.to_string(), "100".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS.to_string(), "rr,dg,er".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_MEM.to_string(), + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM.to_string(), "false".to_string(), ); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_VIRTIO_FS_DAEMON.to_string(), + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON.to_string(), "./virtio_fs".to_string(), ); - anno_hash.insert(KATA_ANNO_CONF_EXPERIMENTAL.to_string(), "c,d,e".to_string()); + anno_hash.insert(KATA_ANNO_CFG_EXPERIMENTAL.to_string(), "c,d,e".to_string()); let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(&content).unwrap(); @@ -286,7 +286,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_BLOCK_DEVICE_DRIVER.to_string(), + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER.to_string(), "fvfvfvfvf".to_string(), ); let anno = Annotation::new(anno_hash); @@ -309,7 +309,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), "false".to_string(), ); let anno = Annotation::new(anno_hash); @@ -332,7 +332,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_PATH.to_string(), "/usr/bin/nle".to_string(), ); let anno = Annotation::new(anno_hash); @@ -360,7 +360,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_KERNEL_PATH.to_string(), + KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH.to_string(), "/usr/bin/cdcd".to_string(), ); let anno = Annotation::new(anno_hash); @@ -382,7 +382,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_MEMORY_SLOTS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS.to_string(), "-1".to_string(), ); let anno = Annotation::new(anno_hash); @@ -405,7 +405,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_MEMORY.to_string(), + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY.to_string(), "10".to_string(), ); let anno = Annotation::new(anno_hash); @@ -428,7 +428,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS.to_string(), "400".to_string(), ); let anno = Annotation::new(anno_hash); @@ -451,7 +451,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP.to_string(), "false1".to_string(), ); let anno = Annotation::new(anno_hash); @@ -474,7 +474,7 @@ mod tests { let mut anno_hash = HashMap::new(); anno_hash.insert( - KATA_ANNO_CONF_HYPERVISOR_DEFAULT_VCPUS.to_string(), + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS.to_string(), "ddc".to_string(), ); let anno = Annotation::new(anno_hash); From 05ad026fc0e6c51bdaecddf17cd87be635efee12 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Fri, 11 Feb 2022 17:10:23 +0800 Subject: [PATCH 0023/1953] libs/types: fix implementation details use ok_or_else to handle get_mut(hypervisor) to substitue unwrap Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/config/hypervisor/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index e167a34bb6d7..2d0ded6b47f5 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -23,7 +23,7 @@ //! hypervisors, so let's contain it... use std::collections::HashMap; -use std::io::Result; +use std::io::{self, Result}; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -974,7 +974,9 @@ impl ConfigOps for Hypervisor { if let Some(plugin) = get_hypervisor_plugin(hypervisor) { plugin.adjust_configuration(conf)?; // Safe to unwrap() because `hypervisor` is a valid key in the hash map. - let hv = conf.hypervisor.get_mut(hypervisor).unwrap(); + let hv = conf.hypervisor.get_mut(hypervisor).ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "hypervisor not found".to_string()) + })?; hv.blockdev_info.adjust_configuration()?; hv.boot_info.adjust_configuration()?; hv.cpu_info.adjust_configuration()?; From b9b6d70aae619533e41cc4c14e3a954e885d7970 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 15 Feb 2022 14:54:33 +0800 Subject: [PATCH 0024/1953] libs/types: modify implementation details 1. fix nit problems 2. use generic type when parsing different type Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/annotations/mod.rs | 398 +++++++++--------- .../kata-types/src/annotations/thirdparty.rs | 2 +- src/libs/kata-types/src/config/default.rs | 16 +- .../src/config/hypervisor/dragonball.rs | 28 +- .../kata-types/src/config/runtime_vendor.rs | 1 - src/libs/kata-types/tests/test_config.rs | 34 +- 6 files changed, 232 insertions(+), 247 deletions(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index ea4e5c03a0ef..d06a67d4305b 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::fs::File; use std::io::{self, BufReader, Result}; +use std::result::{self}; use std::u32; use serde::Deserialize; @@ -330,73 +331,23 @@ impl Annotation { &mut self.annotations } - /// Get the value of annotation with `key` as string. - pub fn get(&self, key: &str) -> Option { - self.annotations.get(key).map(|v| String::from(v.trim())) - } - - /// Get the value of annotation with `key` as bool. - pub fn get_bool(&self, key: &str) -> Result> { - if let Some(value) = self.get(key) { - return value - .parse::() - .map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid input {} for bool", key), - ) - }) - .map(Some); - } - Ok(None) - } - - /// Get the value of annotation with `key` as u32. - pub fn get_u32(&self, key: &str) -> Result> { - if let Some(value) = self.get(key) { - return value - .parse::() - .map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid input {} for u32", key), - ) - }) - .map(Some); - } - Ok(None) - } - - /// Get the value of annotation with `key` as i32. - pub fn get_i32(&self, key: &str) -> Result> { + /// Get the value of annotation with `key` + pub fn get_value( + &self, + key: &str, + ) -> result::Result, ::Err> + where + T: std::str::FromStr, + { if let Some(value) = self.get(key) { - return value - .parse::() - .map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid input {} for i32", key), - ) - }) - .map(Some); + return value.parse::().map(Some); } Ok(None) } - /// Get the value of annotation with `key` as u64. - pub fn get_u64(&self, key: &str) -> Result> { - if let Some(value) = self.get(key) { - return value - .parse::() - .map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Invalid input {} for u64", key), - ) - }) - .map(Some); - } - Ok(None) + /// Get the value of annotation with `key` as string. + pub fn get(&self, key: &str) -> Option { + self.annotations.get(key).map(|v| String::from(v.trim())) } } @@ -419,7 +370,7 @@ impl Annotation { /// Get the annotation to specify the Resources.Memory.Swappiness. pub fn get_container_resource_swappiness(&self) -> Result> { - match self.get_u32(KATA_ANNO_CONTAINER_RES_SWAPPINESS) { + match self.get_value::(KATA_ANNO_CONTAINER_RES_SWAPPINESS) { Ok(r) => { if r.unwrap_or_default() > 100 { return Err(io::Error::new( @@ -430,7 +381,10 @@ impl Annotation { Ok(r) } } - Err(e) => Err(e), + Err(_e) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "parse u32 error".to_string(), + )), } } @@ -461,6 +415,11 @@ impl Annotation { format!("agent {} not found", agent_name), )); } + let bool_err = io::Error::new(io::ErrorKind::InvalidData, "parse bool error".to_string()); + let u32_err = io::Error::new(io::ErrorKind::InvalidData, "parse u32 error".to_string()); + let u64_err = io::Error::new(io::ErrorKind::InvalidData, "parse u64 error".to_string()); + let i32_err = io::Error::new(io::ErrorKind::InvalidData, "parse i32 error".to_string()); + let mut hv = config.hypervisor.get_mut(hypervisor_name).unwrap(); let mut ag = config.agent.get_mut(agent_name).unwrap(); for (key, value) in &self.annotations { @@ -481,74 +440,87 @@ impl Annotation { hv.validate_jailer_path(value)?; hv.jailer_path = value.to_string(); } - KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS => match self.get_value::(key) + { Ok(r) => { hv.enable_iothreads = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, // Hypervisor Block Device related annotations KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER => { hv.blockdev_info.block_device_driver = value.to_string(); } - KATA_ANNO_CFG_HYPERVISOR_DISABLE_BLOCK_DEV_USE => match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.disable_block_device_use = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); - } - }, - KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_SET => match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.block_device_cache_set = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); - } - }, - KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_DIRECT => match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.block_device_cache_direct = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); + KATA_ANNO_CFG_HYPERVISOR_DISABLE_BLOCK_DEV_USE => { + match self.get_value::(key) { + Ok(r) => { + hv.blockdev_info.disable_block_device_use = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - }, - KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH => match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.block_device_cache_noflush = r.unwrap_or_default(); + } + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_SET => { + match self.get_value::(key) { + Ok(r) => { + hv.blockdev_info.block_device_cache_set = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - Err(e) => { - return Err(e); + } + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_DIRECT => { + match self.get_value::(key) { + Ok(r) => { + hv.blockdev_info.block_device_cache_direct = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - }, - KATA_ANNO_CFG_HYPERVISOR_DISABLE_IMAGE_NVDIMM => match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.disable_image_nvdimm = r.unwrap_or_default(); + } + KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH => { + match self.get_value::(key) { + Ok(r) => { + hv.blockdev_info.block_device_cache_noflush = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - Err(e) => { - return Err(e); + } + KATA_ANNO_CFG_HYPERVISOR_DISABLE_IMAGE_NVDIMM => { + match self.get_value::(key) { + Ok(r) => { + hv.blockdev_info.disable_image_nvdimm = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - }, - KATA_ANNO_CFG_HYPERVISOR_MEMORY_OFFSET => match self.get_u64(key) { + } + KATA_ANNO_CFG_HYPERVISOR_MEMORY_OFFSET => match self.get_value::(key) { Ok(r) => { hv.blockdev_info.memory_offset = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(u64_err); } }, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_VHOSTUSER_STORE => match self.get_bool(key) { - Ok(r) => { - hv.blockdev_info.enable_vhost_user_store = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); + KATA_ANNO_CFG_HYPERVISOR_ENABLE_VHOSTUSER_STORE => { + match self.get_value::(key) { + Ok(r) => { + hv.blockdev_info.enable_vhost_user_store = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - }, + } KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH => { hv.blockdev_info.validate_vhost_user_store_path(value)?; hv.blockdev_info.vhost_user_store_path = value.to_string(); @@ -577,7 +549,7 @@ impl Annotation { KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES => { hv.cpu_info.cpu_features = value.to_string(); } - KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS => match self.get_i32(key) { + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS => match self.get_value::(key) { Ok(num_cpus) => { let num_cpus = num_cpus.unwrap_or_default(); if num_cpus @@ -599,49 +571,53 @@ impl Annotation { hv.cpu_info.default_vcpus = num_cpus; } } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(i32_err); } }, - KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MAX_VCPUS => match self.get_u32(key) { - Ok(r) => { - hv.cpu_info.default_maxvcpus = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MAX_VCPUS => { + match self.get_value::(key) { + Ok(r) => { + hv.cpu_info.default_maxvcpus = r.unwrap_or_default(); + } + Err(_e) => { + return Err(u32_err); + } } - }, + } // Hypervisor Device related annotations - KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => match self.get_bool(key) { - Ok(r) => { - hv.device_info.hotplug_vfio_on_root_bus = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); + KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => { + match self.get_value::(key) { + Ok(r) => { + hv.device_info.hotplug_vfio_on_root_bus = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - }, - KATA_ANNO_CFG_HYPERVISOR_PCIE_ROOT_PORT => match self.get_u32(key) { + } + KATA_ANNO_CFG_HYPERVISOR_PCIE_ROOT_PORT => match self.get_value::(key) { Ok(r) => { hv.device_info.pcie_root_port = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(u32_err); } }, - KATA_ANNO_CFG_HYPERVISOR_IOMMU => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_IOMMU => match self.get_value::(key) { Ok(r) => { hv.device_info.enable_iommu = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_IOMMU_PLATFORM => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_IOMMU_PLATFORM => match self.get_value::(key) { Ok(r) => { hv.device_info.enable_iommu_platform = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, // Hypervisor Machine related annotations @@ -656,7 +632,7 @@ impl Annotation { hv.machine_info.entropy_source = value.to_string(); } // Hypervisor Memory related annotations - KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => match self.get_value::(key) { Ok(r) => { let mem = r.unwrap_or_default(); if mem @@ -678,100 +654,106 @@ impl Annotation { hv.memory_info.default_memory = mem; } } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(u32_err); } }, - KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS => match self.get_value::(key) { Ok(v) => { hv.memory_info.memory_slots = v.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(u32_err); } }, - KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC => match self.get_value::(key) { Ok(r) => { hv.memory_info.enable_mem_prealloc = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_value::(key) { Ok(r) => { hv.memory_info.enable_hugepages = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { hv.memory_info.validate_memory_backend_path(value)?; hv.memory_info.file_mem_backend = value.to_string(); } - KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM => match self.get_value::(key) { Ok(r) => { hv.memory_info.enable_virtio_mem = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP => match self.get_value::(key) { Ok(r) => { hv.memory_info.enable_swap = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP => match self.get_value::(key) + { Ok(r) => { hv.memory_info.enable_guest_swap = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, // Hypervisor Network related annotations - KATA_ANNO_CFG_HYPERVISOR_DISABLE_VHOST_NET => match self.get_bool(key) { + KATA_ANNO_CFG_HYPERVISOR_DISABLE_VHOST_NET => match self.get_value::(key) + { Ok(r) => { hv.network_info.disable_vhost_net = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { - Ok(r) => { - hv.network_info.rx_rate_limiter_max_rate = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); - } - }, - KATA_ANNO_CFG_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE => match self.get_u64(key) { - Ok(r) => { - hv.network_info.tx_rate_limiter_max_rate = r.unwrap_or_default(); + KATA_ANNO_CFG_HYPERVISOR_RX_RATE_LIMITER_MAX_RATE => { + match self.get_value::(key) { + Ok(r) => { + hv.network_info.rx_rate_limiter_max_rate = r.unwrap_or_default(); + } + Err(_e) => { + return Err(u64_err); + } } - Err(e) => { - return Err(e); + } + KATA_ANNO_CFG_HYPERVISOR_TX_RATE_LIMITER_MAX_RATE => { + match self.get_value::(key) { + Ok(r) => { + hv.network_info.tx_rate_limiter_max_rate = r.unwrap_or_default(); + } + Err(_e) => { + return Err(u64_err); + } } - }, + } // Hypervisor Security related annotations KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH => { hv.security_info.validate_path(value)?; hv.security_info.guest_hook_path = value.to_string(); } KATA_ANNO_CFG_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR => { - match self.get_bool(key) { + match self.get_value::(key) { Ok(r) => { hv.security_info.rootless = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } } } @@ -788,14 +770,16 @@ impl Annotation { KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE => { hv.shared_fs.virtio_fs_cache = value.to_string(); } - KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => match self.get_u32(key) { - Ok(r) => { - hv.shared_fs.virtio_fs_cache_size = r.unwrap_or_default(); - } - Err(e) => { - return Err(e); + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_CACHE_SIZE => { + match self.get_value::(key) { + Ok(r) => { + hv.shared_fs.virtio_fs_cache_size = r.unwrap_or_default(); + } + Err(_e) => { + return Err(u32_err); + } } - }, + } KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS => { let args: Vec = value.to_string().split(',').map(str::to_string).collect(); @@ -803,12 +787,12 @@ impl Annotation { hv.shared_fs.virtio_fs_extra_args.push(arg.to_string()); } } - KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P => match self.get_u32(key) { + KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P => match self.get_value::(key) { Ok(v) => { hv.shared_fs.msize_9p = v.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(u32_err); } }, @@ -829,37 +813,37 @@ impl Annotation { ag.kernel_modules.push(modules.to_string()); } } - KATA_ANNO_CFG_AGENT_TRACE => match self.get_bool(key) { + KATA_ANNO_CFG_AGENT_TRACE => match self.get_value::(key) { Ok(r) => { ag.enable_tracing = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_AGENT_CONTAINER_PIPE_SIZE => match self.get_u32(key) { + KATA_ANNO_CFG_AGENT_CONTAINER_PIPE_SIZE => match self.get_value::(key) { Ok(v) => { ag.container_pipe_size = v.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(u32_err); } }, //update runtume config - KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP => match self.get_bool(key) { + KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP => match self.get_value::(key) { Ok(r) => { config.runtime.disable_guest_seccomp = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_ENABLE_PPROF => match self.get_bool(key) { + KATA_ANNO_CFG_ENABLE_PPROF => match self.get_value::(key) { Ok(r) => { config.runtime.enable_pprof = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, KATA_ANNO_CFG_EXPERIMENTAL => { @@ -872,20 +856,20 @@ impl Annotation { KATA_ANNO_CFG_INTER_NETWORK_MODEL => { config.runtime.internetworking_model = value.to_string(); } - KATA_ANNO_CFG_SANDBOX_CGROUP_ONLY => match self.get_bool(key) { + KATA_ANNO_CFG_SANDBOX_CGROUP_ONLY => match self.get_value::(key) { Ok(r) => { config.runtime.sandbox_cgroup_only = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, - KATA_ANNO_CFG_DISABLE_NEW_NETNS => match self.get_bool(key) { + KATA_ANNO_CFG_DISABLE_NEW_NETNS => match self.get_value::(key) { Ok(r) => { config.runtime.disable_new_netns = r.unwrap_or_default(); } - Err(e) => { - return Err(e); + Err(_e) => { + return Err(bool_err); } }, KATA_ANNO_CFG_VFIO_MODE => { diff --git a/src/libs/kata-types/src/annotations/thirdparty.rs b/src/libs/kata-types/src/annotations/thirdparty.rs index 28a522d7c69b..e8f2a7168360 100644 --- a/src/libs/kata-types/src/annotations/thirdparty.rs +++ b/src/libs/kata-types/src/annotations/thirdparty.rs @@ -9,4 +9,4 @@ /// Annotation to enable SGX. /// /// Hardware-based isolation and memory encryption. -pub const SGXEPC: &str = "sgx.intel.com/epc"; +pub const SGX_EPC: &str = "sgx.intel.com/epc"; diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index fd40e748be68..6b047ba8ed6e 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -15,7 +15,7 @@ lazy_static! { "/usr/share/defaults/kata-containers/configuration.toml", ]; } -pub const DEFAULT_AGENT_NAME: &str = "kata"; +pub const DEFAULT_AGENT_NAME: &str = "kata-agent"; pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter"; @@ -35,13 +35,13 @@ pub const DEFAULT_GUEST_HOOK_PATH: &str = "/opt"; pub const DEFAULT_GUEST_VCPUS: u32 = 1; // Default configuration for Dragonball -pub const DEFAULT_DB_GUEST_KERNEL_IMAGE: &str = "vmlinuz"; -pub const DEFAULT_DB_GUEST_KERNEL_PARAMS: &str = ""; -pub const DEFAULT_DB_ENTROPY_SOURCE: &str = "/dev/urandom"; -pub const DEFAULT_DB_MEMORY_SIZE: u32 = 128; -pub const DEFAULT_DB_MEMORY_SLOTS: u32 = 128; -pub const MAX_DB_VCPUS: u32 = 256; -pub const MIN_DB_MEMORY_SIZE: u32 = 64; +pub const DEFAULT_DRAGONBALL_GUEST_KERNEL_IMAGE: &str = "vmlinuz"; +pub const DEFAULT_DRAGONBALL_GUEST_KERNEL_PARAMS: &str = ""; +pub const DEFAULT_DRAGONBALL_ENTROPY_SOURCE: &str = "/dev/urandom"; +pub const DEFAULT_DRAGONBALL_MEMORY_SIZE: u32 = 128; +pub const DEFAULT_DRAGONBALL_MEMORY_SLOTS: u32 = 128; +pub const MAX_DRAGONBALL_VCPUS: u32 = 256; +pub const MIN_DRAGONBALL_MEMORY_SIZE: u32 = 64; // Default configuration for qemu pub const DEFAULT_QEMU_BINARY_PATH: &str = "qemu"; pub const DEFAULT_QEMU_CONTROL_PATH: &str = ""; diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs index 95158019fb2f..bcb43eaba33a 100644 --- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use std::u32; use super::{default, register_hypervisor_plugin}; -use crate::config::default::MAX_DB_VCPUS; -use crate::config::default::MIN_DB_MEMORY_SIZE; +use crate::config::default::MAX_DRAGONBALL_VCPUS; +use crate::config::default::MIN_DRAGONBALL_MEMORY_SIZE; use crate::config::hypervisor::{ VIRTIO_BLK, VIRTIO_BLK_MMIO, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM, }; @@ -39,10 +39,10 @@ impl DragonballConfig { impl ConfigPlugin for DragonballConfig { fn get_max_cpus(&self) -> u32 { - MAX_DB_VCPUS + MAX_DRAGONBALL_VCPUS } fn get_min_memory(&self) -> u32 { - MIN_DB_MEMORY_SIZE + MIN_DRAGONBALL_MEMORY_SIZE } fn name(&self) -> &str { HYPERVISOR_NAME_DRAGONBALL @@ -54,25 +54,27 @@ impl ConfigPlugin for DragonballConfig { resolve_path!(db.jailer_path, "Dragonball jailer path {} is invalid: {}")?; if db.boot_info.kernel.is_empty() { - db.boot_info.kernel = default::DEFAULT_DB_GUEST_KERNEL_IMAGE.to_string(); + db.boot_info.kernel = default::DEFAULT_DRAGONBALL_GUEST_KERNEL_IMAGE.to_string(); } if db.boot_info.kernel_params.is_empty() { - db.boot_info.kernel_params = default::DEFAULT_DB_GUEST_KERNEL_PARAMS.to_string(); + db.boot_info.kernel_params = + default::DEFAULT_DRAGONBALL_GUEST_KERNEL_PARAMS.to_string(); } - if db.cpu_info.default_maxvcpus > default::MAX_DB_VCPUS { - db.cpu_info.default_maxvcpus = default::MAX_DB_VCPUS; + if db.cpu_info.default_maxvcpus > default::MAX_DRAGONBALL_VCPUS { + db.cpu_info.default_maxvcpus = default::MAX_DRAGONBALL_VCPUS; } if db.machine_info.entropy_source.is_empty() { - db.machine_info.entropy_source = default::DEFAULT_DB_ENTROPY_SOURCE.to_string(); + db.machine_info.entropy_source = + default::DEFAULT_DRAGONBALL_ENTROPY_SOURCE.to_string(); } if db.memory_info.default_memory == 0 { - db.memory_info.default_memory = default::DEFAULT_DB_MEMORY_SIZE; + db.memory_info.default_memory = default::DEFAULT_DRAGONBALL_MEMORY_SIZE; } if db.memory_info.memory_slots == 0 { - db.memory_info.memory_slots = default::DEFAULT_DB_MEMORY_SLOTS; + db.memory_info.memory_slots = default::DEFAULT_DRAGONBALL_MEMORY_SLOTS; } } Ok(()) @@ -131,8 +133,8 @@ impl ConfigPlugin for DragonballConfig { } if (db.cpu_info.default_vcpus > 0 - && db.cpu_info.default_vcpus as u32 > default::MAX_DB_VCPUS) - || db.cpu_info.default_maxvcpus > default::MAX_DB_VCPUS + && db.cpu_info.default_vcpus as u32 > default::MAX_DRAGONBALL_VCPUS) + || db.cpu_info.default_maxvcpus > default::MAX_DRAGONBALL_VCPUS { return Err(eother!( "Dragonball hypervisor can not support {} vCPUs", diff --git a/src/libs/kata-types/src/config/runtime_vendor.rs b/src/libs/kata-types/src/config/runtime_vendor.rs index a0529f5ea3e6..67fafe21d72f 100644 --- a/src/libs/kata-types/src/config/runtime_vendor.rs +++ b/src/libs/kata-types/src/config/runtime_vendor.rs @@ -6,7 +6,6 @@ //! A sample for vendor to customize the runtime implementation. use super::*; -use crate::{eother, sl}; use slog::Level; /// Vendor customization runtime configuration. #[derive(Debug, Default, Deserialize, Serialize)] diff --git a/src/libs/kata-types/tests/test_config.rs b/src/libs/kata-types/tests/test_config.rs index 6d4cc65ea990..ad668920498e 100644 --- a/src/libs/kata-types/tests/test_config.rs +++ b/src/libs/kata-types/tests/test_config.rs @@ -30,7 +30,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); std::process::Command::new("mkdir") @@ -167,7 +167,7 @@ mod tests { anno_hash.insert(KATA_ANNO_CFG_EXPERIMENTAL.to_string(), "c,d,e".to_string()); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -281,7 +281,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -290,7 +290,7 @@ mod tests { "fvfvfvfvf".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -304,7 +304,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -313,7 +313,7 @@ mod tests { "false".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -327,7 +327,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -374,7 +374,7 @@ mod tests { #[test] fn test_fail_to_change_memory_slots_because_of_less_than_zero() { let content = include_str!("texture/configuration-anno-0.toml"); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let qemu = QemuConfig::new(); @@ -386,7 +386,7 @@ mod tests { "-1".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -400,7 +400,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -409,7 +409,7 @@ mod tests { "10".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -423,7 +423,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -432,7 +432,7 @@ mod tests { "400".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -446,7 +446,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -455,7 +455,7 @@ mod tests { "false1".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") @@ -469,7 +469,7 @@ mod tests { let qemu = QemuConfig::new(); qemu.register(); - let config = TomlConfig::load(&content).unwrap(); + let config = TomlConfig::load(content).unwrap(); KataConfig::set_active_config(Some(config), "qemu", "agent0"); let mut anno_hash = HashMap::new(); @@ -478,7 +478,7 @@ mod tests { "ddc".to_string(), ); let anno = Annotation::new(anno_hash); - let mut config = TomlConfig::load(&content).unwrap(); + let mut config = TomlConfig::load(content).unwrap(); assert!(anno .update_config_by_annotation(&mut config, "qemu", "agent0") From 48c201a1acc1e64e097b97d2ea0153e84e8256d6 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 15 Feb 2022 16:46:27 +0800 Subject: [PATCH 0025/1953] libs/types: make the variable name easier to understand 1. modify default values for hypervisor 2. change the variable name 3. check the min memory limit Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/config/agent.rs | 4 +- src/libs/kata-types/src/config/default.rs | 20 +++---- .../src/config/hypervisor/dragonball.rs | 39 ++++++++------ .../kata-types/src/config/hypervisor/mod.rs | 54 +++++++++---------- .../kata-types/src/config/hypervisor/qemu.rs | 15 ++++-- src/libs/kata-types/src/config/mod.rs | 12 ++--- src/libs/kata-types/src/config/runtime.rs | 4 +- .../kata-types/src/config/runtime_vendor.rs | 2 +- 8 files changed, 82 insertions(+), 68 deletions(-) diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 82d3eb72ff86..ef9546ca9d47 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -56,8 +56,8 @@ pub struct Agent { } impl ConfigOps for Agent { - fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { - AgentVendor::adjust_configuration(conf)?; + fn adjust_config(conf: &mut TomlConfig) -> Result<()> { + AgentVendor::adjust_config(conf)?; Ok(()) } diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 6b047ba8ed6e..d55ccf7e974e 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -26,33 +26,33 @@ pub const DEFAULT_BLOCK_NVDIMM_MEM_OFFSET: u64 = 0; pub const DEFAULT_SHARED_FS_TYPE: &str = "virtio-fs"; pub const DEFAULT_VIRTIO_FS_CACHE_MODE: &str = "none"; pub const DEFAULT_VIRTIO_FS_DAX_SIZE_MB: u32 = 1024; -pub const DEFAULT_SHARED_9PFS_SIZE: u32 = 128 * 1024; -pub const MIN_SHARED_9PFS_SIZE: u32 = 4 * 1024; -pub const MAX_SHARED_9PFS_SIZE: u32 = 8 * 1024 * 1024; +pub const DEFAULT_SHARED_9PFS_SIZE_MB: u32 = 128 * 1024; +pub const MIN_SHARED_9PFS_SIZE_MB: u32 = 4 * 1024; +pub const MAX_SHARED_9PFS_SIZE_MB: u32 = 8 * 1024 * 1024; -pub const DEFAULT_GUEST_HOOK_PATH: &str = "/opt"; +pub const DEFAULT_GUEST_HOOK_PATH: &str = "/opt/kata/hooks"; pub const DEFAULT_GUEST_VCPUS: u32 = 1; -// Default configuration for Dragonball +// Default configuration for dragonball pub const DEFAULT_DRAGONBALL_GUEST_KERNEL_IMAGE: &str = "vmlinuz"; pub const DEFAULT_DRAGONBALL_GUEST_KERNEL_PARAMS: &str = ""; pub const DEFAULT_DRAGONBALL_ENTROPY_SOURCE: &str = "/dev/urandom"; -pub const DEFAULT_DRAGONBALL_MEMORY_SIZE: u32 = 128; +pub const DEFAULT_DRAGONBALL_MEMORY_SIZE_MB: u32 = 128; pub const DEFAULT_DRAGONBALL_MEMORY_SLOTS: u32 = 128; pub const MAX_DRAGONBALL_VCPUS: u32 = 256; -pub const MIN_DRAGONBALL_MEMORY_SIZE: u32 = 64; +pub const MIN_DRAGONBALL_MEMORY_SIZE_MB: u32 = 64; // Default configuration for qemu -pub const DEFAULT_QEMU_BINARY_PATH: &str = "qemu"; +pub const DEFAULT_QEMU_BINARY_PATH: &str = "/usr/bin/qemu-system-x86_64"; pub const DEFAULT_QEMU_CONTROL_PATH: &str = ""; pub const DEFAULT_QEMU_MACHINE_TYPE: &str = "q35"; pub const DEFAULT_QEMU_ENTROPY_SOURCE: &str = "/dev/urandom"; pub const DEFAULT_QEMU_GUEST_KERNEL_IMAGE: &str = "vmlinuz"; pub const DEFAULT_QEMU_GUEST_KERNEL_PARAMS: &str = ""; pub const DEFAULT_QEMU_FIRMWARE_PATH: &str = ""; -pub const DEFAULT_QEMU_MEMORY_SIZE: u32 = 128; +pub const DEFAULT_QEMU_MEMORY_SIZE_MB: u32 = 128; pub const DEFAULT_QEMU_MEMORY_SLOTS: u32 = 128; pub const DEFAULT_QEMU_PCI_BRIDGES: u32 = 2; pub const MAX_QEMU_PCI_BRIDGES: u32 = 5; pub const MAX_QEMU_VCPUS: u32 = 256; -pub const MIN_QEMU_MEMORY_SIZE: u32 = 64; +pub const MIN_QEMU_MEMORY_SIZE_MB: u32 = 64; diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs index bcb43eaba33a..7a8c4c894304 100644 --- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -10,14 +10,14 @@ use std::u32; use super::{default, register_hypervisor_plugin}; use crate::config::default::MAX_DRAGONBALL_VCPUS; -use crate::config::default::MIN_DRAGONBALL_MEMORY_SIZE; +use crate::config::default::MIN_DRAGONBALL_MEMORY_SIZE_MB; use crate::config::hypervisor::{ VIRTIO_BLK, VIRTIO_BLK_MMIO, VIRTIO_FS, VIRTIO_FS_INLINE, VIRTIO_PMEM, }; use crate::config::{ConfigPlugin, TomlConfig}; use crate::{eother, resolve_path, validate_path}; -/// Hypervisor name for qemu, used to index `TomlConfig::hypervisor`. +/// Hypervisor name for dragonball, used to index `TomlConfig::hypervisor`. pub const HYPERVISOR_NAME_DRAGONBALL: &str = "dragonball"; /// Configuration information for dragonball. @@ -42,16 +42,16 @@ impl ConfigPlugin for DragonballConfig { MAX_DRAGONBALL_VCPUS } fn get_min_memory(&self) -> u32 { - MIN_DRAGONBALL_MEMORY_SIZE + MIN_DRAGONBALL_MEMORY_SIZE_MB } fn name(&self) -> &str { HYPERVISOR_NAME_DRAGONBALL } /// Adjust the configuration information after loading from configuration file. - fn adjust_configuration(&self, conf: &mut TomlConfig) -> Result<()> { + fn adjust_config(&self, conf: &mut TomlConfig) -> Result<()> { if let Some(db) = conf.hypervisor.get_mut(HYPERVISOR_NAME_DRAGONBALL) { - resolve_path!(db.jailer_path, "Dragonball jailer path {} is invalid: {}")?; + resolve_path!(db.jailer_path, "dragonball jailer path {} is invalid: {}")?; if db.boot_info.kernel.is_empty() { db.boot_info.kernel = default::DEFAULT_DRAGONBALL_GUEST_KERNEL_IMAGE.to_string(); @@ -71,7 +71,7 @@ impl ConfigPlugin for DragonballConfig { } if db.memory_info.default_memory == 0 { - db.memory_info.default_memory = default::DEFAULT_DRAGONBALL_MEMORY_SIZE; + db.memory_info.default_memory = default::DEFAULT_DRAGONBALL_MEMORY_SIZE_MB; } if db.memory_info.memory_slots == 0 { db.memory_info.memory_slots = default::DEFAULT_DRAGONBALL_MEMORY_SLOTS; @@ -97,9 +97,9 @@ impl ConfigPlugin for DragonballConfig { if !db.valid_ctlpaths.is_empty() { return Err(eother!("CtlPath for dragonball hypervisor should be empty")); } - validate_path!(db.jailer_path, "Dragonball jailer path {} is invalid: {}")?; + validate_path!(db.jailer_path, "dragonball jailer path {} is invalid: {}")?; if db.enable_iothreads { - return Err(eother!("Dragonball hypervisor doesn't support IO threads.")); + return Err(eother!("dragonball hypervisor doesn't support IO threads.")); } if !db.blockdev_info.disable_block_device_use @@ -137,49 +137,56 @@ impl ConfigPlugin for DragonballConfig { || db.cpu_info.default_maxvcpus > default::MAX_DRAGONBALL_VCPUS { return Err(eother!( - "Dragonball hypervisor can not support {} vCPUs", + "dragonball hypervisor can not support {} vCPUs", db.cpu_info.default_maxvcpus )); } if db.device_info.enable_iommu || db.device_info.enable_iommu_platform { - return Err(eother!("Dragonball hypervisor does not support vIOMMU")); + return Err(eother!("dragonball hypervisor does not support vIOMMU")); } if db.device_info.hotplug_vfio_on_root_bus || db.device_info.default_bridges > 0 || db.device_info.pcie_root_port > 0 { return Err(eother!( - "Dragonball hypervisor does not support PCI hotplug options" + "dragonball hypervisor does not support PCI hotplug options" )); } if !db.machine_info.machine_type.is_empty() { return Err(eother!( - "Dragonball hypervisor does not support machine_type" + "dragonball hypervisor does not support machine_type" )); } if !db.machine_info.pflashes.is_empty() { - return Err(eother!("Dragonball hypervisor does not support pflashes")); + return Err(eother!("dragonball hypervisor does not support pflashes")); } if db.memory_info.enable_guest_swap { return Err(eother!( - "Dragonball hypervisor doesn't support enable_guest_swap" + "dragonball hypervisor doesn't support enable_guest_swap" )); } if db.security_info.rootless { return Err(eother!( - "Dragonball hypervisor does not support rootless mode" + "dragonball hypervisor does not support rootless mode" )); } if let Some(v) = db.shared_fs.shared_fs.as_ref() { if v != VIRTIO_FS && v != VIRTIO_FS_INLINE { - return Err(eother!("Dragonball hypervisor doesn't support {}", v)); + return Err(eother!("dragonball hypervisor doesn't support {}", v)); } } + + if db.memory_info.default_memory < MIN_DRAGONBALL_MEMORY_SIZE_MB { + return Err(eother!( + "dragonball hypervisor has minimal memory limitation {}", + MIN_DRAGONBALL_MEMORY_SIZE_MB + )); + } } Ok(()) diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 2d0ded6b47f5..9e0c83445b6f 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -134,7 +134,7 @@ pub struct BlockDeviceInfo { impl BlockDeviceInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { if self.disable_block_device_use { self.block_device_driver = "".to_string(); self.enable_vhost_user_store = false; @@ -217,7 +217,7 @@ pub struct BootInfo { impl BootInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { resolve_path!(self.kernel, "guest kernel image file {} is invalid: {}")?; resolve_path!(self.image, "guest boot image file {} is invalid: {}")?; resolve_path!(self.initrd, "guest initrd image file {} is invalid: {}")?; @@ -286,7 +286,7 @@ pub struct CpuInfo { impl CpuInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { let features: Vec<&str> = self.cpu_features.split(',').map(|v| v.trim()).collect(); self.cpu_features = features.join(","); Ok(()) @@ -347,7 +347,7 @@ pub struct DebugInfo { impl DebugInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { Ok(()) } @@ -406,7 +406,7 @@ pub struct DeviceInfo { impl DeviceInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { if self.default_bridges > MAX_BRIDGE_SIZE { self.default_bridges = MAX_BRIDGE_SIZE; } @@ -463,7 +463,7 @@ pub struct MachineInfo { impl MachineInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { let accelerators: Vec<&str> = self .machine_accelerators .split(',') @@ -566,7 +566,7 @@ pub struct MemoryInfo { impl MemoryInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { resolve_path!( self.file_mem_backend, "Memory backend file {} is invalid: {}" @@ -624,7 +624,7 @@ pub struct NetworkInfo { impl NetworkInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { Ok(()) } @@ -688,7 +688,7 @@ pub struct SecurityInfo { impl SecurityInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { if self.guest_hook_path.is_empty() { self.guest_hook_path = default::DEFAULT_GUEST_HOOK_PATH.to_string(); } @@ -770,7 +770,7 @@ pub struct SharedFsInfo { impl SharedFsInfo { /// Adjust the configuration information after loading from configuration file. - pub fn adjust_configuration(&mut self) -> Result<()> { + pub fn adjust_config(&mut self) -> Result<()> { if self.shared_fs.as_deref() == Some("") { self.shared_fs = Some(default::DEFAULT_SHARED_FS_TYPE.to_string()); } @@ -779,7 +779,7 @@ impl SharedFsInfo { Some(VIRTIO_FS_INLINE) => self.adjust_virtio_fs(true)?, Some(VIRTIO_9P) => { if self.msize_9p == 0 { - self.msize_9p = default::DEFAULT_SHARED_9PFS_SIZE; + self.msize_9p = default::DEFAULT_SHARED_9PFS_SIZE_MB; } } _ => {} @@ -795,12 +795,12 @@ impl SharedFsInfo { Some(VIRTIO_FS) => self.validate_virtio_fs(false), Some(VIRTIO_FS_INLINE) => self.validate_virtio_fs(true), Some(VIRTIO_9P) => { - if self.msize_9p < default::MIN_SHARED_9PFS_SIZE - || self.msize_9p > default::MAX_SHARED_9PFS_SIZE + if self.msize_9p < default::MIN_SHARED_9PFS_SIZE_MB + || self.msize_9p > default::MAX_SHARED_9PFS_SIZE_MB { return Err(eother!( "Invalid 9p configuration msize 0x{:x}, min value is 0x{:x}, max value is 0x{:x}", - self.msize_9p,default::MIN_SHARED_9PFS_SIZE, default::MAX_SHARED_9PFS_SIZE + self.msize_9p,default::MIN_SHARED_9PFS_SIZE_MB, default::MAX_SHARED_9PFS_SIZE_MB )); } Ok(()) @@ -967,26 +967,26 @@ impl Hypervisor { } impl ConfigOps for Hypervisor { - fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { - HypervisorVendor::adjust_configuration(conf)?; + fn adjust_config(conf: &mut TomlConfig) -> Result<()> { + HypervisorVendor::adjust_config(conf)?; let hypervisors: Vec = conf.hypervisor.keys().cloned().collect(); for hypervisor in hypervisors.iter() { if let Some(plugin) = get_hypervisor_plugin(hypervisor) { - plugin.adjust_configuration(conf)?; + plugin.adjust_config(conf)?; // Safe to unwrap() because `hypervisor` is a valid key in the hash map. let hv = conf.hypervisor.get_mut(hypervisor).ok_or_else(|| { io::Error::new(io::ErrorKind::NotFound, "hypervisor not found".to_string()) })?; - hv.blockdev_info.adjust_configuration()?; - hv.boot_info.adjust_configuration()?; - hv.cpu_info.adjust_configuration()?; - hv.debug_info.adjust_configuration()?; - hv.device_info.adjust_configuration()?; - hv.machine_info.adjust_configuration()?; - hv.memory_info.adjust_configuration()?; - hv.network_info.adjust_configuration()?; - hv.security_info.adjust_configuration()?; - hv.shared_fs.adjust_configuration()?; + hv.blockdev_info.adjust_config()?; + hv.boot_info.adjust_config()?; + hv.cpu_info.adjust_config()?; + hv.debug_info.adjust_config()?; + hv.device_info.adjust_config()?; + hv.machine_info.adjust_config()?; + hv.memory_info.adjust_config()?; + hv.network_info.adjust_config()?; + hv.security_info.adjust_config()?; + hv.shared_fs.adjust_config()?; } else { return Err(eother!("Can not find plugin for hypervisor {}", hypervisor)); } diff --git a/src/libs/kata-types/src/config/hypervisor/qemu.rs b/src/libs/kata-types/src/config/hypervisor/qemu.rs index 52fc4f47ddb8..945abc4b48d6 100644 --- a/src/libs/kata-types/src/config/hypervisor/qemu.rs +++ b/src/libs/kata-types/src/config/hypervisor/qemu.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use super::{default, register_hypervisor_plugin}; use crate::config::default::MAX_QEMU_VCPUS; -use crate::config::default::MIN_QEMU_MEMORY_SIZE; +use crate::config::default::MIN_QEMU_MEMORY_SIZE_MB; use crate::config::hypervisor::VIRTIO_BLK_MMIO; use crate::config::{ConfigPlugin, TomlConfig}; @@ -42,14 +42,14 @@ impl ConfigPlugin for QemuConfig { } fn get_min_memory(&self) -> u32 { - MIN_QEMU_MEMORY_SIZE + MIN_QEMU_MEMORY_SIZE_MB } fn name(&self) -> &str { HYPERVISOR_NAME_QEMU } /// Adjust the configuration information after loading from configuration file. - fn adjust_configuration(&self, conf: &mut TomlConfig) -> Result<()> { + fn adjust_config(&self, conf: &mut TomlConfig) -> Result<()> { if let Some(qemu) = conf.hypervisor.get_mut(HYPERVISOR_NAME_QEMU) { if qemu.path.is_empty() { qemu.path = default::DEFAULT_QEMU_BINARY_PATH.to_string(); @@ -83,7 +83,7 @@ impl ConfigPlugin for QemuConfig { } if qemu.memory_info.default_memory == 0 { - qemu.memory_info.default_memory = default::DEFAULT_QEMU_MEMORY_SIZE; + qemu.memory_info.default_memory = default::DEFAULT_QEMU_MEMORY_SIZE_MB; } if qemu.memory_info.memory_slots == 0 { qemu.memory_info.memory_slots = default::DEFAULT_QEMU_MEMORY_SLOTS; @@ -136,6 +136,13 @@ impl ConfigPlugin for QemuConfig { qemu.device_info.default_bridges )); } + + if qemu.memory_info.default_memory < MIN_QEMU_MEMORY_SIZE_MB { + return Err(eother!( + "Qemu hypervisor has minimal memory limitation {}", + MIN_QEMU_MEMORY_SIZE_MB + )); + } } Ok(()) diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 5d3b2d6bce7b..435b8d1f0663 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -36,7 +36,7 @@ pub trait ConfigPlugin: Send + Sync { fn name(&self) -> &str; /// Adjust the configuration information after loading from configuration file. - fn adjust_configuration(&self, _conf: &mut TomlConfig) -> Result<()>; + fn adjust_config(&self, _conf: &mut TomlConfig) -> Result<()>; /// Validate the configuration information. fn validate(&self, _conf: &TomlConfig) -> Result<()>; @@ -51,7 +51,7 @@ pub trait ConfigPlugin: Send + Sync { /// Trait to manipulate Kata configuration information. pub trait ConfigOps { /// Adjust the configuration information after loading from configuration file. - fn adjust_configuration(_conf: &mut TomlConfig) -> Result<()> { + fn adjust_config(_conf: &mut TomlConfig) -> Result<()> { Ok(()) } @@ -64,7 +64,7 @@ pub trait ConfigOps { /// Trait to manipulate global Kata configuration information. pub trait ConfigObjectOps { /// Adjust the configuration information after loading from configuration file. - fn adjust_configuration(&mut self) -> Result<()> { + fn adjust_config(&mut self) -> Result<()> { Ok(()) } @@ -136,9 +136,9 @@ impl TomlConfig { /// Load Kata configuration information from string. pub fn load(content: &str) -> Result { let mut config: TomlConfig = toml::from_str(content)?; - Hypervisor::adjust_configuration(&mut config)?; - Runtime::adjust_configuration(&mut config)?; - Agent::adjust_configuration(&mut config)?; + Hypervisor::adjust_config(&mut config)?; + Runtime::adjust_config(&mut config)?; + Agent::adjust_config(&mut config)?; info!(sl!(), "get kata config: {:?}", config); Ok(config) } diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 48e66858eba5..7c417fdc2750 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -113,8 +113,8 @@ pub struct Runtime { } impl ConfigOps for Runtime { - fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { - RuntimeVendor::adjust_configuration(conf)?; + fn adjust_config(conf: &mut TomlConfig) -> Result<()> { + RuntimeVendor::adjust_config(conf)?; if conf.runtime.internetworking_model.is_empty() { conf.runtime.internetworking_model = default::DEFAULT_INTERNETWORKING_MODEL.to_owned(); } diff --git a/src/libs/kata-types/src/config/runtime_vendor.rs b/src/libs/kata-types/src/config/runtime_vendor.rs index 67fafe21d72f..e12a63f399f8 100644 --- a/src/libs/kata-types/src/config/runtime_vendor.rs +++ b/src/libs/kata-types/src/config/runtime_vendor.rs @@ -20,7 +20,7 @@ pub struct RuntimeVendor { } impl ConfigOps for RuntimeVendor { - fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> { + fn adjust_config(conf: &mut TomlConfig) -> Result<()> { if conf.runtime.vendor.log_level > Level::Debug as u32 { conf.runtime.debug = true; } From 45a00b4f02972d21eac9de42ec541e20bd72f56c Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Wed, 8 Dec 2021 23:09:13 +0800 Subject: [PATCH 0026/1953] libs/sys-util: add kata-sys-util crate under src/libs The kata-sys-util crate is a collection of modules that provides helpers and utilities used by multiple Kata Containers components. Fixes: #3305 Signed-off-by: Liu Jiang --- src/libs/Cargo.lock | 24 +++++------------------- src/libs/Cargo.toml | 1 + src/libs/README.md | 1 + src/libs/kata-sys-util/Cargo.toml | 13 +++++++++++++ src/libs/kata-sys-util/README.md | 19 +++++++++++++++++++ src/libs/kata-sys-util/src/lib.rs | 4 ++++ 6 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 src/libs/kata-sys-util/Cargo.toml create mode 100644 src/libs/kata-sys-util/README.md create mode 100644 src/libs/kata-sys-util/src/lib.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 341abf83b46d..1467aa391377 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -89,7 +89,6 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time", "winapi", ] @@ -307,6 +306,10 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "kata-sys-util" +version = "0.1.0" + [[package]] name = "kata-types" version = "0.1.0" @@ -382,7 +385,7 @@ dependencies = [ "log", "miow", "ntapi", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "winapi", ] @@ -810,17 +813,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "tokio" version = "1.17.0" @@ -938,12 +930,6 @@ dependencies = [ "nix 0.23.1", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/src/libs/Cargo.toml b/src/libs/Cargo.toml index 10887de3c1f5..b485eaa43ecb 100644 --- a/src/libs/Cargo.toml +++ b/src/libs/Cargo.toml @@ -2,6 +2,7 @@ members = [ "logging", "kata-types", + "kata-sys-util", "safe-path", "protocols", "oci", diff --git a/src/libs/README.md b/src/libs/README.md index 0593749a09f1..bb1a655c38d2 100644 --- a/src/libs/README.md +++ b/src/libs/README.md @@ -7,5 +7,6 @@ Currently it provides following library crates: | Library | Description | |-|-| | [logging](logging/) | Facilities to setup logging subsystem based on slog. | +| [system utilities](kata-sys-util/) | Collection of facilities and helpers to access system services. | | [types](kata-types/) | Collection of constants and data types shared by multiple Kata Containers components. | | [safe-path](safe-path/) | Utilities to safely resolve filesystem paths. | diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml new file mode 100644 index 000000000000..52baa85f5f71 --- /dev/null +++ b/src/libs/kata-sys-util/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "kata-sys-util" +version = "0.1.0" +description = "System Utilities for Kata Containers" +keywords = ["kata", "container", "runtime"] +authors = ["The Kata Containers community "] +repository = "https://github.com/kata-containers/kata-containers.git" +homepage = "https://katacontainers.io/" +readme = "README.md" +license = "Apache-2.0" +edition = "2018" + +[dependencies] diff --git a/src/libs/kata-sys-util/README.md b/src/libs/kata-sys-util/README.md new file mode 100644 index 000000000000..0c3f887bcbea --- /dev/null +++ b/src/libs/kata-sys-util/README.md @@ -0,0 +1,19 @@ +# kata-sys-util + +This crate is a collection of utilities and helpers for +[Kata Containers](https://github.com/kata-containers/kata-containers/) components to access system services. + +It provides safe wrappers over system services, such as: +- cgroups +- file systems +- mount +- NUMA + +## Support + +**Operating Systems**: +- Linux + +## License + +This code is licensed under [Apache-2.0](../../../LICENSE). diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs new file mode 100644 index 000000000000..09d8d185811a --- /dev/null +++ b/src/libs/kata-sys-util/src/lib.rs @@ -0,0 +1,4 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// From ccd03e2caeb82dcb46c4cd678d32ecb09c9e75d6 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Wed, 8 Dec 2021 23:12:13 +0800 Subject: [PATCH 0027/1953] libs/sys-util: add wrappers for mount and fs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add some wrappers for mount and fs syscall. Signed-off-by: Liu Jiang Signed-off-by: Bin Liu Signed-off-by: Fupan Li  Signed-off-by: Huamin Tang Signed-off-by: Lei Wang Signed-off-by: Quanwei Zhou --- src/libs/Cargo.lock | 100 ++- src/libs/kata-sys-util/Cargo.toml | 13 + src/libs/kata-sys-util/src/fs.rs | 111 +++ src/libs/kata-sys-util/src/lib.rs | 22 + src/libs/kata-sys-util/src/mount.rs | 1094 +++++++++++++++++++++++++++ 5 files changed, 1331 insertions(+), 9 deletions(-) create mode 100644 src/libs/kata-sys-util/src/fs.rs create mode 100644 src/libs/kata-sys-util/src/mount.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 1467aa391377..7106c4bc1d47 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bitflags" @@ -92,6 +92,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "common-path" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" + [[package]] name = "crossbeam-channel" version = "0.5.2" @@ -129,6 +135,17 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "fail" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" +dependencies = [ + "lazy_static", + "log", + "rand", +] + [[package]] name = "fastrand" version = "1.6.0" @@ -233,6 +250,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + [[package]] name = "glob" version = "0.3.0" @@ -309,6 +337,18 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "kata-sys-util" version = "0.1.0" +dependencies = [ + "common-path", + "fail", + "kata-types", + "lazy_static", + "libc", + "nix 0.23.1", + "slog", + "slog-scope", + "tempfile", + "thiserror", +] [[package]] name = "kata-types" @@ -385,7 +425,7 @@ dependencies = [ "log", "miow", "ntapi", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi", ] @@ -506,6 +546,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + [[package]] name = "proc-macro2" version = "1.0.37" @@ -616,6 +662,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -667,18 +743,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.135" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf9235533494ea2ddcdb794665461814781c53f19d87b76e571a1c35acbad2b" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.135" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dcde03d87d4c973c04be249e7d8f0b35db1c848c487bd43032808e59dd8328d" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" dependencies = [ "proc-macro2", "quote", @@ -806,9 +882,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" dependencies = [ "once_cell", ] @@ -930,6 +1006,12 @@ dependencies = [ "nix 0.23.1", ] +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 52baa85f5f71..264849468c3d 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -11,3 +11,16 @@ license = "Apache-2.0" edition = "2018" [dependencies] +common-path = "=1.0.0" +fail = "0.5.0" +lazy_static = "1.4.0" +libc = "0.2.100" +nix = "0.23.0" +slog = "2.5.2" +slog-scope = "4.4.0" +thiserror = "1.0.30" + +kata-types = { path = "../kata-types" } + +[dev-dependencies] +tempfile = "3.2.0" diff --git a/src/libs/kata-sys-util/src/fs.rs b/src/libs/kata-sys-util/src/fs.rs new file mode 100644 index 000000000000..04da18b33a60 --- /dev/null +++ b/src/libs/kata-sys-util/src/fs.rs @@ -0,0 +1,111 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::ffi::OsString; +use std::fs; +use std::io::Result; +use std::path::{Path, PathBuf}; + +use crate::eother; + +// from linux.git/fs/fuse/inode.c: #define FUSE_SUPER_MAGIC 0x65735546 +const FUSE_SUPER_MAGIC: u32 = 0x65735546; + +/// Get bundle path (current working directory). +pub fn get_bundle_path() -> Result { + std::env::current_dir() +} + +/// Get the basename of the canonicalized path +pub fn get_base_name>(src: P) -> Result { + let s = src.as_ref().canonicalize()?; + s.file_name().map(|v| v.to_os_string()).ok_or_else(|| { + eother!( + "failed to get base name of path {}", + src.as_ref().to_string_lossy() + ) + }) +} + +/// Check whether `path` is on a fuse filesystem. +pub fn is_fuse_fs>(path: P) -> bool { + if let Ok(st) = nix::sys::statfs::statfs(path.as_ref()) { + if st.filesystem_type().0 == FUSE_SUPER_MAGIC as i64 { + return true; + } + } + false +} + +/// Check whether `path` is on a overlay filesystem. +pub fn is_overlay_fs>(path: P) -> bool { + if let Ok(st) = nix::sys::statfs::statfs(path.as_ref()) { + if st.filesystem_type() == nix::sys::statfs::OVERLAYFS_SUPER_MAGIC { + return true; + } + } + false +} + +/// Check whether the given path is a symlink. +pub fn is_symlink>(path: P) -> std::io::Result { + let path = path.as_ref(); + let meta = fs::symlink_metadata(path)?; + + Ok(meta.file_type().is_symlink()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mount::umount_all; + use std::process::Command; + use thiserror::private::PathAsDisplay; + + #[test] + fn test_get_base_name() { + assert_eq!(&get_base_name("/etc/hostname").unwrap(), "hostname"); + assert_eq!(&get_base_name("/bin").unwrap(), "bin"); + assert!(&get_base_name("/").is_err()); + assert!(&get_base_name("").is_err()); + assert!(get_base_name("/no/such/path________yeah").is_err()); + } + + #[test] + fn test_is_symlink() { + let tmpdir = tempfile::tempdir().unwrap(); + let path = tmpdir.path(); + + std::os::unix::fs::symlink(path, path.join("a")).unwrap(); + assert!(is_symlink(path.join("a")).unwrap()); + } + + #[test] + fn test_is_overlayfs() { + let tmpdir1 = tempfile::tempdir().unwrap(); + let tmpdir2 = tempfile::tempdir().unwrap(); + let tmpdir3 = tempfile::tempdir().unwrap(); + let tmpdir4 = tempfile::tempdir().unwrap(); + + let option = format!( + "-o lowerdir={},upperdir={},workdir={}", + tmpdir1.path().as_display(), + tmpdir2.path().display(), + tmpdir3.path().display() + ); + let target = format!("{}", tmpdir4.path().display()); + + Command::new("/bin/mount") + .arg("-t overlay") + .arg(option) + .arg("overlay") + .arg(target) + .output() + .unwrap(); + assert!(is_overlay_fs(tmpdir4.path())); + umount_all(tmpdir4.path(), false).unwrap(); + } +} diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 09d8d185811a..62706c15de4f 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -2,3 +2,25 @@ // // SPDX-License-Identifier: Apache-2.0 // + +#[macro_use] +extern crate slog; + +pub mod fs; +pub mod mount; + +// Convenience macro to obtain the scoped logger +#[macro_export] +macro_rules! sl { + () => { + slog_scope::logger() + }; +} + +#[macro_export] +macro_rules! eother { + () => (std::io::Error::new(std::io::ErrorKind::Other, "")); + ($fmt:expr, $($arg:tt)*) => ({ + std::io::Error::new(std::io::ErrorKind::Other, format!($fmt, $($arg)*)) + }) +} diff --git a/src/libs/kata-sys-util/src/mount.rs b/src/libs/kata-sys-util/src/mount.rs new file mode 100644 index 000000000000..2831ff96dd66 --- /dev/null +++ b/src/libs/kata-sys-util/src/mount.rs @@ -0,0 +1,1094 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Utilities and helpers to execute mount operations on Linux systems. +//! +//! These utilities and helpers are specially designed and implemented to support container runtimes +//! on Linux systems, so they may not be generic enough. +//! +//! # Quotation from [mount(2)](https://man7.org/linux/man-pages/man2/mount.2.html) +//! +//! A call to mount() performs one of a number of general types of operation, depending on the bits +//! specified in mountflags. The choice of which operation to perform is determined by testing the +//! bits set in mountflags, with the tests being conducted in the order listed here: +//! - Remount an existing mount: mountflags includes MS_REMOUNT. +//! - Create a bind mount: mountflags includes MS_BIND. +//! - Change the propagation type of an existing mount: mountflags includes one of MS_SHARED, +//! MS_PRIVATE, MS_SLAVE, or MS_UNBINDABLE. +//! - Move an existing mount to a new location: mountflags includes MS_MOVE. +//! - Create a new mount: mountflags includes none of the above flags. +//! +//! Since Linux 2.6.26, the MS_REMOUNT flag can be used with MS_BIND to modify only the +//! per-mount-point flags. This is particularly useful for setting or clearing the "read-only" +//! flag on a mount without changing the underlying filesystem. Specifying mountflags as: +//! MS_REMOUNT | MS_BIND | MS_RDONLY +//! will make access through this mountpoint read-only, without affecting other mounts. +//! +//! # Safety +//! +//! Mount related operations are sensitive to security flaws, especially when dealing with symlinks. +//! There are several CVEs related to file path handling, for example +//! [CVE-2021-30465](https://github.com/opencontainers/runc/security/advisories/GHSA-c3xm-pvg7-gh7r). +//! +//! So some design rules are adopted here: +//! - all mount variants (`bind_remount_read_only()`, `bind_mount()`, `Mounter::mount()`) assume +//! that all received paths are safe. +//! - the caller must ensure safe version of `PathBuf` are passed to mount variants. +//! - `create_mount_destination()` may be used to generated safe `PathBuf` for mount destinations. +//! - the `safe_path` crate should be used to generate safe `PathBuf` for general cases. + +use std::fmt::Debug; +use std::fs; +use std::io::{self, BufRead}; +use std::os::raw::c_char; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use lazy_static::lazy_static; +use nix::mount::{mount, MntFlags, MsFlags}; +use nix::{unistd, NixPath}; + +use crate::fs::is_symlink; +use crate::sl; + +/// Default permission for directories created for mountpoint. +const MOUNT_PERM: u32 = 0o755; + +const PROC_MOUNTS_FILE: &str = "/proc/mounts"; +const PROC_FIELDS_PER_LINE: usize = 6; +const PROC_DEVICE_INDEX: usize = 0; +const PROC_PATH_INDEX: usize = 1; +const PROC_TYPE_INDEX: usize = 2; + +// Sadly nix/libc doesn't have UMOUNT_NOFOLLOW although it's there since Linux 2.6.34 +const UMOUNT_NOFOLLOW: i32 = 0x8; + +lazy_static! { + static ref MAX_MOUNT_PARAM_SIZE: usize = + if let Ok(Some(v)) = unistd::sysconf(unistd::SysconfVar::PAGE_SIZE) { + v as usize + } else { + panic!("cannot get PAGE_SIZE by sysconf()"); + }; + +// Propagation flags for mounting container volumes. + static ref PROPAGATION_FLAGS: MsFlags = + MsFlags::MS_SHARED | MsFlags::MS_PRIVATE | MsFlags::MS_SLAVE | MsFlags::MS_UNBINDABLE; + +} + +/// Errors related to filesystem mount operations. +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Can not bind mount {0} to {1}: {2}")] + BindMount(PathBuf, PathBuf, nix::Error), + #[error("Failure injection: {0}")] + FailureInject(String), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("Invalid mountpoint entry (expected {0} fields, got {1}) fields: {2}")] + InvalidMountEntry(usize, usize, String), + #[error("Invalid mount option: {0}")] + InvalidMountOption(String), + #[error("Invalid path: {0}")] + InvalidPath(PathBuf), + #[error("Failure in waiting for thread: {0}")] + Join(String), + #[error("Can not mount {0} to {1}: {2}")] + Mount(PathBuf, PathBuf, nix::Error), + #[error("Mount option exceeds 4K size")] + MountOptionTooBig, + #[error("Path for mountpoint is null")] + NullMountPointPath, + #[error("Faile to open file {0} by path, {1}")] + OpenByPath(PathBuf, io::Error), + #[error("Can not read metadata of {0}, {1}")] + ReadMetadata(PathBuf, io::Error), + #[error("Can not remount {0}: {1}")] + Remount(PathBuf, nix::Error), + #[error("Can not find mountpoint for {0}")] + NoMountEntry(String), + #[error("Can not umount {0}, {1}")] + Umount(PathBuf, io::Error), +} + +/// A specialized version of `std::result::Result` for mount operations. +pub type Result = std::result::Result; + +/// Information of mount record from `/proc/mounts`. +pub struct LinuxMountInfo { + /// Source of mount, first field of records from `/proc/mounts`. + pub device: String, + /// Destination of mount, second field of records from `/proc/mounts`. + pub path: String, + /// Filesystem type of mount, third field of records from `/proc/mounts`. + pub fs_type: String, +} + +/// Get the device and file system type of a mount point by parsing `/proc/mounts`. +pub fn get_linux_mount_info(mount_point: &str) -> Result { + let mount_file = fs::File::open(PROC_MOUNTS_FILE)?; + let lines = io::BufReader::new(mount_file).lines(); + + for mount in lines.flatten() { + let fields: Vec<&str> = mount.split(' ').collect(); + + if fields.len() != PROC_FIELDS_PER_LINE { + return Err(Error::InvalidMountEntry( + PROC_FIELDS_PER_LINE, + fields.len(), + mount, + )); + } + + if mount_point == fields[PROC_PATH_INDEX] { + return Ok(LinuxMountInfo { + device: fields[PROC_DEVICE_INDEX].to_string(), + path: fields[PROC_PATH_INDEX].to_string(), + fs_type: fields[PROC_TYPE_INDEX].to_string(), + }); + } + } + + Err(Error::NoMountEntry(mount_point.to_owned())) +} + +/// Recursively create destination for a mount. +/// +/// For a normal mount, the destination will always be a directory. For bind mount, the destination +/// must be a directory if the source is a directory, otherwise the destination must be a normal +/// file. If directories are created, their permissions are initialized to MountPerm. +/// +/// # Safety +/// +/// Every container has a root filesystems `rootfs`. When creating bind mounts for a container, +/// the destination should always be within the container's `rootfs`. Otherwise it's a serious +/// security flaw for container to read/override host side filesystem contents. Please refer to +/// following CVEs for example: +/// - [CVE-2021-30465](https://github.com/opencontainers/runc/security/advisories/GHSA-c3xm-pvg7-gh7r) +/// +/// To ensure security, the `create_mount_destination()` function takes an extra parameter `root`, +/// which is used to ensure that `dst` is within the specified directory. And a safe version of +/// `PathBuf` is returned to avoid TOCTTOU type of flaws. +pub fn create_mount_destination, D: AsRef, R: AsRef>( + src: S, + dst: D, + _root: R, + fs_type: &str, +) -> Result + Debug> { + // TODO: https://github.com/kata-containers/kata-containers/issues/3473 + let dst = dst.as_ref(); + let parent = dst + .parent() + .ok_or_else(|| Error::InvalidPath(dst.to_path_buf()))?; + let mut builder = fs::DirBuilder::new(); + builder.mode(MOUNT_PERM).recursive(true).create(parent)?; + + if fs_type == "bind" { + // The source and destination for bind mounting must be the same type: file or directory. + if !src.as_ref().is_dir() { + fs::OpenOptions::new() + .mode(MOUNT_PERM) + .write(true) + .create(true) + .open(dst)?; + return Ok(dst.to_path_buf()); + } + } + + if let Err(e) = builder.create(dst) { + if e.kind() != std::io::ErrorKind::AlreadyExists { + return Err(e.into()); + } + } + if !dst.is_dir() { + Err(Error::InvalidPath(dst.to_path_buf())) + } else { + Ok(dst.to_path_buf()) + } +} + +/// Remount a bind mount into readonly mode. +/// +/// # Safety +/// Caller needs to ensure safety of the `dst` to avoid possible file path based attacks. +pub fn bind_remount_read_only>(dst: P) -> Result<()> { + let dst = dst.as_ref(); + if dst.is_empty() { + return Err(Error::NullMountPointPath); + } + let dst = dst + .canonicalize() + .map_err(|_e| Error::InvalidPath(dst.to_path_buf()))?; + + do_rebind_mount_read_only(dst, MsFlags::empty()) +} + +/// Bind mount `src` to `dst` in slave mode, optionally in readonly mode if `readonly` is true. +/// +/// # Safety +/// Caller needs to ensure: +/// - `src` exists. +/// - `dst` exists, and is suitable as destination for bind mount. +/// - `dst` is free of file path based attacks. +pub fn bind_mount_unchecked, D: AsRef>( + src: S, + dst: D, + read_only: bool, +) -> Result<()> { + fail::fail_point!("bind_mount", |_| { + Err(Error::FailureInject( + "Bind mount fail point injection".to_string(), + )) + }); + + let src = src.as_ref(); + let dst = dst.as_ref(); + if src.is_empty() { + return Err(Error::NullMountPointPath); + } + if dst.is_empty() { + return Err(Error::NullMountPointPath); + } + let abs_src = src + .canonicalize() + .map_err(|_e| Error::InvalidPath(src.to_path_buf()))?; + + // Bind mount `src` to `dst`. + mount( + Some(&abs_src), + dst, + Some("bind"), + MsFlags::MS_BIND, + Some(""), + ) + .map_err(|e| Error::BindMount(abs_src, dst.to_path_buf(), e))?; + + // Change into slave propagation mode. + mount(Some(""), dst, Some(""), MsFlags::MS_SLAVE, Some("")) + .map_err(|e| Error::Mount(PathBuf::new(), dst.to_path_buf(), e))?; + + // Optionally rebind into readonly mode. + if read_only { + do_rebind_mount_read_only(dst, MsFlags::empty())?; + } + + Ok(()) +} + +/// Trait to mount a `kata_types::mount::Mount`. +pub trait Mounter { + /// Mount to the specified `target`. + /// + /// # Safety + /// Caller needs to ensure: + /// - `target` exists, and is suitable as destination for mount. + /// - `target` is free of file path based attacks. + fn mount>(&self, target: P) -> Result<()>; +} + +impl Mounter for kata_types::mount::Mount { + // This function is modelled after + // [Mount::Mount()](https://github.com/containerd/containerd/blob/main/mount/mount_linux.go) + // from [Containerd](https://github.com/containerd/containerd) project. + fn mount>(&self, target: P) -> Result<()> { + fail::fail_point!("Mount::mount", |_| { + Err(Error::FailureInject( + "Mount::mount() fail point injection".to_string(), + )) + }); + + let target = target.as_ref().to_path_buf(); + let (chdir, (flags, data)) = + // Follow the same algorithm as Containerd: reserve 512 bytes to avoid hitting one page + // limit of mounting argument buffer. + if self.fs_type == "overlay" && self.option_size() >= *MAX_MOUNT_PARAM_SIZE - 512 { + info!( + sl!(), + "overlay mount option too long, maybe failed to mount" + ); + let (chdir, options) = compact_lowerdir_option(&self.options); + (chdir, parse_mount_options(&options)?) + } else { + (None, parse_mount_options(&self.options)?) + }; + + // Ensure propagation type change flags aren't included in other calls. + let o_flag = flags & (!*PROPAGATION_FLAGS); + + // - Normal mount without MS_REMOUNT flag + // - In the case of remounting with changed data (data != ""), need to call mount + if (flags & MsFlags::MS_REMOUNT) == MsFlags::empty() || !data.is_empty() { + mount_at( + chdir, + &self.source, + target.clone(), + &self.fs_type, + o_flag, + &data, + )?; + } + + // Change mount propagation type. + if (flags & *PROPAGATION_FLAGS) != MsFlags::empty() { + let propagation_flag = *PROPAGATION_FLAGS | MsFlags::MS_REC | MsFlags::MS_SILENT; + debug!( + sl!(), + "Change mount propagation flags to: 0x{:x}", + propagation_flag.bits() + ); + mount( + Some(""), + &target, + Some(""), + flags & propagation_flag, + Some(""), + ) + .map_err(|e| Error::Mount(PathBuf::new(), target.clone(), e))?; + } + + // Bind mount readonly. + let bro_flag = MsFlags::MS_BIND | MsFlags::MS_RDONLY; + if (o_flag & bro_flag) == bro_flag { + do_rebind_mount_read_only(target, o_flag)?; + } + + Ok(()) + } +} + +#[inline] +fn do_rebind_mount_read_only>(path: P, flags: MsFlags) -> Result<()> { + mount( + Some(""), + path.as_ref(), + Some(""), + flags | MsFlags::MS_BIND | MsFlags::MS_REMOUNT | MsFlags::MS_RDONLY, + Some(""), + ) + .map_err(|e| Error::Remount(path.as_ref().to_path_buf(), e)) +} + +/// Take fstab style mount options and parses them for use with a standard mount() syscall. +fn parse_mount_options(options: &[String]) -> Result<(MsFlags, String)> { + let mut flags: MsFlags = MsFlags::empty(); + let mut data: Vec = Vec::new(); + + for opt in options.iter() { + if opt == "defaults" { + continue; + } else if opt == "loop" { + return Err(Error::InvalidMountOption("loop".to_string())); + } else if let Some(v) = parse_mount_flags(flags, opt) { + flags = v; + } else { + data.push(opt.clone()); + } + } + + let data = data.join(","); + if data.len() > *MAX_MOUNT_PARAM_SIZE { + return Err(Error::MountOptionTooBig); + } + + Ok((flags, data)) +} + +fn parse_mount_flags(mut flags: MsFlags, flag_str: &str) -> Option { + // Following mount options are applicable to fstab only. + // - _netdev: The filesystem resides on a device that requires network access (used to prevent + // the system from attempting to mount these filesystems until the network has been enabled + // on the system). + // - auto: Can be mounted with the -a option. + // - group: Allow an ordinary user to mount the filesystem if one of that user’s groups matches + // the group of the device. This option implies the options nosuid and nodev (unless + // overridden by subsequent options, as in the option line group,dev,suid). + // - noauto: Can only be mounted explicitly (i.e., the -a option will not cause the filesystem + // to be mounted). + // - nofail: Do not report errors for this device if it does not exist. + // - owner: Allow an ordinary user to mount the filesystem if that user is the owner of the + // device. This option implies the options nosuid and nodev (unless overridden by subsequent + // options, as in the option line owner,dev,suid). + // - user: Allow an ordinary user to mount the filesystem. The name of the mounting user is + // written to the mtab file (or to the private libmount file in /run/mount on systems without + // a regular mtab) so that this same user can unmount the filesystem again. This option + // implies the options noexec, nosuid, and nodev (unless overridden by subsequent options, + // as in the option line user,exec,dev,suid). + // - nouser: Forbid an ordinary user to mount the filesystem. This is the default; it does not + // imply any other options. + // - users: Allow any user to mount and to unmount the filesystem, even when some other ordinary + // user mounted it. This option implies the options noexec, nosuid, and nodev (unless + // overridden by subsequent options, as in the option line users,exec,dev,suid). + match flag_str { + // Clear flags + "async" => flags &= !MsFlags::MS_SYNCHRONOUS, + "atime" => flags &= !MsFlags::MS_NOATIME, + "dev" => flags &= !MsFlags::MS_NODEV, + "diratime" => flags &= !MsFlags::MS_NODIRATIME, + "exec" => flags &= !MsFlags::MS_NOEXEC, + "loud" => flags &= !MsFlags::MS_SILENT, + "noiversion" => flags &= !MsFlags::MS_I_VERSION, + "nomand" => flags &= !MsFlags::MS_MANDLOCK, + "norelatime" => flags &= !MsFlags::MS_RELATIME, + "nostrictatime" => flags &= !MsFlags::MS_STRICTATIME, + "rw" => flags &= !MsFlags::MS_RDONLY, + "suid" => flags &= !MsFlags::MS_NOSUID, + // Set flags + "bind" => flags |= MsFlags::MS_BIND, + "dirsync" => flags |= MsFlags::MS_DIRSYNC, + "iversion" => flags |= MsFlags::MS_I_VERSION, + "mand" => flags |= MsFlags::MS_MANDLOCK, + "noatime" => flags |= MsFlags::MS_NOATIME, + "nodev" => flags |= MsFlags::MS_NODEV, + "nodiratime" => flags |= MsFlags::MS_NODIRATIME, + "noexec" => flags |= MsFlags::MS_NOEXEC, + "nosuid" => flags |= MsFlags::MS_NOSUID, + "rbind" => flags |= MsFlags::MS_BIND | MsFlags::MS_REC, + "relatime" => flags |= MsFlags::MS_RELATIME, + "remount" => flags |= MsFlags::MS_REMOUNT, + "ro" => flags |= MsFlags::MS_RDONLY, + "silent" => flags |= MsFlags::MS_SILENT, + "strictatime" => flags |= MsFlags::MS_STRICTATIME, + "sync" => flags |= MsFlags::MS_SYNCHRONOUS, + flag_str => { + warn!(sl!(), "BUG: unknown mount flag: {:?}", flag_str); + return None; + } + } + Some(flags) +} + +// Do mount, optionally change current working directory if `chdir` is not empty. +fn mount_at>( + chdir: Option, + source: P, + target: PathBuf, + fstype: &str, + flags: MsFlags, + data: &str, +) -> Result<()> { + let chdir = match chdir { + Some(v) => v, + None => { + return mount( + Some(source.as_ref()), + &target, + Some(fstype), + flags, + Some(data), + ) + .map_err(|e| Error::Mount(PathBuf::new(), target, e)); + } + }; + + info!( + sl!(), + "mount_at: chdir {}, source {}, target {} , fstype {}, data {}", + chdir.display(), + source.as_ref().display(), + target.display(), + fstype, + data + ); + + // TODO: https://github.com/kata-containers/kata-containers/issues/3473 + let o_flags = nix::fcntl::OFlag::O_PATH | nix::fcntl::OFlag::O_CLOEXEC; + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(o_flags.bits()) + .open(&chdir) + .map_err(|e| Error::OpenByPath(chdir.to_path_buf(), e))?; + match file.metadata() { + Ok(md) => { + if !md.is_dir() { + return Err(Error::InvalidPath(chdir)); + } + } + Err(e) => return Err(Error::ReadMetadata(chdir, e)), + } + + let cwd = unistd::getcwd().map_err(|e| Error::Io(io::Error::from_raw_os_error(e as i32)))?; + let src = source.as_ref().to_path_buf(); + let tgt = target.clone(); + let ftype = String::from(fstype); + let d = String::from(data); + let rx = Arc::new(AtomicBool::new(false)); + let tx = rx.clone(); + + // A working thread is spawned to ease error handling. + let child = std::thread::Builder::new() + .name("async_mount".to_string()) + .spawn(move || { + match unistd::fchdir(file.as_raw_fd()) { + Ok(_) => info!(sl!(), "chdir from {} to {}", cwd.display(), chdir.display()), + Err(e) => { + error!( + sl!(), + "failed to chdir from {} to {} error {:?}", + cwd.display(), + chdir.display(), + e + ); + return; + } + } + match mount( + Some(src.as_path()), + &tgt, + Some(ftype.as_str()), + flags, + Some(d.as_str()), + ) { + Ok(_) => tx.store(true, Ordering::Release), + Err(e) => error!(sl!(), "failed to mount in chdir {}: {}", chdir.display(), e), + } + match unistd::chdir(&cwd) { + Ok(_) => info!(sl!(), "chdir from {} to {}", chdir.display(), cwd.display()), + Err(e) => { + error!( + sl!(), + "failed to chdir from {} to {} error {:?}", + chdir.display(), + cwd.display(), + e + ); + } + } + })?; + child.join().map_err(|e| Error::Join(format!("{:?}", e)))?; + + if !rx.load(Ordering::Acquire) { + Err(Error::Mount( + source.as_ref().to_path_buf(), + target, + nix::Error::EIO, + )) + } else { + Ok(()) + } +} + +/// When the size of mount options is bigger than one page, try to reduce the size by compressing +/// the `lowerdir` option for overlayfs. The assumption is that lower directories for overlayfs +/// often have a common prefix. +fn compact_lowerdir_option(opts: &[String]) -> (Option, Vec) { + let mut n_opts = opts.to_vec(); + // No need to compact if there is no overlay or only one lowerdir + let (idx, lower_opts) = match find_overlay_lowerdirs(opts) { + None => return (None, n_opts), + Some(v) => { + if v.1.len() <= 1 { + return (None, n_opts); + } + v + } + }; + + let idx = idx as usize; + let common_dir = match get_longest_common_prefix(&lower_opts) { + None => return (None, n_opts), + Some(v) => { + if v.is_absolute() && v.parent().is_none() { + return (None, n_opts); + } + v + } + }; + let common_prefix = match common_dir.as_os_str().to_str() { + None => return (None, n_opts), + Some(v) => { + let mut p = v.to_string(); + p.push('/'); + p + } + }; + + info!( + sl!(), + "compact_lowerdir_option get common prefix: {}", + common_dir.display() + ); + let lower: Vec = lower_opts + .iter() + .map(|c| c.replace(&common_prefix, "")) + .collect(); + n_opts[idx as usize] = format!("lowerdir={}", lower.join(":")); + + (Some(common_dir), n_opts) +} + +fn find_overlay_lowerdirs(opts: &[String]) -> Option<(usize, Vec)> { + for (idx, o) in opts.iter().enumerate() { + if let Some(lower) = o.strip_prefix("lowerdir=") { + if !lower.is_empty() { + let c_opts: Vec = lower.split(':').map(|c| c.to_string()).collect(); + return Some((idx, c_opts)); + } + } + } + + None +} + +fn get_longest_common_prefix(opts: &[String]) -> Option { + if opts.is_empty() { + return None; + } + + let mut paths = Vec::with_capacity(opts.len()); + for opt in opts.iter() { + match Path::new(opt).parent() { + None => return None, + Some(v) => paths.push(v), + } + } + + let mut path = PathBuf::new(); + paths.sort_unstable(); + for (first, last) in paths[0] + .components() + .zip(paths[paths.len() - 1].components()) + { + if first != last { + break; + } + path.push(first); + } + + Some(path) +} + +/// Umount a mountpoint with timeout. +/// +/// # Safety +/// Caller needs to ensure safety of the `path` to avoid possible file path based attacks. +pub fn umount_timeout>(path: P, timeout: u64) -> Result<()> { + // Protect from symlink based attacks, please refer to: + // https://github.com/kata-containers/runtime/issues/2474 + // For Kata specific, we do extra protection for parent directory too. + let path = path.as_ref(); + let parent = path + .parent() + .ok_or_else(|| Error::InvalidPath(path.to_path_buf()))?; + // TODO: https://github.com/kata-containers/kata-containers/issues/3473 + if is_symlink(path).map_err(|e| Error::ReadMetadata(path.to_owned(), e))? + || is_symlink(parent).map_err(|e| Error::ReadMetadata(path.to_owned(), e))? + { + warn!( + sl!(), + "unable to umount {} which is a symbol link", + path.display() + ); + return Ok(()); + } + + if timeout == 0 { + // Lazy unmounting the mountpoint with the MNT_DETACH flag. + umount2(path, true).map_err(|e| Error::Umount(path.to_owned(), e))?; + info!(sl!(), "lazy umount for {}", path.display()); + } else { + let start_time = Instant::now(); + while let Err(e) = umount2(path, false) { + match e.kind() { + // The mountpoint has been concurrently unmounted by other threads. + io::ErrorKind::InvalidInput => break, + io::ErrorKind::WouldBlock => { + let time_now = Instant::now(); + if time_now.duration_since(start_time).as_millis() > timeout as u128 { + warn!(sl!(), + "failed to umount {} in {} ms because of EBUSY, try again with lazy umount", + path.display(), + Instant::now().duration_since(start_time).as_millis()); + return umount2(path, true).map_err(|e| Error::Umount(path.to_owned(), e)); + } + } + _ => return Err(Error::Umount(path.to_owned(), e)), + } + } + + info!( + sl!(), + "umount {} in {} ms", + path.display(), + Instant::now().duration_since(start_time).as_millis() + ); + } + + Ok(()) +} + +/// Umount all filesystems mounted at the `mountpoint`. +/// +/// If `mountpoint` is empty or doesn't exist, `umount_all()` is a noop. Otherwise it will try to +/// unmount all filesystems mounted at `mountpoint` repeatedly. For example: +/// - bind mount /dev/sda to /tmp/mnt +/// - bind mount /tmp/b to /tmp/mnt +/// - umount_all("tmp/mnt") will umount both /tmp/b and /dev/sda +/// +/// # Safety +/// Caller needs to ensure safety of the `path` to avoid possible file path based attacks. +pub fn umount_all>(mountpoint: P, lazy_umount: bool) -> Result<()> { + if mountpoint.as_ref().is_empty() || !mountpoint.as_ref().exists() { + return Ok(()); + } + + loop { + match umount2(mountpoint.as_ref(), lazy_umount) { + Err(e) => { + // EINVAL is returned if the target is not a mount point, indicating that we are + // done. It can also indicate a few other things (such as invalid flags) which we + // unfortunately end up squelching here too. + if e.kind() == io::ErrorKind::InvalidInput { + break; + } else { + return Err(Error::Umount(mountpoint.as_ref().to_path_buf(), e)); + } + } + Ok(()) => (), + } + } + + Ok(()) +} + +// Counterpart of nix::umount2, with support of `UMOUNT_FOLLOW`. +fn umount2>(path: P, lazy_umount: bool) -> std::io::Result<()> { + let path_ptr = path.as_ref().as_os_str().as_bytes().as_ptr() as *const c_char; + let mut flags = UMOUNT_NOFOLLOW; + if lazy_umount { + flags |= MntFlags::MNT_DETACH.bits(); + } + + // Safe because parameter is valid and we have checked the reuslt. + if unsafe { libc::umount2(path_ptr, flags) } < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_get_linux_mount_info() { + let info = get_linux_mount_info("/sys/fs/cgroup").unwrap(); + + assert_eq!(&info.device, "tmpfs"); + assert_eq!(&info.fs_type, "tmpfs"); + assert_eq!(&info.path, "/sys/fs/cgroup"); + + assert!(matches!( + get_linux_mount_info(""), + Err(Error::NoMountEntry(_)) + )); + assert!(matches!( + get_linux_mount_info("/sys/fs/cgroup/do_not_exist/____hi"), + Err(Error::NoMountEntry(_)) + )); + } + + #[test] + fn test_create_mount_destination() { + let tmpdir = tempfile::tempdir().unwrap(); + let src = Path::new("/proc/mounts"); + let mut dst = tmpdir.path().to_owned(); + dst.push("proc"); + dst.push("mounts"); + let dst = create_mount_destination(src, dst.as_path(), tmpdir.path(), "bind").unwrap(); + let abs_dst = dst.as_ref().canonicalize().unwrap(); + assert!(abs_dst.is_file()); + + let dst = Path::new("/"); + assert!(matches!( + create_mount_destination(src, dst, "/", "bind"), + Err(Error::InvalidPath(_)) + )); + + let src = Path::new("/proc"); + let dst = Path::new("/proc/mounts"); + assert!(matches!( + create_mount_destination(src, dst, "/", "bind"), + Err(Error::InvalidPath(_)) + )); + } + + #[test] + #[ignore] + fn test_bind_remount_read_only() { + let tmpdir = tempfile::tempdir().unwrap(); + let tmpdir2 = tempfile::tempdir().unwrap(); + + assert!(matches!( + bind_remount_read_only(&PathBuf::from("")), + Err(Error::NullMountPointPath) + )); + assert!(matches!( + bind_remount_read_only(&PathBuf::from("../______doesn't____exist____nnn")), + Err(Error::InvalidPath(_)) + )); + + bind_mount_unchecked(tmpdir2.path(), tmpdir.path(), true).unwrap(); + bind_remount_read_only(tmpdir.path()).unwrap(); + umount_timeout(tmpdir.path().to_str().unwrap(), 0).unwrap(); + } + + #[test] + #[ignore] + fn test_bind_mount() { + let tmpdir = tempfile::tempdir().unwrap(); + let tmpdir2 = tempfile::tempdir().unwrap(); + let mut src = tmpdir.path().to_owned(); + src.push("src"); + let mut dst = tmpdir.path().to_owned(); + dst.push("src"); + + assert!(matches!( + bind_mount_unchecked(Path::new(""), Path::new(""), false), + Err(Error::NullMountPointPath) + )); + assert!(matches!( + bind_mount_unchecked(tmpdir2.path(), Path::new(""), false), + Err(Error::NullMountPointPath) + )); + assert!(matches!( + bind_mount_unchecked( + Path::new("/_does_not_exist_/___aahhhh"), + Path::new("/tmp/_does_not_exist/___bbb"), + false + ), + Err(Error::InvalidPath(_)) + )); + + let dst = create_mount_destination(tmpdir2.path(), &dst, tmpdir.path(), "bind").unwrap(); + bind_mount_unchecked(tmpdir2.path(), dst.as_ref(), true).unwrap(); + bind_mount_unchecked(&src, dst.as_ref(), false).unwrap(); + umount_all(dst.as_ref(), false).unwrap(); + + let mut src = tmpdir.path().to_owned(); + src.push("file"); + fs::write(&src, "test").unwrap(); + let mut dst = tmpdir.path().to_owned(); + dst.push("file"); + let dst = create_mount_destination(&src, &dst, tmpdir.path(), "bind").unwrap(); + bind_mount_unchecked(&src, dst.as_ref(), false).unwrap(); + assert!(dst.as_ref().is_file()); + umount_timeout(dst.as_ref(), 0).unwrap(); + } + + #[test] + fn test_compact_overlay_lowerdirs() { + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + "lowerdir=/a/b/c/xxxx/1l:/a/b/c/xxxx/2l:/a/b/c/xxxx/3l:/a/b/c/xxxx/4l".to_string(), + ]; + let (prefix, n_options) = compact_lowerdir_option(&options); + assert_eq!(&prefix.unwrap(), Path::new("/a/b/c/xxxx/")); + assert_eq!(n_options.len(), 3); + assert_eq!(n_options[2], "lowerdir=1l:2l:3l:4l"); + + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + "lowerdir=/a/b/c/xxxx:/a/b/c/xxxx/2l:/a/b/c/xxxx/3l:/a/b/c/xxxx/4l".to_string(), + ]; + let (prefix, n_options) = compact_lowerdir_option(&options); + assert_eq!(&prefix.unwrap(), Path::new("/a/b/c/")); + assert_eq!(n_options.len(), 3); + assert_eq!(n_options[2], "lowerdir=xxxx:xxxx/2l:xxxx/3l:xxxx/4l"); + + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + "lowerdir=/1l:/2l:/3l:/4l".to_string(), + ]; + let (prefix, n_options) = compact_lowerdir_option(&options); + assert!(prefix.is_none()); + assert_eq!(n_options, options); + + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + ]; + let (prefix, n_options) = compact_lowerdir_option(&options); + assert!(prefix.is_none()); + assert_eq!(n_options, options); + + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "lowerdir=".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + ]; + let (prefix, n_options) = compact_lowerdir_option(&options); + assert!(prefix.is_none()); + assert_eq!(n_options, options); + } + + #[test] + fn test_find_overlay_lowerdirs() { + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + "lowerdir=/a/b/c/xxxx/1l:/a/b/c/xxxx/2l:/a/b/c/xxxx/3l:/a/b/c/xxxx/4l".to_string(), + ]; + let lower_expect = vec![ + "/a/b/c/xxxx/1l".to_string(), + "/a/b/c/xxxx/2l".to_string(), + "/a/b/c/xxxx/3l".to_string(), + "/a/b/c/xxxx/4l".to_string(), + ]; + + let (idx, lower) = find_overlay_lowerdirs(&options).unwrap(); + assert_eq!(idx, 2); + assert_eq!(lower, lower_expect); + + let common_prefix = get_longest_common_prefix(&lower).unwrap(); + assert_eq!(Path::new("/a/b/c/xxxx/"), &common_prefix); + + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + ]; + let v = find_overlay_lowerdirs(&options); + assert!(v.is_none()); + + let options = vec![ + "workdir=/a/b/c/xxxx/workdir".to_string(), + "lowerdir=".to_string(), + "upperdir=/a/b/c/xxxx/upper".to_string(), + ]; + find_overlay_lowerdirs(&options); + assert!(v.is_none()); + } + + #[test] + fn test_get_common_prefix() { + let lower1 = vec![ + "/a/b/c/xxxx/1l/fs".to_string(), + "/a/b/c/////xxxx/11l/fs".to_string(), + "/a/b/c/././xxxx/13l/fs".to_string(), + "/a/b/c/.////xxxx/14l/fs".to_string(), + ]; + let common_prefix = get_longest_common_prefix(&lower1).unwrap(); + assert_eq!(Path::new("/a/b/c/xxxx/"), &common_prefix); + + let lower2 = vec![ + "/fs".to_string(), + "/s".to_string(), + "/sa".to_string(), + "/s".to_string(), + ]; + let common_prefix = get_longest_common_prefix(&lower2).unwrap(); + assert_eq!(Path::new("/"), &common_prefix); + + let lower3 = vec!["".to_string(), "".to_string()]; + let common_prefix = get_longest_common_prefix(&lower3); + assert!(common_prefix.is_none()); + + let lower = vec!["/".to_string(), "/".to_string()]; + let common_prefix = get_longest_common_prefix(&lower); + assert!(common_prefix.is_none()); + + let lower = vec![ + "/a/b/c".to_string(), + "/a/b/c/d".to_string(), + "/a/b///c".to_string(), + ]; + let common_prefix = get_longest_common_prefix(&lower).unwrap(); + assert_eq!(Path::new("/a/b"), &common_prefix); + + let lower = vec!["a/b/c/e".to_string(), "a/b/c/d".to_string()]; + let common_prefix = get_longest_common_prefix(&lower).unwrap(); + assert_eq!(Path::new("a/b/c"), &common_prefix); + + let lower = vec!["a/b/c".to_string(), "a/b/c/d".to_string()]; + let common_prefix = get_longest_common_prefix(&lower).unwrap(); + assert_eq!(Path::new("a/b"), &common_prefix); + + let lower = vec!["/test".to_string()]; + let common_prefix = get_longest_common_prefix(&lower).unwrap(); + assert_eq!(Path::new("/"), &common_prefix); + + let lower = vec![]; + let common_prefix = get_longest_common_prefix(&lower); + assert!(&common_prefix.is_none()); + } + + #[test] + fn test_parse_mount_options() { + let options = vec![]; + let (flags, data) = parse_mount_options(&options).unwrap(); + assert!(flags.is_empty()); + assert!(data.is_empty()); + + let mut options = vec![ + "dev".to_string(), + "ro".to_string(), + "defaults".to_string(), + "data-option".to_string(), + ]; + let (flags, data) = parse_mount_options(&options).unwrap(); + assert_eq!(flags, MsFlags::MS_RDONLY); + assert_eq!(&data, "data-option"); + + options.push("loop".to_string()); + assert!(parse_mount_options(&options).is_err()); + + let idx = options.len() - 1; + options[idx] = " ".repeat(4097); + assert!(parse_mount_options(&options).is_err()); + } + + #[test] + #[ignore] + fn test_mount_at() { + let tmpdir = tempfile::tempdir().unwrap(); + let path = tmpdir.path().to_path_buf(); + mount_at( + Some(path.clone()), + "/___does_not_exist____a___", + PathBuf::from("/tmp/etc/host.conf"), + "", + MsFlags::empty(), + "", + ) + .unwrap_err(); + + mount_at( + Some(PathBuf::from("/___does_not_exist____a___")), + "/etc/host.conf", + PathBuf::from("/tmp/etc/host.conf"), + "", + MsFlags::empty(), + "", + ) + .unwrap_err(); + + let src = path.join("src"); + fs::write(&src, "test").unwrap(); + let dst = path.join("dst"); + fs::write(&dst, "test1").unwrap(); + mount_at( + Some(path), + "src", + PathBuf::from("dst"), + "bind", + MsFlags::MS_BIND, + "", + ) + .unwrap(); + let content = fs::read_to_string(&dst).unwrap(); + assert_eq!(&content, "test"); + } +} From 87887026f60d6c7ebe2629ce18a7fc591e37e30f Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Thu, 9 Dec 2021 16:22:22 +0800 Subject: [PATCH 0028/1953] libs/sys-util: add utilities to manipulate cgroup Add utilities to manipulate cgroup, currently only v1 is supported. Signed-off-by: Liu Jiang Signed-off-by: He Rongguang Signed-off-by: Jiahuan Chao Signed-off-by: Qingyuan Hou Signed-off-by: Quanwei Zhou Signed-off-by: Tim Zhang --- src/libs/Cargo.lock | 97 ++++ src/libs/kata-sys-util/Cargo.toml | 5 + src/libs/kata-sys-util/src/cgroup.rs | 735 +++++++++++++++++++++++++++ src/libs/kata-sys-util/src/lib.rs | 1 + src/libs/kata-types/src/cpu.rs | 70 +++ src/libs/kata-types/src/lib.rs | 5 +- 6 files changed, 912 insertions(+), 1 deletion(-) create mode 100644 src/libs/kata-sys-util/src/cgroup.rs create mode 100644 src/libs/kata-types/src/cpu.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 7106c4bc1d47..ac6b2b8def53 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -80,6 +80,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cgroups-rs" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b827f9d9f6c2fff719d25f5d44cbc8d2ef6df1ef00d055c5c14d5dc25529579" +dependencies = [ + "libc", + "log", + "nix 0.23.1", + "regex", +] + [[package]] name = "chrono" version = "0.4.19" @@ -89,6 +101,7 @@ dependencies = [ "libc", "num-integer", "num-traits", + "time", "winapi", ] @@ -338,12 +351,17 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" name = "kata-sys-util" version = "0.1.0" dependencies = [ + "cgroups-rs", + "chrono", "common-path", "fail", "kata-types", "lazy_static", "libc", "nix 0.23.1", + "num_cpus", + "once_cell", + "serial_test", "slog", "slog-scope", "tempfile", @@ -379,6 +397,16 @@ version = "0.2.124" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.16" @@ -524,6 +552,31 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "petgraph" version = "0.5.1" @@ -741,6 +794,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.136" @@ -772,6 +831,28 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d" +dependencies = [ + "lazy_static", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "slab" version = "0.4.6" @@ -819,6 +900,12 @@ dependencies = [ "slog", ] +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + [[package]] name = "socket2" version = "0.4.4" @@ -889,6 +976,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "tokio" version = "1.17.0" diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 264849468c3d..f56ff9627b4d 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -11,11 +11,14 @@ license = "Apache-2.0" edition = "2018" [dependencies] +cgroups = { package = "cgroups-rs", version = "0.2.7" } +chrono = "0.4.0" common-path = "=1.0.0" fail = "0.5.0" lazy_static = "1.4.0" libc = "0.2.100" nix = "0.23.0" +once_cell = "1.9.0" slog = "2.5.2" slog-scope = "4.4.0" thiserror = "1.0.30" @@ -23,4 +26,6 @@ thiserror = "1.0.30" kata-types = { path = "../kata-types" } [dev-dependencies] +num_cpus = "1.13.1" +serial_test = "0.5.1" tempfile = "3.2.0" diff --git a/src/libs/kata-sys-util/src/cgroup.rs b/src/libs/kata-sys-util/src/cgroup.rs new file mode 100644 index 000000000000..3edcf98245ab --- /dev/null +++ b/src/libs/kata-sys-util/src/cgroup.rs @@ -0,0 +1,735 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::ops::Deref; +use std::path::{Component, Path, PathBuf}; +use std::sync::Mutex; + +use cgroups::{Cgroup, CgroupPid, Controllers, Hierarchy, Subsystem}; +use lazy_static::lazy_static; +use once_cell::sync::Lazy; + +use crate::sl; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Can not add tgid {0} to cgroup, {1:?}")] + AddTgid(u64, #[source] cgroups::error::Error), + #[error("failed to apply resources to cgroup: {0:?}")] + ApplyResource(#[source] cgroups::error::Error), + #[error("failed to delete cgroup after {0} retries")] + DeleteCgroup(u64), + #[error("Invalid cgroup path {0}")] + InvalidCgroupPath(String), +} + +pub type Result = std::result::Result; + +lazy_static! { + /// Disable cgroup v1 subsystems. + pub static ref DISABLED_HIERARCHIES: Mutex> = Mutex::new(Vec::new()); +} + +/// Update the disabled cgroup subsystems. +/// +/// Some cgroup controllers may be disabled by runtime configuration file. The sandbox may call +/// this method to disable those cgroup controllers once. +pub fn update_disabled_cgroup_list(hierarchies: &[String]) { + let mut disabled_hierarchies = DISABLED_HIERARCHIES.lock().unwrap(); + disabled_hierarchies.clear(); + for hierarchy in hierarchies { + //disabled_hierarchies.push(hie.clone()); + match hierarchy.as_str() { + "blkio" => disabled_hierarchies.push(cgroups::Controllers::BlkIo), + "cpu" => disabled_hierarchies.push(cgroups::Controllers::Cpu), + "cpuset" => disabled_hierarchies.push(cgroups::Controllers::CpuSet), + "cpuacct" => disabled_hierarchies.push(cgroups::Controllers::CpuAcct), + "devices" => disabled_hierarchies.push(cgroups::Controllers::Devices), + "freezer" => disabled_hierarchies.push(cgroups::Controllers::Freezer), + "hugetlb" => disabled_hierarchies.push(cgroups::Controllers::HugeTlb), + "memory" => disabled_hierarchies.push(cgroups::Controllers::Mem), + "net_cls" => disabled_hierarchies.push(cgroups::Controllers::NetCls), + "net_prio" => disabled_hierarchies.push(cgroups::Controllers::NetPrio), + "perf_event" => disabled_hierarchies.push(cgroups::Controllers::PerfEvent), + "pids" => disabled_hierarchies.push(cgroups::Controllers::Pids), + "systemd" => disabled_hierarchies.push(cgroups::Controllers::Systemd), + _ => warn!(sl!(), "unknown cgroup controller {}", hierarchy), + } + } + debug!( + sl!(), + "disable cgroup list {:?} from {:?}", disabled_hierarchies, hierarchies + ); +} + +/// Filter out disabled cgroup subsystems. +pub fn filter_disabled_cgroup(controllers: &mut Vec) { + let disabled_hierarchies = DISABLED_HIERARCHIES.lock().unwrap(); + controllers.retain(|x| !disabled_hierarchies.contains(x)); +} + +#[derive(Copy, Clone, Debug)] +pub enum PidType { + /// Add pid to `tasks` + Tasks, + /// Add pid to `cgroup.procs` + CgroupProcs, +} + +/// Get the singleton instance for cgroup v1 hierarchy object. +pub fn get_cgroup_hierarchies() -> &'static cgroups::hierarchies::V1 { + static GLOBAL: Lazy = Lazy::new(cgroups::hierarchies::V1::new); + GLOBAL.deref() +} + +// Prepend a kata specific string to oci cgroup path to form a different cgroup path, thus cAdvisor +// couldn't find kata containers cgroup path on host to prevent it from grabbing the stats data. +const CGROUP_KATA_PREFIX: &str = "kata"; + +/// Convert to a Kata specific cgroup path. +pub fn gen_kata_cgroup_path(path: &str) -> PathBuf { + // Be careful to trim off the possible '/' prefix. Joining an absolute path to a `Path` object + // will replace the old `Path` instead of concat. + Path::new(CGROUP_KATA_PREFIX).join(path.trim_start_matches('/')) +} + +/// Convert to a cgroup path for K8S sandbox. +pub fn gen_sandbox_cgroup_path(path: &str) -> PathBuf { + PathBuf::from(path) +} + +/// A customized cgroup v1 hierarchy object with configurable filters for supported subsystems. +#[derive(Debug)] +pub struct V1Customized { + mount_point: PathBuf, + controllers: Vec, +} + +impl V1Customized { + /// Create a new instance of [`V1Customized`]. + /// + /// The `controllers` configures the subsystems to enable. + /// + /// Note : + /// 1. When enabling both blkio and memory cgroups, blkio cgroup must be enabled before memory + /// cgroup due to a limitation in writeback control of blkio cgroup. + /// 2. cpu, cpuset, cpuacct should be adjacent to each other. + pub fn new(controllers: Vec) -> Self { + let mount_point = get_cgroup_hierarchies().root(); + + V1Customized { + mount_point, + controllers, + } + } +} + +impl Hierarchy for V1Customized { + fn subsystems(&self) -> Vec { + let subsystems = get_cgroup_hierarchies().subsystems(); + + subsystems + .into_iter() + .filter(|sub| { + self.controllers + .contains(&sub.to_controller().control_type()) + }) + .collect::>() + } + + fn root(&self) -> PathBuf { + self.mount_point.clone() + } + + fn root_control_group(&self) -> Cgroup { + Cgroup::load(Box::new(V1Customized::new(self.controllers.clone())), "") + } + + fn v2(&self) -> bool { + false + } +} + +/// An boxed cgroup hierarchy object. +pub type BoxedHierarchyObject = Box; + +/// Create a cgroup hierarchy object with all subsystems disabled. +pub fn get_empty_hierarchy() -> BoxedHierarchyObject { + Box::new(V1Customized::new(vec![])) +} + +/// Create a cgroup hierarchy object for pod sandbox. +pub fn get_sandbox_hierarchy(no_mem: bool) -> BoxedHierarchyObject { + let mut controllers = vec![ + cgroups::Controllers::BlkIo, + cgroups::Controllers::Cpu, + cgroups::Controllers::CpuSet, + cgroups::Controllers::CpuAcct, + cgroups::Controllers::PerfEvent, + ]; + + if !no_mem { + controllers.push(cgroups::Controllers::Mem); + } + filter_disabled_cgroup(&mut controllers); + Box::new(V1Customized::new(controllers)) +} + +/// Create a cgroup hierarchy object with mem subsystem. +/// +/// Note: the mem subsystem may have been disabled, so it will get filtered out. +pub fn get_mem_hierarchy() -> BoxedHierarchyObject { + let mut controllers = vec![cgroups::Controllers::Mem]; + filter_disabled_cgroup(&mut controllers); + Box::new(V1Customized::new(controllers)) +} + +/// Create a cgroup hierarchy object with CPU related subsystems. +/// +/// Note: the mem subsystem may have been disabled, so it will get filtered out. +pub fn get_cpu_hierarchy() -> BoxedHierarchyObject { + let mut controllers = vec![ + cgroups::Controllers::Cpu, + cgroups::Controllers::CpuSet, + cgroups::Controllers::CpuAcct, + ]; + filter_disabled_cgroup(&mut controllers); + Box::new(V1Customized::new(controllers)) +} + +/// Get cgroup hierarchy object from `path`. +pub fn get_hierarchy_by_path(path: &str) -> Result { + let v1 = get_cgroup_hierarchies().clone(); + let valid_path = valid_cgroup_path(path)?; + let cg = cgroups::Cgroup::load(Box::new(v1), valid_path.as_str()); + + let mut hierarchy = vec![]; + for subsys in cg.subsystems() { + let controller = subsys.to_controller(); + if controller.exists() { + hierarchy.push(controller.control_type()); + } + } + + Ok(Box::new(V1Customized::new(hierarchy))) +} + +/// Create or load a cgroup object from a path. +pub fn create_or_load_cgroup(path: &str) -> Result { + let hie = Box::new(get_cgroup_hierarchies().clone()); + + create_or_load_cgroup_with_hier(hie, path) +} + +/// Create or load a cgroup v1 object from a path, with a given hierarchy object. +pub fn create_or_load_cgroup_with_hier(hie: BoxedHierarchyObject, path: &str) -> Result { + let valid_path = valid_cgroup_path(path)?; + if is_cgroup_exist(valid_path.as_str()) { + Ok(cgroups::Cgroup::load(hie, valid_path.as_str())) + } else { + Ok(cgroups::Cgroup::new(hie, valid_path.as_str())) + } +} + +/// Check whether `path` hosts a cgroup hierarchy directory. +pub fn is_cgroup_exist(path: &str) -> bool { + let valid_path = match valid_cgroup_path(path) { + Ok(v) => v, + Err(e) => { + warn!(sl!(), "{}", e); + return false; + } + }; + + let v1 = get_cgroup_hierarchies().clone(); + let cg = cgroups::Cgroup::load(Box::new(v1), valid_path.as_str()); + for subsys in cg.subsystems() { + if subsys.to_controller().exists() { + debug!(sl!(), "cgroup {} exist", path); + return true; + } + } + + false +} + +// Validate the cgroup path is a relative path, do not include ".", "..". +fn valid_cgroup_path(path: &str) -> Result { + let path = path.trim_start_matches('/').to_string(); + + for comp in Path::new(&path).components() { + if !matches!(comp, Component::Normal(_)) { + return Err(Error::InvalidCgroupPath(path.to_string())); + } + } + + Ok(path) +} + +/// Remove all task from cgroup and delete the cgroup. +pub fn force_delete_cgroup(cg: cgroups::Cgroup) -> Result<()> { + delete_cgroup_with_retry(cg, |cg: &Cgroup| { + // if task exist need to delete first. + for cg_pid in cg.tasks() { + warn!(sl!(), "Delete cgroup task pid {}", cg_pid.pid); + cg.remove_task(cg_pid); + } + }) +} + +/// Try to delete a cgroup, call the `do_process` handler at each iteration. +pub fn delete_cgroup_with_retry(cg: Cgroup, mut do_process: F) -> Result<()> +where + F: FnMut(&Cgroup), +{ + // sleep DURATION + const SLEEP_MILLISECS: u64 = 10; + const RETRY_COUNT: u64 = 200; + + // In case of deletion failure caused by "Resource busy", sleep DURATION and retry RETRY times. + for index in 0..RETRY_COUNT { + do_process(&cg); + + if cg.delete().is_ok() { + if index > 0 { + info!( + sl!(), + "cgroup delete cgroup cost {} ms, retry {} times", + index * SLEEP_MILLISECS, + index, + ); + } + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(SLEEP_MILLISECS)) + } + + Err(Error::DeleteCgroup(RETRY_COUNT)) +} + +/// Move the process `pid` into the cgroup `to`. +pub fn move_tgid(pid: u64, to: &Cgroup) -> Result<()> { + info!(sl!(), "try to move tid {:?}", pid); + to.add_task_by_tgid(CgroupPid::from(pid)) + .map_err(|e| Error::AddTgid(pid, e)) +} + +/// Move all processes tasks from `from` to `to`. +pub fn move_cgroup_task(from: &Cgroup, to: &Cgroup) -> Result<()> { + info!(sl!(), "try to move tasks {:?}", from.tasks()); + for cg_pid in from.tasks() { + from.remove_task(CgroupPid::from(cg_pid.pid)); + // TODO: enhance cgroups to implement Copy for CgroupPid + // https://github.com/kata-containers/cgroups-rs/issues/70 + let pid = cg_pid.pid; + to.add_task(cg_pid).map_err(|e| Error::AddTgid(pid, e))?; + } + + Ok(()) +} + +/// Associate a group of tasks with a cgroup, and optionally configure resources for the cgroup. +pub fn update_cgroup_task_resources( + hierarchy: BoxedHierarchyObject, + path: &str, + pids: &[u64], + pid_type: PidType, + resources: Option<&cgroups::Resources>, +) -> Result<()> { + if hierarchy.subsystems().is_empty() { + return Ok(()); + } + fail::fail_point!("update_cgroup_task_resources", |_| { () }); + + let cg = create_or_load_cgroup_with_hier(hierarchy, path)?; + for pid in pids { + let result = match pid_type { + PidType::Tasks => cg.add_task(CgroupPid { pid: *pid }), + PidType::CgroupProcs => cg.add_task_by_tgid(CgroupPid { pid: *pid }), + }; + if let Err(err) = result { + return Err(Error::AddTgid(*pid, err)); + } + } + + if let Some(res) = resources { + cg.apply(res).map_err(Error::ApplyResource)?; + } + + debug!( + sl!(), + "update {:?} {:?} resources {:?} for cgroup {}", pid_type, pids, resources, path + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use cgroups::Controllers; + use serial_test::serial; + use std::sync::atomic::{AtomicUsize, Ordering}; + + static GLOBAL_COUNTER: AtomicUsize = AtomicUsize::new(0); + + fn gen_test_path() -> String { + let pid = nix::unistd::getpid().as_raw(); + let index = GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst); + let path = format!("kata-tests-{}-{}", pid, index); + println!("test path {}", path); + path + } + + fn get_hierarchy(controllers: Vec) -> Box { + Box::new(V1Customized::new(controllers)) + } + + #[test] + fn test_v1_customized_cgroup() { + update_disabled_cgroup_list(&[]); + + let c = V1Customized::new(vec![]); + assert_eq!(c.subsystems().len(), 0); + assert!(!c.v2()); + + let c = V1Customized::new(vec![Controllers::Cpu, Controllers::CpuSet]); + assert_eq!(c.subsystems().len(), 2); + assert!(!c.v2()); + } + + #[test] + #[serial] + fn test_filter_disabled_cgroup() { + update_disabled_cgroup_list(&[]); + assert_eq!(DISABLED_HIERARCHIES.lock().unwrap().len(), 0); + + let disabeld = ["perf_event".to_string()]; + update_disabled_cgroup_list(&disabeld); + assert_eq!(DISABLED_HIERARCHIES.lock().unwrap().len(), 1); + assert_eq!( + DISABLED_HIERARCHIES.lock().unwrap()[0], + Controllers::PerfEvent + ); + + let mut subsystems = vec![Controllers::BlkIo, Controllers::PerfEvent, Controllers::Cpu]; + filter_disabled_cgroup(&mut subsystems); + assert_eq!(subsystems.len(), 2); + assert_eq!(subsystems[0], Controllers::BlkIo); + assert_eq!(subsystems[1], Controllers::Cpu); + + let disabeld = ["cpu".to_string(), "cpuset".to_string()]; + update_disabled_cgroup_list(&disabeld); + assert_eq!(DISABLED_HIERARCHIES.lock().unwrap().len(), 2); + + let mut subsystems = vec![Controllers::BlkIo, Controllers::PerfEvent, Controllers::Cpu]; + filter_disabled_cgroup(&mut subsystems); + assert_eq!(subsystems.len(), 2); + assert_eq!(subsystems[0], Controllers::BlkIo); + assert_eq!(subsystems[1], Controllers::PerfEvent); + + update_disabled_cgroup_list(&[]); + } + + #[test] + fn test_create_empty_hierarchy() { + update_disabled_cgroup_list(&[]); + + let controller = get_empty_hierarchy(); + assert_eq!(controller.subsystems().len(), 0); + assert!(!controller.root_control_group().v2()); + } + + #[test] + #[serial] + fn test_create_sandbox_hierarchy() { + update_disabled_cgroup_list(&[]); + + let controller = get_sandbox_hierarchy(true); + assert_eq!(controller.subsystems().len(), 5); + assert!(!controller.root_control_group().v2()); + + let controller = get_sandbox_hierarchy(false); + assert_eq!(controller.subsystems().len(), 6); + assert!(!controller.root_control_group().v2()); + } + + #[test] + #[serial] + fn test_get_hierarchy() { + update_disabled_cgroup_list(&[]); + + let controller = get_mem_hierarchy(); + assert!(!controller.v2()); + assert_eq!(controller.subsystems().len(), 1); + + let controller = get_cpu_hierarchy(); + assert!(!controller.v2()); + assert_eq!(controller.subsystems().len(), 3); + } + + #[test] + #[serial] + fn test_create_cgroup_default() { + update_disabled_cgroup_list(&[]); + // test need root permission + if !nix::unistd::getuid().is_root() { + println!("test need root permission"); + return; + } + + let v1 = Box::new(cgroups::hierarchies::V1::new()); + let test_path = gen_test_path(); + let cg_path = test_path.as_str(); + assert!(!is_cgroup_exist(cg_path)); + + // new cgroup + let cg = cgroups::Cgroup::new(v1, cg_path); + assert!(is_cgroup_exist(cg_path)); + + // add task + let _ = cg.add_task(cgroups::CgroupPid { + pid: nix::unistd::getpid().as_raw() as u64, + }); + + // delete cgroup + force_delete_cgroup(cg).unwrap(); + assert!(!is_cgroup_exist(cg_path)); + } + + #[test] + #[serial] + fn test_create_cgroup_cpus() { + update_disabled_cgroup_list(&[]); + // test need root permission + if !nix::unistd::getuid().is_root() { + println!("test need root permission"); + return; + } + if num_cpus::get() <= 1 { + println!("The unit test is only supported on SMP systems."); + return; + } + + let test_path = gen_test_path(); + let cg_path = test_path.as_str(); + assert!(!is_cgroup_exist(cg_path)); + + // new cgroup + let cgroup = create_or_load_cgroup(cg_path).unwrap(); + let cpus: &cgroups::cpuset::CpuSetController = cgroup.controller_of().unwrap(); + cpus.set_cpus("0-1").unwrap(); + assert!(is_cgroup_exist(cg_path)); + + // current cgroup + let current_cgroup = create_or_load_cgroup(cg_path).unwrap(); + let current_cpus: &cgroups::cpuset::CpuSetController = + current_cgroup.controller_of().unwrap(); + // check value + assert_eq!(cpus.cpuset().cpus, current_cpus.cpuset().cpus); + + // delete cgroup + force_delete_cgroup(cgroup).unwrap(); + assert!(!is_cgroup_exist(cg_path)); + } + + #[test] + #[serial] + fn test_create_cgroup_with_parent() { + update_disabled_cgroup_list(&[]); + // test need root permission + if !nix::unistd::getuid().is_root() { + println!("test need root permission"); + return; + } + if num_cpus::get() <= 1 { + println!("The unit test is only supported on SMP systems."); + return; + } + + let test_path = gen_test_path(); + let cg_path = test_path.as_str(); + assert!(!is_cgroup_exist(cg_path)); + + // new cgroup + let cg = create_or_load_cgroup(cg_path).unwrap(); + let cpus: &cgroups::cpuset::CpuSetController = cg.controller_of().unwrap(); + cpus.set_cpus("0-1").unwrap(); + assert!(is_cgroup_exist(cg_path)); + + // new cgroup 1 + let cg_test_path_1 = format!("{}/vcpu0", test_path); + let cg_path_1 = cg_test_path_1.as_str(); + let cg1 = create_or_load_cgroup(cg_path_1).unwrap(); + let cpus1: &cgroups::cpuset::CpuSetController = cg1.controller_of().unwrap(); + cpus1.set_cpus("0").unwrap(); + assert!(is_cgroup_exist(cg_path_1)); + + // new cgroup 2 + let cg_test_path_2 = format!("{}/vcpu1", test_path); + let cg_path_2 = cg_test_path_2.as_str(); + // new cgroup + let cg2 = create_or_load_cgroup(cg_path_2).unwrap(); + let cpus2: &cgroups::cpuset::CpuSetController = cg2.controller_of().unwrap(); + cpus2.set_cpus("1").unwrap(); + assert!(is_cgroup_exist(cg_path_2)); + + // must delete sub dir first + force_delete_cgroup(cg1).unwrap(); + assert!(!is_cgroup_exist(cg_path_1)); + force_delete_cgroup(cg2).unwrap(); + assert!(!is_cgroup_exist(cg_path_2)); + force_delete_cgroup(cg).unwrap(); + assert!(!is_cgroup_exist(cg_path)); + } + + fn assert_customize_path_exist(path: &str, current_subsystems: &[Subsystem], expect: bool) { + println!("assert customize path {} exist expect {}", path, expect); + let v1 = Box::new(cgroups::hierarchies::V1::new()); + let v1_cg = Cgroup::load(v1, path); + let v1_subsystems = v1_cg.subsystems(); + + for v1_sub in v1_subsystems { + let check_expect = || -> bool { + for current_sub in current_subsystems { + if v1_sub.to_controller().control_type() + == current_sub.to_controller().control_type() + { + return expect; + } + } + false + }(); + assert_eq!( + check_expect, + v1_sub.to_controller().exists(), + "failed to check path {:?} subsystem {:?}", + path, + v1_sub + ) + } + } + + fn clean_cgroup_v1(path: &str) { + let v1 = Box::new(cgroups::hierarchies::V1::new()); + let cg = Cgroup::load(v1.clone(), path); + delete_cgroup_with_retry(cg, |_: &Cgroup| {}).unwrap(); + + let check_cg = Cgroup::load(v1, path); + assert_customize_path_exist(path, check_cg.subsystems(), false); + } + + #[test] + #[serial] + fn test_customize_hierarchies() { + update_disabled_cgroup_list(&[]); + // test need root permission + if !nix::unistd::getuid().is_root() { + println!("test need root permission"); + return; + } + + let cg_path_1 = "test_customize_hierarchies1"; + let cg_path_2 = "test_customize_hierarchies2"; + + // clean + clean_cgroup_v1(cg_path_1); + clean_cgroup_v1(cg_path_2); + + // check customized cgroup + // With some kernels, Cpu and CpuAcct are combined into one directory, so enable both + // to ease test code. + let controllers_1 = vec![Controllers::Cpu, Controllers::CpuAcct]; + let controllers_2 = vec![Controllers::Cpu, Controllers::CpuSet, Controllers::CpuAcct]; + let cg_1 = Cgroup::new(get_hierarchy(controllers_1.clone()), cg_path_1); + let cg_2 = Cgroup::new(get_hierarchy(controllers_2.clone()), cg_path_2); + + assert_customize_path_exist(cg_path_1, cg_1.subsystems(), true); + assert_customize_path_exist(cg_path_2, cg_2.subsystems(), true); + + // delete + let _ = cg_1.delete(); + let _ = cg_2.delete(); + + // check after delete + let check_cg_1 = Cgroup::load(get_hierarchy(controllers_1), cg_path_1); + let check_cg_2 = Cgroup::load(get_hierarchy(controllers_2), cg_path_2); + assert_customize_path_exist(cg_path_1, check_cg_1.subsystems(), false); + assert_customize_path_exist(cg_path_2, check_cg_2.subsystems(), false); + } + + #[test] + #[serial] + fn test_task_move() { + update_disabled_cgroup_list(&[]); + // test need root permission + if !nix::unistd::getuid().is_root() { + println!("test need root permission"); + return; + } + + let cg_path_1 = "test_task_move_before"; + let cg_path_2 = "test_task_move_after"; + + // clean + clean_cgroup_v1(cg_path_1); + clean_cgroup_v1(cg_path_2); + + // With some kernels, Cpu and CpuAcct are combined into one directory, so enable both + // to ease test code. + let controllers = vec![Controllers::Cpu, Controllers::CpuAcct]; + let cg_1 = Cgroup::new(get_hierarchy(controllers.clone()), cg_path_1); + let cg_2 = Cgroup::new(get_hierarchy(controllers.clone()), cg_path_2); + + assert_customize_path_exist(cg_path_1, cg_1.subsystems(), true); + assert_customize_path_exist(cg_path_2, cg_2.subsystems(), true); + + // add task + let pid = libc::pid_t::from(nix::unistd::getpid()) as u64; + let _ = cg_1.add_task(CgroupPid::from(pid)).unwrap(); + let mut cg_task_1 = cg_1.tasks(); + let mut cg_task_2 = cg_2.tasks(); + assert_eq!(1, cg_task_1.len()); + assert_eq!(0, cg_task_2.len()); + + // move task + let _ = cg_2.add_task(CgroupPid::from(pid)).unwrap(); + cg_task_1 = cg_1.tasks(); + cg_task_2 = cg_2.tasks(); + assert_eq!(0, cg_task_1.len()); + assert_eq!(1, cg_task_2.len()); + + cg_2.remove_task(CgroupPid::from(pid)); + + // delete + cg_1.delete().unwrap(); + // delete cg_2 with retry because of possible unknown failed + // caused by "Resource busy", we do the same in the production + // code, so it makes sense in the test. + delete_cgroup_with_retry(cg_2, |_| {}).unwrap(); + + // check after delete + let check_cg_1 = Cgroup::load(get_hierarchy(controllers.clone()), cg_path_1); + let check_cg_2 = Cgroup::load(get_hierarchy(controllers), cg_path_2); + assert_customize_path_exist(cg_path_1, check_cg_1.subsystems(), false); + assert_customize_path_exist(cg_path_2, check_cg_2.subsystems(), false); + } + + #[test] + fn test_gen_kata_cgroup_path() { + assert_eq!( + &gen_kata_cgroup_path("sandbox1/container2"), + Path::new("kata/sandbox1/container2") + ); + assert_eq!( + &gen_kata_cgroup_path("/sandbox1/container2"), + Path::new("kata/sandbox1/container2") + ); + assert_eq!( + &gen_kata_cgroup_path("/sandbox1:container2"), + Path::new("kata/sandbox1:container2") + ); + } +} diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 62706c15de4f..a4786f5f1fe6 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -6,6 +6,7 @@ #[macro_use] extern crate slog; +pub mod cgroup; pub mod fs; pub mod mount; diff --git a/src/libs/kata-types/src/cpu.rs b/src/libs/kata-types/src/cpu.rs new file mode 100644 index 000000000000..b209834b3e2b --- /dev/null +++ b/src/libs/kata-types/src/cpu.rs @@ -0,0 +1,70 @@ +// Copyright (c) 2022 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +/// A list of CPU IDs. +#[derive(Debug)] +pub struct CpuSet(Vec); + +impl CpuSet { + /// Create a new instance of `CpuSet`. + pub fn new() -> Self { + CpuSet(vec![]) + } + + /// Add new CPUs into the set. + pub fn extend(&mut self, cpus: &[u32]) { + self.0.extend_from_slice(cpus); + self.0.sort_unstable(); + self.0.dedup(); + } + + /// Returns true if the CPU set contains elements. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Get number of elements in the CPU set. + pub fn len(&self) -> usize { + self.0.len() + } +} + +impl From> for CpuSet { + fn from(mut cpus: Vec) -> Self { + cpus.sort_unstable(); + cpus.dedup(); + CpuSet(cpus) + } +} + +/// Test whether two CPU sets are equal. +impl PartialEq for CpuSet { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_cpu_list_equal() { + let cpuset1 = CpuSet::from(vec![1, 2, 3]); + let cpuset2 = CpuSet::from(vec![3, 2, 1]); + let cpuset3 = CpuSet::from(vec![]); + let cpuset4 = CpuSet::from(vec![3, 2, 4]); + let cpuset5 = CpuSet::from(vec![1, 2, 3, 3, 2, 1]); + + assert_eq!(cpuset1.len(), 3); + assert!(cpuset3.is_empty()); + assert_eq!(cpuset5.len(), 3); + + assert_eq!(cpuset1, cpuset2); + assert_eq!(cpuset1, cpuset5); + assert_ne!(cpuset1, cpuset3); + assert_ne!(cpuset1, cpuset4); + } +} diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs index c05efd320792..06c8cf295688 100644 --- a/src/libs/kata-types/src/lib.rs +++ b/src/libs/kata-types/src/lib.rs @@ -11,7 +11,7 @@ extern crate slog; #[macro_use] extern crate serde; -/// Constants and data types annotations. +/// Constants and data types related to annotations. pub mod annotations; /// Kata configuration information from configuration file. @@ -20,6 +20,9 @@ pub mod config; /// Constants and data types related to container. pub mod container; +/// Constants and data types related to CPU. +pub mod cpu; + /// Constants and data types related to Kubernetes/kubelet. pub mod k8s; From 1d5c898d7fe84342d30c7ce20754aba94671a5d8 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Fri, 10 Dec 2021 00:23:23 +0800 Subject: [PATCH 0029/1953] libs/sys-util: add utilities to parse NUMA information Add utilities to parse NUMA information. Signed-off-by: Liu Jiang Signed-off-by: Qingyuan Hou Signed-off-by: Simon Guo --- src/libs/kata-sys-util/src/lib.rs | 1 + src/libs/kata-sys-util/src/numa.rs | 221 ++++++++++++++ .../sys/devices/system/cpu/cpu0/node0/cpulist | 1 + .../sys/devices/system/cpu/cpu0/node0/cpumap | 1 + .../sys/devices/system/cpu/cpu1/node0/cpulist | 1 + .../devices/system/cpu/cpu64/node1/cpulist | 1 + .../sys/devices/system/node/node0/cpulist | 1 + .../sys/devices/system/node/node0/cpumap | 1 + .../sys/devices/system/node/node1/cpulist | 1 + .../sys/devices/system/node/node1/cpumap | 1 + src/libs/kata-types/src/cpu.rs | 269 +++++++++++++++--- src/libs/kata-types/src/lib.rs | 10 + src/libs/kata-types/src/utils/mod.rs | 6 + src/libs/kata-types/src/utils/u32_set.rs | 163 +++++++++++ 14 files changed, 636 insertions(+), 42 deletions(-) create mode 100644 src/libs/kata-sys-util/src/numa.rs create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpulist create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpumap create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu1/node0/cpulist create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu64/node1/cpulist create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpulist create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpumap create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpulist create mode 100644 src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpumap create mode 100644 src/libs/kata-types/src/utils/mod.rs create mode 100644 src/libs/kata-types/src/utils/u32_set.rs diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index a4786f5f1fe6..4fa93c8b2d10 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -9,6 +9,7 @@ extern crate slog; pub mod cgroup; pub mod fs; pub mod mount; +pub mod numa; // Convenience macro to obtain the scoped logger #[macro_export] diff --git a/src/libs/kata-sys-util/src/numa.rs b/src/libs/kata-sys-util/src/numa.rs new file mode 100644 index 000000000000..ece5cd8e7f7c --- /dev/null +++ b/src/libs/kata-sys-util/src/numa.rs @@ -0,0 +1,221 @@ +// Copyright (c) 2021 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::collections::HashMap; +use std::fs::DirEntry; +use std::io::Read; +use std::path::PathBuf; + +use kata_types::cpu::CpuSet; +use lazy_static::lazy_static; + +use crate::sl; +use std::str::FromStr; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Invalid CPU number {0}")] + InvalidCpu(u32), + #[error("Invalid node file name {0}")] + InvalidNodeFileName(String), + #[error("Can not read directory {1}: {0}")] + ReadDirectory(#[source] std::io::Error, String), + #[error("Can not read from file {0}, {1:?}")] + ReadFile(String, #[source] std::io::Error), + #[error("Can not open from file {0}, {1:?}")] + OpenFile(String, #[source] std::io::Error), + #[error("Can not parse CPU info, {0:?}")] + ParseCpuInfo(#[from] kata_types::Error), +} + +pub type Result = std::result::Result; + +// global config in UT +#[cfg(test)] +lazy_static! { + static ref SYS_FS_PREFIX: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test/texture"); + // numa node file for UT, we can mock data + static ref NUMA_NODE_PATH: PathBuf = (&*SYS_FS_PREFIX).join("sys/devices/system/node"); + // sysfs directory for CPU devices + static ref NUMA_CPU_PATH: PathBuf = (&*SYS_FS_PREFIX).join("sys/devices/system/cpu"); +} + +// global config in release +#[cfg(not(test))] +lazy_static! { + // numa node file for UT, we can mock data + static ref NUMA_NODE_PATH: PathBuf = PathBuf::from("/sys/devices/system/node"); + // sysfs directory for CPU devices + static ref NUMA_CPU_PATH: PathBuf = PathBuf::from("/sys/devices/system/cpu"); +} + +const NUMA_NODE_PREFIX: &str = "node"; +const NUMA_NODE_CPU_LIST_NAME: &str = "cpulist"; + +/// Get numa node id for a CPU +pub fn get_node_id(cpu: u32) -> Result { + let path = NUMA_CPU_PATH.join(format!("cpu{}", cpu)); + let dirs = path.read_dir().map_err(|_| Error::InvalidCpu(cpu))?; + + for d in dirs { + let d = d.map_err(|e| Error::ReadDirectory(e, path.to_string_lossy().to_string()))?; + if let Some(file_name) = d.file_name().to_str() { + if !file_name.starts_with(NUMA_NODE_PREFIX) { + continue; + } + let index_str = file_name.trim_start_matches(NUMA_NODE_PREFIX); + if let Ok(i) = index_str.parse::() { + return Ok(i); + } + } + } + + // Default to node 0 on UMA systems. + Ok(0) +} + +/// Map cpulist to NUMA node, returns a HashMap>. +pub fn get_node_map(cpus: &str) -> Result>> { + // > + let mut node_map: HashMap> = HashMap::new(); + let cpuset = CpuSet::from_str(cpus)?; + + for c in cpuset.iter() { + let node_id = get_node_id(*c)?; + node_map.entry(node_id).or_insert_with(Vec::new).push(*c); + } + + Ok(node_map) +} + +/// Get CPU to NUMA node mapping by reading `/sys/devices/system/node/nodex/cpulist`. +/// +/// Return a HashMap. The hashmap will be empty if NUMA is not enabled on the +/// system. +pub fn get_numa_nodes() -> Result> { + let mut numa_nodes = HashMap::new(); + let numa_node_path = &*NUMA_NODE_PATH; + if !numa_node_path.exists() { + debug!(sl!(), "no numa node available on this system"); + return Ok(numa_nodes); + } + + let dirs = numa_node_path + .read_dir() + .map_err(|e| Error::ReadDirectory(e, numa_node_path.to_string_lossy().to_string()))?; + for d in dirs { + match d { + Err(e) => { + return Err(Error::ReadDirectory( + e, + numa_node_path.to_string_lossy().to_string(), + )) + } + Ok(d) => { + if let Ok(file_name) = d.file_name().into_string() { + if file_name.starts_with(NUMA_NODE_PREFIX) { + let index_string = file_name.trim_start_matches(NUMA_NODE_PREFIX); + info!( + sl!(), + "get node dir {} node index {}", &file_name, index_string + ); + match index_string.parse::() { + Ok(nid) => read_cpu_info_from_node(&d, nid, &mut numa_nodes)?, + Err(_e) => { + return Err(Error::InvalidNodeFileName(file_name.to_string())) + } + } + } + } + } + } + } + + Ok(numa_nodes) +} + +fn read_cpu_info_from_node( + d: &DirEntry, + node_index: u32, + numa_nodes: &mut HashMap, +) -> Result<()> { + let cpu_list_path = d.path().join(NUMA_NODE_CPU_LIST_NAME); + let mut file = std::fs::File::open(&cpu_list_path) + .map_err(|e| Error::OpenFile(cpu_list_path.to_string_lossy().to_string(), e))?; + let mut cpu_list_string = String::new(); + if let Err(e) = file.read_to_string(&mut cpu_list_string) { + return Err(Error::ReadFile( + cpu_list_path.to_string_lossy().to_string(), + e, + )); + } + let split_cpus = CpuSet::from_str(cpu_list_string.trim())?; + info!( + sl!(), + "node {} list {:?} from {}", node_index, split_cpus, &cpu_list_string + ); + for split_cpu_id in split_cpus.iter() { + numa_nodes.insert(*split_cpu_id, node_index); + } + + Ok(()) +} + +/// Check whether all specified CPUs have associated NUMA node. +pub fn is_valid_numa_cpu(cpus: &[u32]) -> Result { + let numa_nodes = get_numa_nodes()?; + + for cpu in cpus { + if numa_nodes.get(cpu).is_none() { + return Ok(false); + } + } + + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_node_id() { + assert_eq!(get_node_id(0).unwrap(), 0); + assert_eq!(get_node_id(1).unwrap(), 0); + assert_eq!(get_node_id(64).unwrap(), 1); + get_node_id(65).unwrap_err(); + } + + #[test] + fn test_get_node_map() { + let map = get_node_map("0-1,64").unwrap(); + assert_eq!(map.len(), 2); + assert_eq!(map.get(&0).unwrap().len(), 2); + assert_eq!(map.get(&1).unwrap().len(), 1); + + get_node_map("0-1,64,65").unwrap_err(); + } + + #[test] + fn test_get_numa_nodes() { + let map = get_numa_nodes().unwrap(); + assert_eq!(map.len(), 65); + assert_eq!(*map.get(&0).unwrap(), 0); + assert_eq!(*map.get(&1).unwrap(), 0); + assert_eq!(*map.get(&63).unwrap(), 0); + assert_eq!(*map.get(&64).unwrap(), 1); + } + + #[test] + fn test_is_valid_numa_cpu() { + assert!(is_valid_numa_cpu(&[0]).unwrap()); + assert!(is_valid_numa_cpu(&[1]).unwrap()); + assert!(is_valid_numa_cpu(&[63]).unwrap()); + assert!(is_valid_numa_cpu(&[64]).unwrap()); + assert!(is_valid_numa_cpu(&[0, 1, 64]).unwrap()); + assert!(!is_valid_numa_cpu(&[0, 1, 64, 65]).unwrap()); + assert!(!is_valid_numa_cpu(&[65]).unwrap()); + } +} diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpulist b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpulist new file mode 100644 index 000000000000..4cfe9ed52f53 --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpulist @@ -0,0 +1 @@ +0,1-63 diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpumap b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpumap new file mode 100644 index 000000000000..2f3bb0650db2 --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu0/node0/cpumap @@ -0,0 +1 @@ +ffffffff,ffffffff diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu1/node0/cpulist b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu1/node0/cpulist new file mode 100644 index 000000000000..4cfe9ed52f53 --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu1/node0/cpulist @@ -0,0 +1 @@ +0,1-63 diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu64/node1/cpulist b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu64/node1/cpulist new file mode 100644 index 000000000000..900731ffd51f --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/cpu/cpu64/node1/cpulist @@ -0,0 +1 @@ +64 diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpulist b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpulist new file mode 100644 index 000000000000..3498c1f2daa5 --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpulist @@ -0,0 +1 @@ +0-63 diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpumap b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpumap new file mode 100644 index 000000000000..2f3bb0650db2 --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node0/cpumap @@ -0,0 +1 @@ +ffffffff,ffffffff diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpulist b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpulist new file mode 100644 index 000000000000..900731ffd51f --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpulist @@ -0,0 +1 @@ +64 diff --git a/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpumap b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpumap new file mode 100644 index 000000000000..62fe293eb1fd --- /dev/null +++ b/src/libs/kata-sys-util/test/texture/sys/devices/system/node/node1/cpumap @@ -0,0 +1 @@ +1,00000000,00000000 diff --git a/src/libs/kata-types/src/cpu.rs b/src/libs/kata-types/src/cpu.rs index b209834b3e2b..0020de097b26 100644 --- a/src/libs/kata-types/src/cpu.rs +++ b/src/libs/kata-types/src/cpu.rs @@ -3,46 +3,156 @@ // SPDX-License-Identifier: Apache-2.0 // -/// A list of CPU IDs. -#[derive(Debug)] -pub struct CpuSet(Vec); +use std::convert::TryFrom; +use std::str::FromStr; -impl CpuSet { - /// Create a new instance of `CpuSet`. - pub fn new() -> Self { - CpuSet(vec![]) +use oci::LinuxCpu; + +/// A set of CPU ids. +pub type CpuSet = crate::utils::u32_set::U32Set; + +/// A set of NUMA memory nodes. +pub type NumaNodeSet = crate::utils::u32_set::U32Set; + +/// Error code for CPU related operations. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// Invalid CPU list. + #[error("Invalid CPU list: {0}")] + InvalidCpuSet(crate::Error), + /// Invalid NUMA memory node list. + #[error("Invalid NUMA memory node list: {0}")] + InvalidNodeSet(crate::Error), +} + +/// Assigned CPU resources for a Linux container. +#[derive(Default, Debug)] +pub struct LinuxContainerCpuResources { + shares: u64, + period: u64, + quota: i64, + cpuset: CpuSet, + nodeset: NumaNodeSet, + calculated_vcpu_time_ms: Option, +} + +impl LinuxContainerCpuResources { + /// Get the CPU shares. + pub fn shares(&self) -> u64 { + self.shares } - /// Add new CPUs into the set. - pub fn extend(&mut self, cpus: &[u32]) { - self.0.extend_from_slice(cpus); - self.0.sort_unstable(); - self.0.dedup(); + /// Get the CPU schedule period. + pub fn period(&self) -> u64 { + self.period } - /// Returns true if the CPU set contains elements. - pub fn is_empty(&self) -> bool { - self.0.is_empty() + /// Get the CPU schedule quota. + pub fn quota(&self) -> i64 { + self.quota } - /// Get number of elements in the CPU set. - pub fn len(&self) -> usize { - self.0.len() + /// Get the CPU set. + pub fn cpuset(&self) -> &CpuSet { + &self.cpuset + } + + /// Get the NUMA memory node set. + pub fn nodeset(&self) -> &NumaNodeSet { + &self.nodeset + } + + /// Get number of vCPUs to fulfill the CPU resource request, `None` means unconstrained. + pub fn get_vcpus(&self) -> Option { + self.calculated_vcpu_time_ms + .map(|v| v.saturating_add(999) / 1000) } } -impl From> for CpuSet { - fn from(mut cpus: Vec) -> Self { - cpus.sort_unstable(); - cpus.dedup(); - CpuSet(cpus) +impl TryFrom<&LinuxCpu> for LinuxContainerCpuResources { + type Error = Error; + + // Unhandled fields: realtime_runtime, realtime_period, mems + fn try_from(value: &LinuxCpu) -> Result { + let period = value.period.unwrap_or(0); + let quota = value.quota.unwrap_or(-1); + let cpuset = CpuSet::from_str(&value.cpus).map_err(Error::InvalidCpuSet)?; + let nodeset = NumaNodeSet::from_str(&value.mems).map_err(Error::InvalidNodeSet)?; + + // If quota is -1, it means the CPU resource request is unconstrained. In that case, + // we don't currently assign additional CPUs. + let milli_sec = if quota >= 0 && period != 0 { + Some((quota as u64).saturating_mul(1000) / period) + } else { + None + }; + + Ok(LinuxContainerCpuResources { + shares: value.shares.unwrap_or(0), + period, + quota, + cpuset, + nodeset, + calculated_vcpu_time_ms: milli_sec, + }) } } -/// Test whether two CPU sets are equal. -impl PartialEq for CpuSet { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 +/// Assigned CPU resources for a Linux sandbox/pod. +#[derive(Default, Debug)] +pub struct LinuxSandboxCpuResources { + shares: u64, + calculated_vcpu_time_ms: u64, + cpuset: CpuSet, + nodeset: NumaNodeSet, +} + +impl LinuxSandboxCpuResources { + /// Create a new instance of `LinuxSandboxCpuResources`. + pub fn new(shares: u64) -> Self { + Self { + shares, + ..Default::default() + } + } + + /// Get the CPU shares. + pub fn shares(&self) -> u64 { + self.shares + } + + /// Get assigned vCPU time in ms. + pub fn calculated_vcpu_time_ms(&self) -> u64 { + self.calculated_vcpu_time_ms + } + + /// Get the CPU set. + pub fn cpuset(&self) -> &CpuSet { + &self.cpuset + } + + /// Get the NUMA memory node set. + pub fn nodeset(&self) -> &NumaNodeSet { + &self.nodeset + } + + /// Get number of vCPUs to fulfill the CPU resource request. + pub fn get_vcpus(&self) -> u64 { + if self.calculated_vcpu_time_ms == 0 && !self.cpuset.is_empty() { + self.cpuset.len() as u64 + } else { + self.calculated_vcpu_time_ms.saturating_add(999) / 1000 + } + } + + /// Merge resources assigned to a container into the sandbox/pod resources. + pub fn merge(&mut self, container_resource: &LinuxContainerCpuResources) -> &mut Self { + if let Some(v) = container_resource.calculated_vcpu_time_ms.as_ref() { + self.calculated_vcpu_time_ms += v; + } + self.cpuset.extend(&container_resource.cpuset); + self.nodeset.extend(&container_resource.nodeset); + self } } @@ -51,20 +161,95 @@ mod tests { use super::*; #[test] - fn test_is_cpu_list_equal() { - let cpuset1 = CpuSet::from(vec![1, 2, 3]); - let cpuset2 = CpuSet::from(vec![3, 2, 1]); - let cpuset3 = CpuSet::from(vec![]); - let cpuset4 = CpuSet::from(vec![3, 2, 4]); - let cpuset5 = CpuSet::from(vec![1, 2, 3, 3, 2, 1]); - - assert_eq!(cpuset1.len(), 3); - assert!(cpuset3.is_empty()); - assert_eq!(cpuset5.len(), 3); - - assert_eq!(cpuset1, cpuset2); - assert_eq!(cpuset1, cpuset5); - assert_ne!(cpuset1, cpuset3); - assert_ne!(cpuset1, cpuset4); + fn test_linux_container_cpu_resources() { + let resources = LinuxContainerCpuResources::default(); + + assert_eq!(resources.shares(), 0); + assert_eq!(resources.calculated_vcpu_time_ms, None); + assert!(resources.cpuset.is_empty()); + assert!(resources.nodeset.is_empty()); + assert!(resources.calculated_vcpu_time_ms.is_none()); + + let oci = oci::LinuxCpu { + shares: Some(2048), + quota: Some(1001), + period: Some(100), + realtime_runtime: None, + realtime_period: None, + cpus: "1,2,3".to_string(), + mems: "1".to_string(), + }; + let resources = LinuxContainerCpuResources::try_from(&oci).unwrap(); + assert_eq!(resources.shares(), 2048); + assert_eq!(resources.period(), 100); + assert_eq!(resources.quota(), 1001); + assert_eq!(resources.calculated_vcpu_time_ms, Some(10010)); + assert_eq!(resources.get_vcpus().unwrap(), 11); + assert_eq!(resources.cpuset().len(), 3); + assert_eq!(resources.nodeset().len(), 1); + + let oci = oci::LinuxCpu { + shares: Some(2048), + quota: None, + period: None, + realtime_runtime: None, + realtime_period: None, + cpus: "1".to_string(), + mems: "1-2".to_string(), + }; + let resources = LinuxContainerCpuResources::try_from(&oci).unwrap(); + assert_eq!(resources.shares(), 2048); + assert_eq!(resources.period(), 0); + assert_eq!(resources.quota(), -1); + assert_eq!(resources.calculated_vcpu_time_ms, None); + assert!(resources.get_vcpus().is_none()); + assert_eq!(resources.cpuset().len(), 1); + assert_eq!(resources.nodeset().len(), 2); + } + + #[test] + fn test_linux_sandbox_cpu_resources() { + let mut sandbox = LinuxSandboxCpuResources::new(1024); + + assert_eq!(sandbox.shares(), 1024); + assert_eq!(sandbox.get_vcpus(), 0); + assert_eq!(sandbox.calculated_vcpu_time_ms(), 0); + assert!(sandbox.cpuset().is_empty()); + assert!(sandbox.nodeset().is_empty()); + + let oci = oci::LinuxCpu { + shares: Some(2048), + quota: Some(1001), + period: Some(100), + realtime_runtime: None, + realtime_period: None, + cpus: "1,2,3".to_string(), + mems: "1".to_string(), + }; + let resources = LinuxContainerCpuResources::try_from(&oci).unwrap(); + sandbox.merge(&resources); + assert_eq!(sandbox.shares(), 1024); + assert_eq!(sandbox.get_vcpus(), 11); + assert_eq!(sandbox.calculated_vcpu_time_ms(), 10010); + assert_eq!(sandbox.cpuset().len(), 3); + assert_eq!(sandbox.nodeset().len(), 1); + + let oci = oci::LinuxCpu { + shares: Some(2048), + quota: None, + period: None, + realtime_runtime: None, + realtime_period: None, + cpus: "1,4".to_string(), + mems: "1-2".to_string(), + }; + let resources = LinuxContainerCpuResources::try_from(&oci).unwrap(); + sandbox.merge(&resources); + + assert_eq!(sandbox.shares(), 1024); + assert_eq!(sandbox.get_vcpus(), 11); + assert_eq!(sandbox.calculated_vcpu_time_ms(), 10010); + assert_eq!(sandbox.cpuset().len(), 4); + assert_eq!(sandbox.nodeset().len(), 2); } } diff --git a/src/libs/kata-types/src/lib.rs b/src/libs/kata-types/src/lib.rs index 06c8cf295688..ce43d29607d8 100644 --- a/src/libs/kata-types/src/lib.rs +++ b/src/libs/kata-types/src/lib.rs @@ -29,6 +29,16 @@ pub mod k8s; /// Constants and data types related to mount point. pub mod mount; +pub(crate) mod utils; + +/// Common error codes. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// Invalid configuration list. + #[error("invalid list {0}")] + InvalidList(String), +} + /// Convenience macro to obtain the scoped logger #[macro_export] macro_rules! sl { diff --git a/src/libs/kata-types/src/utils/mod.rs b/src/libs/kata-types/src/utils/mod.rs new file mode 100644 index 000000000000..abcb4c227715 --- /dev/null +++ b/src/libs/kata-types/src/utils/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) 2022 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub(crate) mod u32_set; diff --git a/src/libs/kata-types/src/utils/u32_set.rs b/src/libs/kata-types/src/utils/u32_set.rs new file mode 100644 index 000000000000..3742e4d54fb6 --- /dev/null +++ b/src/libs/kata-types/src/utils/u32_set.rs @@ -0,0 +1,163 @@ +// Copyright (c) 2022 Alibaba Cloud +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::ops::Deref; +use std::slice::Iter; +use std::str::FromStr; + +use crate::Error; + +/// A set of unique `u32` IDs. +/// +/// The `U32Set` may be used to save CPUs parsed from a CPU list file or NUMA nodes parsed from +/// a NUMA node list file. +#[derive(Default, Debug)] +pub struct U32Set(Vec); + +impl U32Set { + /// Create a new instance of `U32Set`. + pub fn new() -> Self { + U32Set(vec![]) + } + + /// Add the `cpu` to the CPU set. + pub fn add(&mut self, cpu: u32) { + self.0.push(cpu); + self.0.sort_unstable(); + self.0.dedup(); + } + + /// Add new CPUs into the set. + pub fn extend(&mut self, cpus: &[u32]) { + self.0.extend_from_slice(cpus); + self.0.sort_unstable(); + self.0.dedup(); + } + + /// Returns true if the CPU set contains elements. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Get number of elements in the CPU set. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Get an iterator over the CPU set. + pub fn iter(&self) -> Iter { + self.0.iter() + } +} + +impl From> for U32Set { + fn from(mut cpus: Vec) -> Self { + cpus.sort_unstable(); + cpus.dedup(); + U32Set(cpus) + } +} + +impl FromStr for U32Set { + type Err = Error; + + fn from_str(cpus_str: &str) -> Result { + if cpus_str.is_empty() { + return Ok(U32Set::new()); + } + + let mut cpus = Vec::new(); + for split_cpu in cpus_str.split(',') { + if !split_cpu.contains('-') { + if !split_cpu.is_empty() { + if let Ok(cpu_id) = split_cpu.parse::() { + cpus.push(cpu_id); + continue; + } + } + } else { + let fields: Vec<&str> = split_cpu.split('-').collect(); + if fields.len() == 2 { + if let Ok(start) = fields[0].parse::() { + if let Ok(end) = fields[1].parse::() { + if start < end { + for cpu in start..=end { + cpus.push(cpu); + } + continue; + } + } + } + } + } + + return Err(Error::InvalidList(cpus_str.to_string())); + } + + Ok(U32Set::from(cpus)) + } +} + +impl Deref for U32Set { + type Target = [u32]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Test whether two CPU sets are equal. +impl PartialEq for U32Set { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_cpuset_equal() { + let cpuset1 = U32Set::from(vec![1, 2, 3]); + let cpuset2 = U32Set::from(vec![3, 2, 1]); + let cpuset3 = U32Set::from(vec![]); + let cpuset4 = U32Set::from(vec![3, 2, 4]); + let cpuset5 = U32Set::from(vec![1, 2, 3, 3, 2, 1]); + + assert_eq!(cpuset1.len(), 3); + assert!(cpuset3.is_empty()); + assert_eq!(cpuset5.len(), 3); + + assert_eq!(cpuset1, cpuset2); + assert_eq!(cpuset1, cpuset5); + assert_ne!(cpuset1, cpuset3); + assert_ne!(cpuset1, cpuset4); + } + + #[test] + fn test_cpuset_from_str() { + assert!(U32Set::from_str("").unwrap().is_empty()); + + let support_cpus1 = U32Set::from(vec![1, 2, 3]); + assert_eq!(support_cpus1, U32Set::from_str("1,2,3").unwrap()); + assert_eq!(support_cpus1, U32Set::from_str("1-2,3").unwrap()); + + let support_cpus2 = U32Set::from(vec![1, 3, 4, 6, 7, 8]); + assert_eq!(support_cpus2, U32Set::from_str("1,3,4,6,7,8").unwrap()); + assert_eq!(support_cpus2, U32Set::from_str("1,3-4,6-8").unwrap()); + + assert!(U32Set::from_str("1-2-3,3").is_err()); + assert!(U32Set::from_str("1-2,,3").is_err()); + assert!(U32Set::from_str("1-2.5,3").is_err()); + assert!(U32Set::from_str("1-1").is_err()); + assert!(U32Set::from_str("2-1").is_err()); + assert!(U32Set::from_str("0,,1").is_err()); + assert!(U32Set::from_str("-1").is_err()); + assert!(U32Set::from_str("1-").is_err()); + assert!(U32Set::from_str("-1--2").is_err()); + assert!(U32Set::from_str("999999999999999999999999999999999999999999999").is_err()); + } +} From 5300ea23ad4a2ef4b9d1f697d9598e2a4b79c752 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Fri, 24 Dec 2021 12:47:16 +0800 Subject: [PATCH 0030/1953] libs/sys-util: implement reflink_copy() Implement reflink_copy() to copy file by reflink, and fallback to normal file copy. Signed-off-by: Liu Jiang Signed-off-by: Eryu Guan --- src/libs/kata-sys-util/src/fs.rs | 151 +++++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 29 deletions(-) diff --git a/src/libs/kata-sys-util/src/fs.rs b/src/libs/kata-sys-util/src/fs.rs index 04da18b33a60..a875b832d432 100644 --- a/src/libs/kata-sys-util/src/fs.rs +++ b/src/libs/kata-sys-util/src/fs.rs @@ -5,11 +5,13 @@ // use std::ffi::OsString; -use std::fs; -use std::io::Result; +use std::fs::{self, File}; +use std::io::{Error, Result}; +use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; +use std::process::Command; -use crate::eother; +use crate::{eother, sl}; // from linux.git/fs/fuse/inode.c: #define FUSE_SUPER_MAGIC 0x65735546 const FUSE_SUPER_MAGIC: u32 = 0x65735546; @@ -58,12 +60,114 @@ pub fn is_symlink>(path: P) -> std::io::Result { Ok(meta.file_type().is_symlink()) } +/// Reflink copy src to dst, and falls back to regular copy if reflink copy fails. +/// +/// # Safety +/// The `reflink_copy()` doesn't preserve permission/security context for the copied file, +/// so caller needs to take care of it. +pub fn reflink_copy, D: AsRef>(src: S, dst: D) -> Result<()> { + let src_path = src.as_ref(); + let dst_path = dst.as_ref(); + let src = src_path.to_string_lossy(); + let dst = dst_path.to_string_lossy(); + + if !src_path.is_file() { + return Err(eother!("reflink_copy src {} is not a regular file", src)); + } + + // Make sure dst's parent exist. If dst is a regular file, then unlink it for later copy. + if dst_path.exists() { + if !dst_path.is_file() { + return Err(eother!("reflink_copy dst {} is not a regular file", dst)); + } else { + fs::remove_file(dst_path)?; + } + } else if let Some(dst_parent) = dst_path.parent() { + if !dst_parent.exists() { + if let Err(e) = fs::create_dir_all(dst_parent) { + return Err(eother!( + "reflink_copy: create_dir_all {} failed: {:?}", + dst_parent.to_str().unwrap(), + e + )); + } + } else if !dst_parent.is_dir() { + return Err(eother!("reflink_copy parent of {} is not a directory", dst)); + } + } + + // Reflink copy, and fallback to regular copy if reflink fails. + let src_file = fs::File::open(src_path)?; + let dst_file = fs::File::create(dst_path)?; + if let Err(e) = do_reflink_copy(src_file, dst_file) { + match e.raw_os_error() { + // Cross dev copy or filesystem doesn't support reflink, do regular copy + Some(os_err) + if os_err == nix::Error::EXDEV as i32 + || os_err == nix::Error::EOPNOTSUPP as i32 => + { + warn!( + sl!(), + "reflink_copy: reflink is not supported ({:?}), do regular copy instead", e, + ); + if let Err(e) = do_regular_copy(src.as_ref(), dst.as_ref()) { + return Err(eother!( + "reflink_copy: regular copy {} to {} failed: {:?}", + src, + dst, + e + )); + } + } + // Reflink copy failed + _ => { + return Err(eother!( + "reflink_copy: copy {} to {} failed: {:?}", + src, + dst, + e, + )) + } + } + } + + Ok(()) +} + +// Copy file using cp command, which handles sparse file copy. +fn do_regular_copy(src: &str, dst: &str) -> Result<()> { + let mut cmd = Command::new("/bin/cp"); + cmd.args(&["--sparse=auto", src, dst]); + + match cmd.output() { + Ok(output) => match output.status.success() { + true => Ok(()), + false => Err(eother!("`{:?}` failed: {:?}", cmd, output)), + }, + Err(e) => Err(eother!("`{:?}` failed: {:?}", cmd, e)), + } +} + +/// Copy file by reflink +fn do_reflink_copy(src: File, dst: File) -> Result<()> { + use nix::ioctl_write_int; + // FICLONE ioctl number definition, from include/linux/fs.h + const FS_IOC_MAGIC: u8 = 0x94; + const FS_IOC_FICLONE: u8 = 9; + // Define FICLONE ioctl using nix::ioctl_write_int! macro. + // The generated function has the following signature: + // pub unsafe fn ficlone(fd: libc::c_int, data: libc::c_ulang) -> Result + ioctl_write_int!(ficlone, FS_IOC_MAGIC, FS_IOC_FICLONE); + + // Safe because the `src` and `dst` are valid file objects and we have checked the result. + unsafe { ficlone(dst.as_raw_fd(), src.as_raw_fd() as u64) } + .map(|_| ()) + .map_err(|e| Error::from_raw_os_error(e as i32)) +} + #[cfg(test)] mod tests { use super::*; - use crate::mount::umount_all; - use std::process::Command; - use thiserror::private::PathAsDisplay; #[test] fn test_get_base_name() { @@ -84,28 +188,17 @@ mod tests { } #[test] - fn test_is_overlayfs() { - let tmpdir1 = tempfile::tempdir().unwrap(); - let tmpdir2 = tempfile::tempdir().unwrap(); - let tmpdir3 = tempfile::tempdir().unwrap(); - let tmpdir4 = tempfile::tempdir().unwrap(); - - let option = format!( - "-o lowerdir={},upperdir={},workdir={}", - tmpdir1.path().as_display(), - tmpdir2.path().display(), - tmpdir3.path().display() - ); - let target = format!("{}", tmpdir4.path().display()); - - Command::new("/bin/mount") - .arg("-t overlay") - .arg(option) - .arg("overlay") - .arg(target) - .output() - .unwrap(); - assert!(is_overlay_fs(tmpdir4.path())); - umount_all(tmpdir4.path(), false).unwrap(); + fn test_reflink_copy() { + let tmpdir = tempfile::tempdir().unwrap(); + let path = tmpdir.path().join("mounts"); + reflink_copy("/proc/mounts", &path).unwrap(); + let content = fs::read_to_string(&path).unwrap(); + assert!(!content.is_empty()); + reflink_copy("/proc/mounts", &path).unwrap(); + let content = fs::read_to_string(&path).unwrap(); + assert!(!content.is_empty()); + + reflink_copy("/proc/mounts", tmpdir.path()).unwrap_err(); + reflink_copy("/proc/mounts_not_exist", &path).unwrap_err(); } } From 6d59e8e197c3b1cec288a225fac40e995d954a0c Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Sat, 25 Dec 2021 11:10:55 +0800 Subject: [PATCH 0031/1953] libs/sys-util: introduce function to get device id Introduce get_devid() to get major/minor number of a block device. Signed-off-by: Liu Jiang Signed-off-by: Eryu Guan --- src/libs/kata-sys-util/src/device.rs | 104 +++++++++++++++++++++++++++ src/libs/kata-sys-util/src/lib.rs | 1 + 2 files changed, 105 insertions(+) create mode 100644 src/libs/kata-sys-util/src/device.rs diff --git a/src/libs/kata-sys-util/src/device.rs b/src/libs/kata-sys-util/src/device.rs new file mode 100644 index 000000000000..00a2ade12788 --- /dev/null +++ b/src/libs/kata-sys-util/src/device.rs @@ -0,0 +1,104 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fs; +use std::io::Result; +use std::os::unix::fs::{FileTypeExt, MetadataExt}; +use std::path::{Path, PathBuf}; + +use nix::sys::stat; + +use crate::{eother, sl}; + +const SYS_DEV_BLOCK_PATH: &str = "/sys/dev/block"; +const BLKDEV_PARTITION: &str = "partition"; +const BLKDEV_DEV_FILE: &str = "dev"; + +/// Get major and minor number of the device or of the device hosting the regular file/directory. +pub fn get_devid_for_blkio_cgroup>(path: P) -> Result> { + let md = fs::metadata(path)?; + + if md.is_dir() || md.is_file() { + // For regular file/directory, get major/minor of the block device hosting it. + // Note that we need to get the major/minor of the block device instead of partition, + // e.g. /dev/sda instead of /dev/sda3, because blkio cgroup works with block major/minor. + let id = md.dev(); + Ok(Some((stat::major(id), stat::minor(id)))) + } else if md.file_type().is_block_device() { + // For block device, get major/minor of the device special file itself + get_block_device_id(md.rdev()) + } else { + Ok(None) + } +} + +/// Get the block device major/minor number from a partition/block device(itself). +/// +/// For example, given the dev_t of /dev/sda3 returns major and minor of /dev/sda. We rely on the +/// fact that if /sys/dev/block/$major:$minor/partition exists, then it's a partition, and find its +/// parent for the real device. +fn get_block_device_id(dev: stat::dev_t) -> Result> { + let major = stat::major(dev); + let minor = stat::minor(dev); + let mut blk_dev_path = PathBuf::from(SYS_DEV_BLOCK_PATH) + .join(format!("{}:{}", major, minor)) + .canonicalize()?; + + // If 'partition' file exists, then it's a partition of the real device, take its parent. + // Otherwise it's already the real device. + loop { + if !blk_dev_path.join(BLKDEV_PARTITION).exists() { + break; + } + blk_dev_path = match blk_dev_path.parent() { + Some(p) => p.to_path_buf(), + None => { + return Err(eother!( + "Can't find real device for dev {}:{}", + major, + minor + )) + } + }; + } + + // Parse major:minor in dev file + let dev_path = blk_dev_path.join(BLKDEV_DEV_FILE); + let dev_buf = fs::read_to_string(&dev_path)?; + let dev_buf = dev_buf.trim_end(); + debug!( + sl!(), + "get_real_devid: dev {}:{} -> {:?} ({})", major, minor, blk_dev_path, dev_buf + ); + + if let Some((major, minor)) = dev_buf.split_once(':') { + let major = major + .parse::() + .map_err(|_e| eother!("Failed to parse major number: {}", major))?; + let minor = minor + .parse::() + .map_err(|_e| eother!("Failed to parse minor number: {}", minor))?; + Ok(Some((major, minor))) + } else { + Err(eother!( + "Wrong format in {}: {}", + dev_path.to_string_lossy(), + dev_buf + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_devid() { + //let (major, minor) = get_devid_for_blkio_cgroup("/dev/vda1").unwrap().unwrap(); + assert!(get_devid_for_blkio_cgroup("/dev/tty").unwrap().is_none()); + get_devid_for_blkio_cgroup("/do/not/exist/file_______name").unwrap_err(); + } +} diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 4fa93c8b2d10..b11adf9639bb 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -7,6 +7,7 @@ extern crate slog; pub mod cgroup; +pub mod device; pub mod fs; pub mod mount; pub mod numa; From 8509de0aea330fee75d16a32317d8d08e16ca1b7 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Fri, 24 Dec 2021 11:51:29 +0800 Subject: [PATCH 0032/1953] libs/sys-util: add function to detect and update K8s emptyDir volume Add function to detect and update K8s emptyDir volume. Signed-off-by: Liu Jiang Signed-off-by: Qingyuan Hou --- src/libs/kata-sys-util/Cargo.toml | 1 + src/libs/kata-sys-util/src/k8s.rs | 69 +++++++++++++++++++++++++++++++ src/libs/kata-sys-util/src/lib.rs | 1 + 3 files changed, 71 insertions(+) create mode 100644 src/libs/kata-sys-util/src/k8s.rs diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index f56ff9627b4d..5c6b30354d74 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -24,6 +24,7 @@ slog-scope = "4.4.0" thiserror = "1.0.30" kata-types = { path = "../kata-types" } +oci = { path = "../../agent/oci" } [dev-dependencies] num_cpus = "1.13.1" diff --git a/src/libs/kata-sys-util/src/k8s.rs b/src/libs/kata-sys-util/src/k8s.rs new file mode 100644 index 000000000000..be95d5d33023 --- /dev/null +++ b/src/libs/kata-sys-util/src/k8s.rs @@ -0,0 +1,69 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Utilities to support Kubernetes (K8s). +//! +//! This module depends on kubelet internal implementation details, a better way is needed +//! to detect K8S EmptyDir medium type from `oci::spec::Mount` objects. + +use kata_types::mount; +use oci::Spec; + +use crate::mount::get_linux_mount_info; + +pub use kata_types::k8s::is_empty_dir; + +/// Check whether the given path is a kubernetes ephemeral volume. +/// +/// This method depends on a specific path used by k8s to detect if it's type of ephemeral. +/// As of now, this is a very k8s specific solution that works but in future there should be a +/// better way for this method to determine if the path is for ephemeral volume type. +pub fn is_ephemeral_volume(path: &str) -> bool { + if is_empty_dir(path) { + if let Ok(info) = get_linux_mount_info(path) { + if info.fs_type == "tmpfs" { + return true; + } + } + } + + false +} + +/// Check whether the given path is a kubernetes empty-dir volume of medium "default". +/// +/// K8s `EmptyDir` volumes are directories on the host. If the fs type is tmpfs, it's a ephemeral +/// volume instead of a `EmptyDir` volume. +pub fn is_host_empty_dir(path: &str) -> bool { + if is_empty_dir(path) { + if let Ok(info) = get_linux_mount_info(path) { + if info.fs_type != "tmpfs" { + return true; + } + } + } + + false +} + +// set_ephemeral_storage_type sets the mount type to 'ephemeral' +// if the mount source path is provisioned by k8s for ephemeral storage. +// For the given pod ephemeral volume is created only once +// backed by tmpfs inside the VM. For successive containers +// of the same pod the already existing volume is reused. +pub fn update_ephemeral_storage_type(oci_spec: &mut Spec) { + for m in oci_spec.mounts.iter_mut() { + if mount::is_kata_guest_mount_volume(&m.r#type) { + continue; + } + + if is_ephemeral_volume(&m.source) { + m.r#type = String::from(mount::KATA_EPHEMERAL_VOLUME_TYPE); + } else if is_host_empty_dir(&m.source) { + m.r#type = String::from(mount::KATA_HOST_DIR_VOLUME_TYPE); + } + } +} diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index b11adf9639bb..251588a9fa4d 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -9,6 +9,7 @@ extern crate slog; pub mod cgroup; pub mod device; pub mod fs; +pub mod k8s; pub mod mount; pub mod numa; From aee9633cedc77db394fbee98b62eee9ff0985018 Mon Sep 17 00:00:00 2001 From: Liu Jiang Date: Sat, 25 Dec 2021 18:28:43 +0800 Subject: [PATCH 0033/1953] libs/sys-util: provide functions to execute hooks Provide functions to execute OCI hooks. Signed-off-by: Liu Jiang Signed-off-by: Bin Liu Signed-off-by: Huamin Tang Signed-off-by: Lei Wang Signed-off-by: Quanwei Zhou --- src/libs/Cargo.lock | 13 + src/libs/kata-sys-util/Cargo.toml | 4 +- src/libs/kata-sys-util/src/hooks.rs | 541 ++++++++++++++++++++++++++++ src/libs/kata-sys-util/src/lib.rs | 1 + 4 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 src/libs/kata-sys-util/src/hooks.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index ac6b2b8def53..31b72fc7cc57 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -360,10 +360,13 @@ dependencies = [ "libc", "nix 0.23.1", "num_cpus", + "oci", "once_cell", + "serde_json", "serial_test", "slog", "slog-scope", + "subprocess", "tempfile", "thiserror", ] @@ -916,6 +919,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "syn" version = "1.0.91" diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 5c6b30354d74..524a9b3bb16c 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -19,12 +19,14 @@ lazy_static = "1.4.0" libc = "0.2.100" nix = "0.23.0" once_cell = "1.9.0" +serde_json = "1.0.73" slog = "2.5.2" slog-scope = "4.4.0" +subprocess = "0.2.8" thiserror = "1.0.30" kata-types = { path = "../kata-types" } -oci = { path = "../../agent/oci" } +oci = { path = "../oci" } [dev-dependencies] num_cpus = "1.13.1" diff --git a/src/libs/kata-sys-util/src/hooks.rs b/src/libs/kata-sys-util/src/hooks.rs new file mode 100644 index 000000000000..78e3ae662ed9 --- /dev/null +++ b/src/libs/kata-sys-util/src/hooks.rs @@ -0,0 +1,541 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::collections::HashMap; +use std::ffi::OsString; +use std::hash::{Hash, Hasher}; +use std::io::{self, Read, Result}; +use std::path::Path; +use std::time::Duration; + +use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; + +use crate::{eother, sl}; + +const DEFAULT_HOOK_TIMEOUT_SEC: i32 = 10; + +/// A simple wrapper over `oci::Hook` to provide `Hash, Eq`. +/// +/// The `oci::Hook` is auto-generated from protobuf source file, which doesn't implement `Hash, Eq`. +#[derive(Debug, Default, Clone)] +struct HookKey(oci::Hook); + +impl From<&oci::Hook> for HookKey { + fn from(hook: &oci::Hook) -> Self { + HookKey(hook.clone()) + } +} + +impl PartialEq for HookKey { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl Eq for HookKey {} + +impl Hash for HookKey { + fn hash(&self, state: &mut H) { + self.0.path.hash(state); + self.0.args.hash(state); + self.0.env.hash(state); + self.0.timeout.hash(state); + } +} + +/// Execution state of OCI hooks. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HookState { + /// Hook is pending for executing/retry. + Pending, + /// Hook has been successfully executed. + Done, + /// Hook has been marked as ignore. + Ignored, +} + +/// Structure to maintain state for hooks. +#[derive(Default)] +pub struct HookStates { + states: HashMap, +} + +impl HookStates { + /// Create a new instance of [`HookStates`]. + pub fn new() -> Self { + Self::default() + } + + /// Get execution state of a hook. + pub fn get(&self, hook: &oci::Hook) -> HookState { + self.states + .get(&hook.into()) + .copied() + .unwrap_or(HookState::Pending) + } + + /// Update execution state of a hook. + pub fn update(&mut self, hook: &oci::Hook, state: HookState) { + self.states.insert(hook.into(), state); + } + + /// Remove an execution state of a hook. + pub fn remove(&mut self, hook: &oci::Hook) { + self.states.remove(&hook.into()); + } + + /// Check whether some hooks are still pending and should retry execution. + pub fn should_retry(&self) -> bool { + for state in self.states.values() { + if *state == HookState::Pending { + return true; + } + } + false + } + + /// Execute an OCI hook. + /// + /// If `state` is valid, it will be sent to subprocess' STDIN. + /// + /// The [OCI Runtime specification 1.0.0](https://github.com/opencontainers/runtime-spec/releases/download/v1.0.0/oci-runtime-spec-v1.0.0.pdf) + /// states: + /// - path (string, REQUIRED) with similar semantics to IEEE Std 1003.1-2008 execv's path. + /// This specification extends the IEEE standard in that path MUST be absolute. + /// - args (array of strings, OPTIONAL) with the same semantics as IEEE Std 1003.1-2008 execv's + /// argv. + /// - env (array of strings, OPTIONAL) with the same semantics as IEEE Std 1003.1-2008's environ. + /// - timeout (int, OPTIONAL) is the number of seconds before aborting the hook. If set, timeout + /// MUST be greater than zero. + /// + /// The OCI spec also defines the context to invoke hooks, caller needs to take the responsibility + /// to setup execution context, such as namespace etc. + pub fn execute_hook(&mut self, hook: &oci::Hook, state: Option) -> Result<()> { + if self.get(hook) != HookState::Pending { + return Ok(()); + } + + fail::fail_point!("execute_hook", |_| { + Err(eother!("execute hook fail point injection")) + }); + info!(sl!(), "execute hook {:?}", hook); + + self.states.insert(hook.into(), HookState::Pending); + + let mut executor = HookExecutor::new(hook)?; + let stdin = if state.is_some() { + Redirection::Pipe + } else { + Redirection::None + }; + let mut popen = Popen::create( + &executor.args, + PopenConfig { + stdin, + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + executable: executor.executable.to_owned(), + detached: true, + env: Some(executor.envs.clone()), + ..Default::default() + }, + ) + .map_err(|e| eother!("failed to create subprocess for hook {:?}: {}", hook, e))?; + + if let Some(state) = state { + executor.execute_with_input(&mut popen, state)?; + } + executor.execute_and_wait(&mut popen)?; + info!(sl!(), "hook {} finished", hook.path); + self.states.insert(hook.into(), HookState::Done); + + Ok(()) + } + + /// Try to execute hooks and remember execution result. + /// + /// The `execute_hooks()` will be called multiple times. + /// It will first be called before creating the VMM when creating the sandbox, so hooks could be + /// used to setup environment for the VMM, such as creating tap device etc. + /// It will also be called during starting containers, to setup environment for those containers. + /// + /// The execution result will be recorded for each hook. Once a hook returns success, it will not + /// be invoked anymore. + pub fn execute_hooks(&mut self, hooks: &[oci::Hook], state: Option) -> Result<()> { + for hook in hooks.iter() { + if let Err(e) = self.execute_hook(hook, state.clone()) { + // Ignore error and try next hook, the caller should retry. + error!(sl!(), "hook {} failed: {}", hook.path, e); + } + } + + Ok(()) + } +} + +struct HookExecutor<'a> { + hook: &'a oci::Hook, + executable: Option, + args: Vec, + envs: Vec<(OsString, OsString)>, + timeout: u64, +} + +impl<'a> HookExecutor<'a> { + fn new(hook: &'a oci::Hook) -> Result { + // Ensure Hook.path is present and is an absolute path. + let executable = if hook.path.is_empty() { + return Err(eother!("path of hook {:?} is empty", hook)); + } else { + let path = Path::new(&hook.path); + if !path.is_absolute() { + return Err(eother!("path of hook {:?} is not absolute", hook)); + } + Some(path.as_os_str().to_os_string()) + }; + + // Hook.args is optional, use Hook.path as arg0 if Hook.args is empty. + let args = if hook.args.is_empty() { + vec![hook.path.clone()] + } else { + hook.args.clone() + }; + + let mut envs: Vec<(OsString, OsString)> = Vec::new(); + for e in hook.env.iter() { + match e.split_once('=') { + Some((key, value)) => envs.push((OsString::from(key), OsString::from(value))), + None => warn!(sl!(), "env {} of hook {:?} is invalid", e, hook), + } + } + + // Use Hook.timeout if it's valid, otherwise default to 10s. + let mut timeout = DEFAULT_HOOK_TIMEOUT_SEC as u64; + if let Some(t) = hook.timeout { + if t > 0 { + timeout = t as u64; + } + } + + Ok(HookExecutor { + hook, + executable, + args, + envs, + timeout, + }) + } + + fn execute_with_input(&mut self, popen: &mut Popen, state: oci::State) -> Result<()> { + let st = serde_json::to_string(&state)?; + let (stdout, stderr) = popen + .communicate_start(Some(st.as_bytes().to_vec())) + .limit_time(Duration::from_secs(self.timeout)) + .read_string() + .map_err(|e| e.error)?; + if let Some(err) = stderr { + if !err.is_empty() { + error!(sl!(), "hook {} exec failed: {}", self.hook.path, err); + } + } + if let Some(out) = stdout { + if !out.is_empty() { + info!(sl!(), "hook {} exec stdout: {}", self.hook.path, out); + } + } + // Give a grace period for `execute_and_wait()`. + self.timeout = 1; + Ok(()) + } + + fn execute_and_wait(&mut self, popen: &mut Popen) -> Result<()> { + match popen.wait_timeout(Duration::from_secs(self.timeout)) { + Ok(v) => self.handle_exit_status(v, popen), + Err(e) => self.handle_popen_wait_error(e, popen), + } + } + + fn handle_exit_status(&mut self, result: Option, popen: &mut Popen) -> Result<()> { + if let Some(exit_status) = result { + // the process has finished + info!( + sl!(), + "exit status of hook {:?} : {:?}", self.hook, exit_status + ); + self.print_result(popen); + match exit_status { + subprocess::ExitStatus::Exited(code) => { + if code == 0 { + info!(sl!(), "hook {:?} succeeds", self.hook); + Ok(()) + } else { + warn!(sl!(), "hook {:?} exit status with {}", self.hook, code,); + Err(eother!("hook {:?} exit status with {}", self.hook, code)) + } + } + _ => { + error!( + sl!(), + "no exit code for hook {:?}: {:?}", self.hook, exit_status + ); + Err(eother!( + "no exit code for hook {:?}: {:?}", + self.hook, + exit_status + )) + } + } + } else { + // may be timeout + error!(sl!(), "hook poll failed, kill it"); + // it is still running, kill it + popen.kill()?; + let _ = popen.wait(); + self.print_result(popen); + Err(io::Error::from(io::ErrorKind::TimedOut)) + } + } + + fn handle_popen_wait_error(&mut self, e: PopenError, popen: &mut Popen) -> Result<()> { + self.print_result(popen); + error!(sl!(), "wait_timeout for hook {:?} failed: {}", self.hook, e); + Err(eother!( + "wait_timeout for hook {:?} failed: {}", + self.hook, + e + )) + } + + fn print_result(&mut self, popen: &mut Popen) { + if let Some(file) = popen.stdout.as_mut() { + let mut buffer = String::new(); + file.read_to_string(&mut buffer).ok(); + if !buffer.is_empty() { + info!(sl!(), "hook stdout: {}", buffer); + } + } + if let Some(file) = popen.stderr.as_mut() { + let mut buffer = String::new(); + file.read_to_string(&mut buffer).ok(); + if !buffer.is_empty() { + info!(sl!(), "hook stderr: {}", buffer); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::{self, set_permissions, File, Permissions}; + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + use std::time::Instant; + + fn test_hook_eq(hook1: &oci::Hook, hook2: &oci::Hook, expected: bool) { + let key1 = HookKey::from(hook1); + let key2 = HookKey::from(hook2); + + assert_eq!(key1 == key2, expected); + } + #[test] + fn test_hook_key() { + let hook = oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string(), "3".to_string()], + env: vec![], + timeout: Some(0), + }; + let cases = [ + ( + oci::Hook { + path: "1000".to_string(), + args: vec!["2".to_string(), "3".to_string()], + env: vec![], + timeout: Some(0), + }, + false, + ), + ( + oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string(), "4".to_string()], + env: vec![], + timeout: Some(0), + }, + false, + ), + ( + oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string()], + env: vec![], + timeout: Some(0), + }, + false, + ), + ( + oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string(), "3".to_string()], + env: vec!["5".to_string()], + timeout: Some(0), + }, + false, + ), + ( + oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string(), "3".to_string()], + env: vec![], + timeout: Some(6), + }, + false, + ), + ( + oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string(), "3".to_string()], + env: vec![], + timeout: None, + }, + false, + ), + ( + oci::Hook { + path: "1".to_string(), + args: vec!["2".to_string(), "3".to_string()], + env: vec![], + timeout: Some(0), + }, + true, + ), + ]; + + for case in cases.iter() { + test_hook_eq(&hook, &case.0, case.1); + } + } + + #[test] + fn test_execute_hook() { + // test need root permission + if !nix::unistd::getuid().is_root() { + println!("test need root permission"); + return; + } + + let tmpdir = tempfile::tempdir().unwrap(); + let file = tmpdir.path().join("data"); + let file_str = file.to_string_lossy(); + let mut states = HookStates::new(); + + // test case 1: normal + // execute hook + let hook = oci::Hook { + path: "/bin/touch".to_string(), + args: vec!["touch".to_string(), file_str.to_string()], + env: vec![], + timeout: Some(0), + }; + let ret = states.execute_hook(&hook, None); + assert!(ret.is_ok()); + assert!(fs::metadata(&file).is_ok()); + assert!(!states.should_retry()); + + // test case 2: timeout in 10s + let hook = oci::Hook { + path: "/bin/sleep".to_string(), + args: vec!["sleep".to_string(), "3600".to_string()], + env: vec![], + timeout: Some(0), // default timeout is 10 seconds + }; + let start = Instant::now(); + let ret = states.execute_hook(&hook, None).unwrap_err(); + let duration = start.elapsed(); + let used = duration.as_secs(); + assert!((10..12u64).contains(&used)); + assert_eq!(ret.kind(), io::ErrorKind::TimedOut); + assert_eq!(states.get(&hook), HookState::Pending); + assert!(states.should_retry()); + states.remove(&hook); + + // test case 3: timeout in 5s + let hook = oci::Hook { + path: "/bin/sleep".to_string(), + args: vec!["sleep".to_string(), "3600".to_string()], + env: vec![], + timeout: Some(5), // timeout is set to 5 seconds + }; + let start = Instant::now(); + let ret = states.execute_hook(&hook, None).unwrap_err(); + let duration = start.elapsed(); + let used = duration.as_secs(); + assert!((5..7u64).contains(&used)); + assert_eq!(ret.kind(), io::ErrorKind::TimedOut); + assert_eq!(states.get(&hook), HookState::Pending); + assert!(states.should_retry()); + states.remove(&hook); + + // test case 4: with envs + let create_shell = |shell_path: &str, data_path: &str| -> Result<()> { + let shell = format!( + r#"#!/bin/sh +echo -n "K1=${{K1}}" > {} +"#, + data_path + ); + let mut output = File::create(shell_path)?; + output.write_all(shell.as_bytes())?; + + // set to executable + let permissions = Permissions::from_mode(0o755); + set_permissions(shell_path, permissions)?; + + Ok(()) + }; + let shell_path = format!("{}/test.sh", tmpdir.path().to_string_lossy()); + let ret = create_shell(&shell_path, file_str.as_ref()); + assert!(ret.is_ok()); + let hook = oci::Hook { + path: shell_path, + args: vec![], + env: vec!["K1=V1".to_string()], + timeout: Some(5), + }; + let ret = states.execute_hook(&hook, None); + assert!(ret.is_ok()); + assert!(!states.should_retry()); + let contents = fs::read_to_string(file); + match contents { + Err(e) => panic!("got error {}", e), + Ok(s) => assert_eq!(s, "K1=V1"), + } + + // test case 5: timeout in 5s with state + let hook = oci::Hook { + path: "/bin/sleep".to_string(), + args: vec!["sleep".to_string(), "3600".to_string()], + env: vec![], + timeout: Some(6), // timeout is set to 5 seconds + }; + let state = oci::State { + version: "".to_string(), + id: "".to_string(), + status: oci::ContainerState::Creating, + pid: 10, + bundle: "nouse".to_string(), + annotations: Default::default(), + }; + let start = Instant::now(); + let ret = states.execute_hook(&hook, Some(state)).unwrap_err(); + let duration = start.elapsed(); + let used = duration.as_secs(); + assert!((6..8u64).contains(&used)); + assert_eq!(ret.kind(), io::ErrorKind::TimedOut); + assert!(states.should_retry()); + } +} diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 251588a9fa4d..4efc4fb8cde2 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -9,6 +9,7 @@ extern crate slog; pub mod cgroup; pub mod device; pub mod fs; +pub mod hooks; pub mod k8s; pub mod mount; pub mod numa; From d2a9bc6674c8c86ddc57138e96659bfd85562cf5 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Tue, 22 Feb 2022 15:41:10 +0800 Subject: [PATCH 0034/1953] agent: agent-protocol support async 1. support async. 2. update ttrpc and protobuf update ttrpc to 0.6.0 update protobuf to 2.23.0 3. support trans from oci Fixes: #3746 Signed-off-by: Quanwei Zhou --- src/agent/Cargo.lock | 32 +- src/agent/Cargo.toml | 8 +- src/agent/rustjail/Cargo.toml | 4 +- src/agent/src/rpc.rs | 18 +- src/libs/Cargo.lock | 38 +- src/libs/protocols/Cargo.toml | 8 +- src/libs/protocols/build.rs | 47 +- src/libs/protocols/src/agent_ttrpc_async.rs | 816 +++++++++++++ src/libs/protocols/src/health_ttrpc_async.rs | 90 ++ src/libs/protocols/src/lib.rs | 5 + src/libs/protocols/src/trans.rs | 1085 ++++++++++++++++++ src/tools/agent-ctl/Cargo.lock | 93 +- src/tools/agent-ctl/Cargo.toml | 2 +- src/tools/agent-ctl/src/client.rs | 2 +- src/tools/agent-ctl/src/main.rs | 4 +- src/tools/agent-ctl/src/utils.rs | 6 +- 16 files changed, 2092 insertions(+), 166 deletions(-) create mode 100644 src/libs/protocols/src/agent_ttrpc_async.rs create mode 100644 src/libs/protocols/src/health_ttrpc_async.rs create mode 100644 src/libs/protocols/src/trans.rs diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 096a81a8e5c9..58bac5207df7 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -807,19 +807,6 @@ dependencies = [ "void", ] -[[package]] -name = "nix" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e06129fb611568ef4e868c14b326274959aa70ff7776e9d55323531c374945" -dependencies = [ - "bitflags", - "cc", - "cfg-if 1.0.0", - "libc", - "memoffset", -] - [[package]] name = "nix" version = "0.22.2" @@ -1183,9 +1170,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" dependencies = [ "serde", "serde_derive", @@ -1193,18 +1180,18 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c" +checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" dependencies = [ "protobuf", ] [[package]] name = "protobuf-codegen-pure" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440" +checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" dependencies = [ "protobuf", "protobuf-codegen", @@ -1215,6 +1202,7 @@ name = "protocols" version = "0.1.0" dependencies = [ "async-trait", + "oci", "protobuf", "ttrpc", "ttrpc-codegen", @@ -1831,16 +1819,16 @@ dependencies = [ [[package]] name = "ttrpc" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a973ce6d5eaa20c173635b29ffb660dafbc7ef109172c0015ba44e47a23711" +checksum = "0c7d6c992964a013c17814c08d31708d577b0aae44ebadb58755659dd824c2d1" dependencies = [ "async-trait", "byteorder", "futures", "libc", "log", - "nix 0.20.2", + "nix 0.23.1", "protobuf", "protobuf-codegen-pure", "thiserror", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 77ec06d8ac51..93ecde93c631 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -7,10 +7,10 @@ edition = "2018" [dependencies] oci = { path = "../libs/oci" } rustjail = { path = "rustjail" } -protocols = { path = "../libs/protocols" } +protocols = { path = "../libs/protocols", features = ["async"] } lazy_static = "1.3.0" -ttrpc = { version = "0.5.0", features = ["async", "protobuf-codec"], default-features = false } -protobuf = "=2.14.0" +ttrpc = { version = "0.6.0", features = ["async"], default-features = false } +protobuf = "2.23.0" libc = "0.2.58" nix = "0.23.0" capctl = "0.2.0" @@ -31,7 +31,7 @@ futures = "0.3.17" tokio = { version = "1.14.0", features = ["full"] } tokio-vsock = "0.3.1" -netlink-sys = { version = "0.7.0", features = ["tokio_socket",]} +netlink-sys = { version = "0.7.0", features = ["tokio_socket"]} rtnetlink = "0.8.0" netlink-packet-utils = "0.4.1" ipnetwork = "0.17.0" diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index 306af2795d78..628b0275bdbe 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -16,7 +16,7 @@ scopeguard = "1.0.0" capctl = "0.2.0" lazy_static = "1.3.0" libc = "0.2.58" -protobuf = "=2.14.0" +protobuf = "2.23.0" slog = "2.5.2" slog-scope = "4.1.2" scan_fmt = "0.2.6" @@ -27,7 +27,7 @@ cgroups = { package = "cgroups-rs", version = "0.2.8" } rlimit = "0.5.3" cfg-if = "0.1.0" -tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] } +tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] } futures = "0.3.17" async-trait = "0.1.31" inotify = "0.9.2" diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index bcf2096d2b4a..6d14da1d98fa 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -34,6 +34,7 @@ use protocols::health::{ HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse, }; use protocols::types::Interface; +use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc}; use rustjail::cgroups::notifier; use rustjail::container::{BaseContainer, Container, LinuxContainer}; use rustjail::process::Process; @@ -650,7 +651,7 @@ impl AgentService { } #[async_trait] -impl protocols::agent_ttrpc::AgentService for AgentService { +impl agent_ttrpc::AgentService for AgentService { async fn create_container( &self, ctx: &TtrpcContext, @@ -1536,7 +1537,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService { struct HealthService; #[async_trait] -impl protocols::health_ttrpc::Health for HealthService { +impl health_ttrpc::Health for HealthService { async fn check( &self, _ctx: &TtrpcContext, @@ -1675,18 +1676,17 @@ async fn read_stream(reader: Arc>>, l: usize) -> Resu } pub fn start(s: Arc>, server_address: &str) -> Result { - let agent_service = Box::new(AgentService { sandbox: s }) - as Box; + let agent_service = + Box::new(AgentService { sandbox: s }) as Box; let agent_worker = Arc::new(agent_service); - let health_service = - Box::new(HealthService {}) as Box; + let health_service = Box::new(HealthService {}) as Box; let health_worker = Arc::new(health_service); - let aservice = protocols::agent_ttrpc::create_agent_service(agent_worker); + let aservice = agent_ttrpc::create_agent_service(agent_worker); - let hservice = protocols::health_ttrpc::create_health(health_worker); + let hservice = health_ttrpc::create_health(health_worker); let server = TtrpcServer::new() .bind(server_address)? @@ -2021,7 +2021,7 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> { mod tests { use super::*; use crate::{ - assert_result, namespace::Namespace, protocols::agent_ttrpc::AgentService as _, + assert_result, namespace::Namespace, protocols::agent_ttrpc_async::AgentService as _, skip_if_not_root, }; use nix::mount; diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 31b72fc7cc57..0917f9552daa 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -88,7 +88,7 @@ checksum = "1b827f9d9f6c2fff719d25f5d44cbc8d2ef6df1ef00d055c5c14d5dc25529579" dependencies = [ "libc", "log", - "nix 0.23.1", + "nix", "regex", ] @@ -358,7 +358,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix 0.23.1", + "nix", "num_cpus", "oci", "once_cell", @@ -475,19 +475,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nix" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e06129fb611568ef4e868c14b326274959aa70ff7776e9d55323531c374945" -dependencies = [ - "bitflags", - "cc", - "cfg-if", - "libc", - "memoffset", -] - [[package]] name = "nix" version = "0.23.1" @@ -670,9 +657,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" dependencies = [ "serde", "serde_derive", @@ -680,18 +667,18 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c" +checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" dependencies = [ "protobuf", ] [[package]] name = "protobuf-codegen-pure" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440" +checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" dependencies = [ "protobuf", "protobuf-codegen", @@ -702,6 +689,7 @@ name = "protocols" version = "0.1.0" dependencies = [ "async-trait", + "oci", "protobuf", "serde", "serde_json", @@ -1050,16 +1038,16 @@ dependencies = [ [[package]] name = "ttrpc" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a973ce6d5eaa20c173635b29ffb660dafbc7ef109172c0015ba44e47a23711" +checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8" dependencies = [ "async-trait", "byteorder", "futures", "libc", "log", - "nix 0.20.2", + "nix", "protobuf", "protobuf-codegen-pure", "thiserror", @@ -1113,7 +1101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" dependencies = [ "libc", - "nix 0.23.1", + "nix", ] [[package]] diff --git a/src/libs/protocols/Cargo.toml b/src/libs/protocols/Cargo.toml index ae93e7fa191a..eda2eeffc7da 100644 --- a/src/libs/protocols/Cargo.toml +++ b/src/libs/protocols/Cargo.toml @@ -7,13 +7,15 @@ edition = "2018" [features] default = [] with-serde = [ "serde", "serde_json" ] +async = ["ttrpc/async", "async-trait"] [dependencies] -ttrpc = { version = "0.5.0", features = ["async"] } -async-trait = "0.1.42" -protobuf = { version = "=2.14.0", features = ["with-serde"] } +ttrpc = { version = "0.6.0" } +async-trait = { version = "0.1.42", optional = true } +protobuf = { version = "2.23.0", features = ["with-serde"] } serde = { version = "1.0.130", features = ["derive"], optional = true } serde_json = { version = "1.0.68", optional = true } +oci = { path = "../oci" } [build-dependencies] ttrpc-codegen = "0.2.0" diff --git a/src/libs/protocols/build.rs b/src/libs/protocols/build.rs index 4a43f36777d5..8a2725ae0527 100644 --- a/src/libs/protocols/build.rs +++ b/src/libs/protocols/build.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::fs::File; +use std::fs::{self, File}; use std::io::{BufRead, BufReader, Read, Write}; use std::path::Path; use std::process::exit; @@ -90,17 +90,8 @@ fn handle_file(autogen_comment: &str, rust_filename: &str) -> Result<(), std::io Ok(()) } -fn real_main() -> Result<(), std::io::Error> { - let autogen_comment = format!("\n//! Generated by {:?} ({:?})", file!(), module_path!()); - - let protos = vec![ - "protos/agent.proto", - "protos/csi.proto", - "protos/google/protobuf/empty.proto", - "protos/health.proto", - "protos/oci.proto", - "protos/types.proto", - ]; +fn codegen(path: &str, protos: &[&str], async_all: bool) -> Result<(), std::io::Error> { + fs::create_dir_all(path).unwrap(); // Tell Cargo that if the .proto files changed, to rerun this build script. protos @@ -108,7 +99,7 @@ fn real_main() -> Result<(), std::io::Error> { .for_each(|p| println!("cargo:rerun-if-changed={}", &p)); let ttrpc_options = Customize { - async_server: true, + async_all, ..Default::default() }; @@ -121,13 +112,14 @@ fn real_main() -> Result<(), std::io::Error> { Codegen::new() .out_dir(out_dir) - .inputs(&protos) + .inputs(protos) .include("protos") .customize(ttrpc_options) .rust_protobuf() .rust_protobuf_customize(protobuf_options) .run()?; + let autogen_comment = format!("\n//! Generated by {:?} ({:?})", file!(), module_path!()); for file in protos.iter() { let proto_filename = Path::new(file).file_name().unwrap(); @@ -147,6 +139,31 @@ fn real_main() -> Result<(), std::io::Error> { handle_file(&autogen_comment, out_file_str)?; } + use_serde(protos, out_dir)?; + Ok(()) +} +fn real_main() -> Result<(), std::io::Error> { + codegen( + "src", + &[ + "protos/google/protobuf/empty.proto", + "protos/oci.proto", + "protos/types.proto", + ], + false, + )?; + + // generate async + #[cfg(feature = "async")] + { + codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?; + + fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?; + fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?; + } + + codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?; + // There is a message named 'Box' in oci.proto // so there is a struct named 'Box', we should replace Box to ::std::boxed::Box // to avoid the conflict. @@ -156,8 +173,6 @@ fn real_main() -> Result<(), std::io::Error> { "self: ::std::boxed::Box", )?; - use_serde(&protos, out_dir)?; - Ok(()) } diff --git a/src/libs/protocols/src/agent_ttrpc_async.rs b/src/libs/protocols/src/agent_ttrpc_async.rs new file mode 100644 index 000000000000..fd8979907d2f --- /dev/null +++ b/src/libs/protocols/src/agent_ttrpc_async.rs @@ -0,0 +1,816 @@ +// This file is generated by ttrpc-compiler 0.4.1. Do not edit +// @generated + +// https://github.com/Manishearth/rust-clippy/issues/702 +#![allow(unknown_lints)] +#![allow(clipto_camel_casepy)] + +#![cfg_attr(rustfmt, rustfmt_skip)] + +#![allow(box_pointers)] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(trivial_casts)] +#![allow(unsafe_code)] +#![allow(unused_imports)] +#![allow(unused_results)] +use protobuf::{CodedInputStream, CodedOutputStream, Message}; +use std::collections::HashMap; +use std::sync::Arc; +use async_trait::async_trait; + +#[derive(Clone)] +pub struct AgentServiceClient { + client: ::ttrpc::r#async::Client, +} + +impl AgentServiceClient { + pub fn new(client: ::ttrpc::r#async::Client) -> Self { + AgentServiceClient { + client: client, + } + } + + pub async fn create_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::CreateContainerRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "CreateContainer", cres); + } + + pub async fn start_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::StartContainerRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "StartContainer", cres); + } + + pub async fn remove_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::RemoveContainerRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "RemoveContainer", cres); + } + + pub async fn exec_process(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ExecProcessRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ExecProcess", cres); + } + + pub async fn signal_process(&mut self, ctx: ttrpc::context::Context, req: &super::agent::SignalProcessRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "SignalProcess", cres); + } + + pub async fn wait_process(&mut self, ctx: ttrpc::context::Context, req: &super::agent::WaitProcessRequest) -> ::ttrpc::Result { + let mut cres = super::agent::WaitProcessResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "WaitProcess", cres); + } + + pub async fn update_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::UpdateContainerRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "UpdateContainer", cres); + } + + pub async fn stats_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::StatsContainerRequest) -> ::ttrpc::Result { + let mut cres = super::agent::StatsContainerResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "StatsContainer", cres); + } + + pub async fn pause_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::PauseContainerRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "PauseContainer", cres); + } + + pub async fn resume_container(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ResumeContainerRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ResumeContainer", cres); + } + + pub async fn write_stdin(&mut self, ctx: ttrpc::context::Context, req: &super::agent::WriteStreamRequest) -> ::ttrpc::Result { + let mut cres = super::agent::WriteStreamResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "WriteStdin", cres); + } + + pub async fn read_stdout(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ReadStreamRequest) -> ::ttrpc::Result { + let mut cres = super::agent::ReadStreamResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ReadStdout", cres); + } + + pub async fn read_stderr(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ReadStreamRequest) -> ::ttrpc::Result { + let mut cres = super::agent::ReadStreamResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ReadStderr", cres); + } + + pub async fn close_stdin(&mut self, ctx: ttrpc::context::Context, req: &super::agent::CloseStdinRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "CloseStdin", cres); + } + + pub async fn tty_win_resize(&mut self, ctx: ttrpc::context::Context, req: &super::agent::TtyWinResizeRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "TtyWinResize", cres); + } + + pub async fn update_interface(&mut self, ctx: ttrpc::context::Context, req: &super::agent::UpdateInterfaceRequest) -> ::ttrpc::Result { + let mut cres = super::types::Interface::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "UpdateInterface", cres); + } + + pub async fn update_routes(&mut self, ctx: ttrpc::context::Context, req: &super::agent::UpdateRoutesRequest) -> ::ttrpc::Result { + let mut cres = super::agent::Routes::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "UpdateRoutes", cres); + } + + pub async fn list_interfaces(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ListInterfacesRequest) -> ::ttrpc::Result { + let mut cres = super::agent::Interfaces::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ListInterfaces", cres); + } + + pub async fn list_routes(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ListRoutesRequest) -> ::ttrpc::Result { + let mut cres = super::agent::Routes::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ListRoutes", cres); + } + + pub async fn add_arp_neighbors(&mut self, ctx: ttrpc::context::Context, req: &super::agent::AddARPNeighborsRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "AddARPNeighbors", cres); + } + + pub async fn get_ip_tables(&mut self, ctx: ttrpc::context::Context, req: &super::agent::GetIPTablesRequest) -> ::ttrpc::Result { + let mut cres = super::agent::GetIPTablesResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "GetIPTables", cres); + } + + pub async fn set_ip_tables(&mut self, ctx: ttrpc::context::Context, req: &super::agent::SetIPTablesRequest) -> ::ttrpc::Result { + let mut cres = super::agent::SetIPTablesResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "SetIPTables", cres); + } + + pub async fn get_metrics(&mut self, ctx: ttrpc::context::Context, req: &super::agent::GetMetricsRequest) -> ::ttrpc::Result { + let mut cres = super::agent::Metrics::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "GetMetrics", cres); + } + + pub async fn create_sandbox(&mut self, ctx: ttrpc::context::Context, req: &super::agent::CreateSandboxRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "CreateSandbox", cres); + } + + pub async fn destroy_sandbox(&mut self, ctx: ttrpc::context::Context, req: &super::agent::DestroySandboxRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "DestroySandbox", cres); + } + + pub async fn online_cpu_mem(&mut self, ctx: ttrpc::context::Context, req: &super::agent::OnlineCPUMemRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "OnlineCPUMem", cres); + } + + pub async fn reseed_random_dev(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ReseedRandomDevRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ReseedRandomDev", cres); + } + + pub async fn get_guest_details(&mut self, ctx: ttrpc::context::Context, req: &super::agent::GuestDetailsRequest) -> ::ttrpc::Result { + let mut cres = super::agent::GuestDetailsResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "GetGuestDetails", cres); + } + + pub async fn mem_hotplug_by_probe(&mut self, ctx: ttrpc::context::Context, req: &super::agent::MemHotplugByProbeRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "MemHotplugByProbe", cres); + } + + pub async fn set_guest_date_time(&mut self, ctx: ttrpc::context::Context, req: &super::agent::SetGuestDateTimeRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "SetGuestDateTime", cres); + } + + pub async fn copy_file(&mut self, ctx: ttrpc::context::Context, req: &super::agent::CopyFileRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "CopyFile", cres); + } + + pub async fn get_oom_event(&mut self, ctx: ttrpc::context::Context, req: &super::agent::GetOOMEventRequest) -> ::ttrpc::Result { + let mut cres = super::agent::OOMEvent::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "GetOOMEvent", cres); + } + + pub async fn add_swap(&mut self, ctx: ttrpc::context::Context, req: &super::agent::AddSwapRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "AddSwap", cres); + } + + pub async fn get_volume_stats(&mut self, ctx: ttrpc::context::Context, req: &super::agent::VolumeStatsRequest) -> ::ttrpc::Result { + let mut cres = super::csi::VolumeStatsResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "GetVolumeStats", cres); + } + + pub async fn resize_volume(&mut self, ctx: ttrpc::context::Context, req: &super::agent::ResizeVolumeRequest) -> ::ttrpc::Result { + let mut cres = super::empty::Empty::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.AgentService", "ResizeVolume", cres); + } +} + +struct CreateContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for CreateContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, CreateContainerRequest, create_container); + } +} + +struct StartContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for StartContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, StartContainerRequest, start_container); + } +} + +struct RemoveContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for RemoveContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, RemoveContainerRequest, remove_container); + } +} + +struct ExecProcessMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ExecProcessMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ExecProcessRequest, exec_process); + } +} + +struct SignalProcessMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for SignalProcessMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, SignalProcessRequest, signal_process); + } +} + +struct WaitProcessMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for WaitProcessMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, WaitProcessRequest, wait_process); + } +} + +struct UpdateContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for UpdateContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, UpdateContainerRequest, update_container); + } +} + +struct StatsContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for StatsContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, StatsContainerRequest, stats_container); + } +} + +struct PauseContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for PauseContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, PauseContainerRequest, pause_container); + } +} + +struct ResumeContainerMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ResumeContainerMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ResumeContainerRequest, resume_container); + } +} + +struct WriteStdinMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for WriteStdinMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, WriteStreamRequest, write_stdin); + } +} + +struct ReadStdoutMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ReadStdoutMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ReadStreamRequest, read_stdout); + } +} + +struct ReadStderrMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ReadStderrMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ReadStreamRequest, read_stderr); + } +} + +struct CloseStdinMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for CloseStdinMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, CloseStdinRequest, close_stdin); + } +} + +struct TtyWinResizeMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for TtyWinResizeMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, TtyWinResizeRequest, tty_win_resize); + } +} + +struct UpdateInterfaceMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for UpdateInterfaceMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, UpdateInterfaceRequest, update_interface); + } +} + +struct UpdateRoutesMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for UpdateRoutesMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, UpdateRoutesRequest, update_routes); + } +} + +struct ListInterfacesMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ListInterfacesMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ListInterfacesRequest, list_interfaces); + } +} + +struct ListRoutesMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ListRoutesMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ListRoutesRequest, list_routes); + } +} + +struct AddArpNeighborsMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for AddArpNeighborsMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, AddARPNeighborsRequest, add_arp_neighbors); + } +} + +struct GetIpTablesMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for GetIpTablesMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, GetIPTablesRequest, get_ip_tables); + } +} + +struct SetIpTablesMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for SetIpTablesMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, SetIPTablesRequest, set_ip_tables); + } +} + +struct GetMetricsMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for GetMetricsMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, GetMetricsRequest, get_metrics); + } +} + +struct CreateSandboxMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for CreateSandboxMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, CreateSandboxRequest, create_sandbox); + } +} + +struct DestroySandboxMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for DestroySandboxMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, DestroySandboxRequest, destroy_sandbox); + } +} + +struct OnlineCpuMemMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for OnlineCpuMemMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, OnlineCPUMemRequest, online_cpu_mem); + } +} + +struct ReseedRandomDevMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ReseedRandomDevMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ReseedRandomDevRequest, reseed_random_dev); + } +} + +struct GetGuestDetailsMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for GetGuestDetailsMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, GuestDetailsRequest, get_guest_details); + } +} + +struct MemHotplugByProbeMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for MemHotplugByProbeMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, MemHotplugByProbeRequest, mem_hotplug_by_probe); + } +} + +struct SetGuestDateTimeMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for SetGuestDateTimeMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, SetGuestDateTimeRequest, set_guest_date_time); + } +} + +struct CopyFileMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for CopyFileMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, CopyFileRequest, copy_file); + } +} + +struct GetOomEventMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for GetOomEventMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, GetOOMEventRequest, get_oom_event); + } +} + +struct AddSwapMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for AddSwapMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, AddSwapRequest, add_swap); + } +} + +struct GetVolumeStatsMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for GetVolumeStatsMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, VolumeStatsRequest, get_volume_stats); + } +} + +struct ResizeVolumeMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for ResizeVolumeMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, agent, ResizeVolumeRequest, resize_volume); + } +} + +#[async_trait] +pub trait AgentService: Sync { + async fn create_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::CreateContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CreateContainer is not supported".to_string()))) + } + async fn start_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::StartContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/StartContainer is not supported".to_string()))) + } + async fn remove_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::RemoveContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/RemoveContainer is not supported".to_string()))) + } + async fn exec_process(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ExecProcessRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ExecProcess is not supported".to_string()))) + } + async fn signal_process(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::SignalProcessRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/SignalProcess is not supported".to_string()))) + } + async fn wait_process(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::WaitProcessRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/WaitProcess is not supported".to_string()))) + } + async fn update_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::UpdateContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/UpdateContainer is not supported".to_string()))) + } + async fn stats_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::StatsContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/StatsContainer is not supported".to_string()))) + } + async fn pause_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::PauseContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/PauseContainer is not supported".to_string()))) + } + async fn resume_container(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ResumeContainerRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ResumeContainer is not supported".to_string()))) + } + async fn write_stdin(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::WriteStreamRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/WriteStdin is not supported".to_string()))) + } + async fn read_stdout(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ReadStreamRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ReadStdout is not supported".to_string()))) + } + async fn read_stderr(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ReadStreamRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ReadStderr is not supported".to_string()))) + } + async fn close_stdin(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::CloseStdinRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CloseStdin is not supported".to_string()))) + } + async fn tty_win_resize(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::TtyWinResizeRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/TtyWinResize is not supported".to_string()))) + } + async fn update_interface(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::UpdateInterfaceRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/UpdateInterface is not supported".to_string()))) + } + async fn update_routes(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::UpdateRoutesRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/UpdateRoutes is not supported".to_string()))) + } + async fn list_interfaces(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ListInterfacesRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ListInterfaces is not supported".to_string()))) + } + async fn list_routes(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ListRoutesRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ListRoutes is not supported".to_string()))) + } + async fn add_arp_neighbors(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::AddARPNeighborsRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/AddARPNeighbors is not supported".to_string()))) + } + async fn get_ip_tables(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::GetIPTablesRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetIPTables is not supported".to_string()))) + } + async fn set_ip_tables(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::SetIPTablesRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/SetIPTables is not supported".to_string()))) + } + async fn get_metrics(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::GetMetricsRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetMetrics is not supported".to_string()))) + } + async fn create_sandbox(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::CreateSandboxRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CreateSandbox is not supported".to_string()))) + } + async fn destroy_sandbox(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::DestroySandboxRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/DestroySandbox is not supported".to_string()))) + } + async fn online_cpu_mem(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::OnlineCPUMemRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/OnlineCPUMem is not supported".to_string()))) + } + async fn reseed_random_dev(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ReseedRandomDevRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ReseedRandomDev is not supported".to_string()))) + } + async fn get_guest_details(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::GuestDetailsRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetGuestDetails is not supported".to_string()))) + } + async fn mem_hotplug_by_probe(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::MemHotplugByProbeRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/MemHotplugByProbe is not supported".to_string()))) + } + async fn set_guest_date_time(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::SetGuestDateTimeRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/SetGuestDateTime is not supported".to_string()))) + } + async fn copy_file(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::CopyFileRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/CopyFile is not supported".to_string()))) + } + async fn get_oom_event(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::GetOOMEventRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetOOMEvent is not supported".to_string()))) + } + async fn add_swap(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::AddSwapRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/AddSwap is not supported".to_string()))) + } + async fn get_volume_stats(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::VolumeStatsRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/GetVolumeStats is not supported".to_string()))) + } + async fn resize_volume(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::agent::ResizeVolumeRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.AgentService/ResizeVolume is not supported".to_string()))) + } +} + +pub fn create_agent_service(service: Arc>) -> HashMap > { + let mut methods = HashMap::new(); + + methods.insert("/grpc.AgentService/CreateContainer".to_string(), + std::boxed::Box::new(CreateContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/StartContainer".to_string(), + std::boxed::Box::new(StartContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/RemoveContainer".to_string(), + std::boxed::Box::new(RemoveContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ExecProcess".to_string(), + std::boxed::Box::new(ExecProcessMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/SignalProcess".to_string(), + std::boxed::Box::new(SignalProcessMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/WaitProcess".to_string(), + std::boxed::Box::new(WaitProcessMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/UpdateContainer".to_string(), + std::boxed::Box::new(UpdateContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/StatsContainer".to_string(), + std::boxed::Box::new(StatsContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/PauseContainer".to_string(), + std::boxed::Box::new(PauseContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ResumeContainer".to_string(), + std::boxed::Box::new(ResumeContainerMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/WriteStdin".to_string(), + std::boxed::Box::new(WriteStdinMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ReadStdout".to_string(), + std::boxed::Box::new(ReadStdoutMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ReadStderr".to_string(), + std::boxed::Box::new(ReadStderrMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/CloseStdin".to_string(), + std::boxed::Box::new(CloseStdinMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/TtyWinResize".to_string(), + std::boxed::Box::new(TtyWinResizeMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/UpdateInterface".to_string(), + std::boxed::Box::new(UpdateInterfaceMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/UpdateRoutes".to_string(), + std::boxed::Box::new(UpdateRoutesMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ListInterfaces".to_string(), + std::boxed::Box::new(ListInterfacesMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ListRoutes".to_string(), + std::boxed::Box::new(ListRoutesMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/AddARPNeighbors".to_string(), + std::boxed::Box::new(AddArpNeighborsMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/GetIPTables".to_string(), + std::boxed::Box::new(GetIpTablesMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/SetIPTables".to_string(), + std::boxed::Box::new(SetIpTablesMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/GetMetrics".to_string(), + std::boxed::Box::new(GetMetricsMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/CreateSandbox".to_string(), + std::boxed::Box::new(CreateSandboxMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/DestroySandbox".to_string(), + std::boxed::Box::new(DestroySandboxMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/OnlineCPUMem".to_string(), + std::boxed::Box::new(OnlineCpuMemMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ReseedRandomDev".to_string(), + std::boxed::Box::new(ReseedRandomDevMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/GetGuestDetails".to_string(), + std::boxed::Box::new(GetGuestDetailsMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/MemHotplugByProbe".to_string(), + std::boxed::Box::new(MemHotplugByProbeMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/SetGuestDateTime".to_string(), + std::boxed::Box::new(SetGuestDateTimeMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/CopyFile".to_string(), + std::boxed::Box::new(CopyFileMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/GetOOMEvent".to_string(), + std::boxed::Box::new(GetOomEventMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/AddSwap".to_string(), + std::boxed::Box::new(AddSwapMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/GetVolumeStats".to_string(), + std::boxed::Box::new(GetVolumeStatsMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.AgentService/ResizeVolume".to_string(), + std::boxed::Box::new(ResizeVolumeMethod{service: service.clone()}) as std::boxed::Box); + + methods +} diff --git a/src/libs/protocols/src/health_ttrpc_async.rs b/src/libs/protocols/src/health_ttrpc_async.rs new file mode 100644 index 000000000000..5f77b30e7546 --- /dev/null +++ b/src/libs/protocols/src/health_ttrpc_async.rs @@ -0,0 +1,90 @@ +// This file is generated by ttrpc-compiler 0.4.1. Do not edit +// @generated + +// https://github.com/Manishearth/rust-clippy/issues/702 +#![allow(unknown_lints)] +#![allow(clipto_camel_casepy)] + +#![cfg_attr(rustfmt, rustfmt_skip)] + +#![allow(box_pointers)] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(trivial_casts)] +#![allow(unsafe_code)] +#![allow(unused_imports)] +#![allow(unused_results)] +use protobuf::{CodedInputStream, CodedOutputStream, Message}; +use std::collections::HashMap; +use std::sync::Arc; +use async_trait::async_trait; + +#[derive(Clone)] +pub struct HealthClient { + client: ::ttrpc::r#async::Client, +} + +impl HealthClient { + pub fn new(client: ::ttrpc::r#async::Client) -> Self { + HealthClient { + client: client, + } + } + + pub async fn check(&mut self, ctx: ttrpc::context::Context, req: &super::health::CheckRequest) -> ::ttrpc::Result { + let mut cres = super::health::HealthCheckResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.Health", "Check", cres); + } + + pub async fn version(&mut self, ctx: ttrpc::context::Context, req: &super::health::CheckRequest) -> ::ttrpc::Result { + let mut cres = super::health::VersionCheckResponse::new(); + ::ttrpc::async_client_request!(self, ctx, req, "grpc.Health", "Version", cres); + } +} + +struct CheckMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for CheckMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, health, CheckRequest, check); + } +} + +struct VersionMethod { + service: Arc>, +} + +#[async_trait] +impl ::ttrpc::r#async::MethodHandler for VersionMethod { + async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<(u32, Vec)> { + ::ttrpc::async_request_handler!(self, ctx, req, health, CheckRequest, version); + } +} + +#[async_trait] +pub trait Health: Sync { + async fn check(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::health::CheckRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.Health/Check is not supported".to_string()))) + } + async fn version(&self, _ctx: &::ttrpc::r#async::TtrpcContext, _req: super::health::CheckRequest) -> ::ttrpc::Result { + Err(::ttrpc::Error::RpcStatus(::ttrpc::get_status(::ttrpc::Code::NOT_FOUND, "/grpc.Health/Version is not supported".to_string()))) + } +} + +pub fn create_health(service: Arc>) -> HashMap > { + let mut methods = HashMap::new(); + + methods.insert("/grpc.Health/Check".to_string(), + std::boxed::Box::new(CheckMethod{service: service.clone()}) as std::boxed::Box); + + methods.insert("/grpc.Health/Version".to_string(), + std::boxed::Box::new(VersionMethod{service: service.clone()}) as std::boxed::Box); + + methods +} diff --git a/src/libs/protocols/src/lib.rs b/src/libs/protocols/src/lib.rs index 14298e52d925..0c62b8a933ec 100644 --- a/src/libs/protocols/src/lib.rs +++ b/src/libs/protocols/src/lib.rs @@ -7,9 +7,14 @@ pub mod agent; pub mod agent_ttrpc; +#[cfg(feature = "async")] +pub mod agent_ttrpc_async; pub mod csi; pub mod empty; pub mod health; pub mod health_ttrpc; +#[cfg(feature = "async")] +pub mod health_ttrpc_async; pub mod oci; +pub mod trans; pub mod types; diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs new file mode 100644 index 000000000000..52fb0d99ea4b --- /dev/null +++ b/src/libs/protocols/src/trans.rs @@ -0,0 +1,1085 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::collections::HashMap; +use std::convert::From; + +use oci::{ + Hook, Hooks, Linux, LinuxBlockIo, LinuxCapabilities, LinuxCpu, LinuxDevice, LinuxHugepageLimit, + LinuxIdMapping, LinuxIntelRdt, LinuxInterfacePriority, LinuxMemory, LinuxNamespace, + LinuxNetwork, LinuxPids, LinuxResources, LinuxSeccomp, LinuxSeccompArg, LinuxSyscall, + LinuxThrottleDevice, LinuxWeightDevice, Mount, PosixRlimit, Process, Root, Spec, User, +}; + +// translate from interface to ttprc tools +fn from_option>(from: Option) -> ::protobuf::SingularPtrField { + match from { + Some(f) => ::protobuf::SingularPtrField::from_option(Some(T::from(f))), + None => ::protobuf::SingularPtrField::none(), + } +} + +fn from_vec>(from: Vec) -> ::protobuf::RepeatedField { + let mut to: Vec = vec![]; + for data in from { + to.push(T::from(data)); + } + ::protobuf::RepeatedField::from_vec(to) +} + +impl From for crate::oci::Box { + fn from(from: oci::Box) -> Self { + crate::oci::Box { + Height: from.height, + Width: from.width, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::User { + fn from(from: User) -> Self { + crate::oci::User { + UID: from.uid, + GID: from.gid, + AdditionalGids: from.additional_gids, + Username: from.username, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxCapabilities { + fn from(from: LinuxCapabilities) -> Self { + crate::oci::LinuxCapabilities { + Bounding: from_vec(from.bounding), + Effective: from_vec(from.effective), + Inheritable: from_vec(from.inheritable), + Permitted: from_vec(from.permitted), + Ambient: from_vec(from.ambient), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::POSIXRlimit { + fn from(from: PosixRlimit) -> Self { + crate::oci::POSIXRlimit { + Type: from.r#type, + Hard: from.hard, + Soft: from.soft, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Process { + fn from(from: Process) -> Self { + crate::oci::Process { + Terminal: from.terminal, + ConsoleSize: from_option(from.console_size), + User: from_option(Some(from.user)), + Args: from_vec(from.args), + Env: from_vec(from.env), + Cwd: from.cwd, + Capabilities: from_option(from.capabilities), + Rlimits: from_vec(from.rlimits), + NoNewPrivileges: from.no_new_privileges, + ApparmorProfile: from.apparmor_profile, + OOMScoreAdj: from.oom_score_adj.map_or(0, |t| t as i64), + SelinuxLabel: from.selinux_label, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxDeviceCgroup { + fn from(from: oci::LinuxDeviceCgroup) -> Self { + crate::oci::LinuxDeviceCgroup { + Allow: from.allow, + Type: from.r#type, + Major: from.major.map_or(0, |t| t as i64), + Minor: from.minor.map_or(0, |t| t as i64), + Access: from.access, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxMemory { + fn from(from: LinuxMemory) -> Self { + crate::oci::LinuxMemory { + Limit: from.limit.map_or(0, |t| t), + Reservation: from.reservation.map_or(0, |t| t), + Swap: from.swap.map_or(0, |t| t), + Kernel: from.kernel.map_or(0, |t| t), + KernelTCP: from.kernel_tcp.map_or(0, |t| t), + Swappiness: from.swappiness.map_or(0, |t| t as u64), + DisableOOMKiller: from.disable_oom_killer.map_or(false, |t| t), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxCPU { + fn from(from: LinuxCpu) -> Self { + crate::oci::LinuxCPU { + Shares: from.shares.map_or(0, |t| t), + Quota: from.quota.map_or(0, |t| t), + Period: from.period.map_or(0, |t| t), + RealtimeRuntime: from.realtime_runtime.map_or(0, |t| t), + RealtimePeriod: from.realtime_period.map_or(0, |t| t), + Cpus: from.cpus, + Mems: from.mems, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxPids { + fn from(from: LinuxPids) -> Self { + crate::oci::LinuxPids { + Limit: from.limit, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxWeightDevice { + fn from(from: LinuxWeightDevice) -> Self { + crate::oci::LinuxWeightDevice { + // TODO : check + Major: 0, + Minor: 0, + Weight: from.weight.map_or(0, |t| t as u32), + LeafWeight: from.leaf_weight.map_or(0, |t| t as u32), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxThrottleDevice { + fn from(from: LinuxThrottleDevice) -> Self { + crate::oci::LinuxThrottleDevice { + // TODO : check + Major: 0, + Minor: 0, + Rate: from.rate, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxBlockIO { + fn from(from: LinuxBlockIo) -> Self { + crate::oci::LinuxBlockIO { + Weight: from.weight.map_or(0, |t| t as u32), + LeafWeight: from.leaf_weight.map_or(0, |t| t as u32), + WeightDevice: from_vec(from.weight_device), + ThrottleReadBpsDevice: from_vec(from.throttle_read_bps_device), + ThrottleWriteBpsDevice: from_vec(from.throttle_write_bps_device), + ThrottleReadIOPSDevice: from_vec(from.throttle_read_iops_device), + ThrottleWriteIOPSDevice: from_vec(from.throttle_write_iops_device), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxHugepageLimit { + fn from(from: LinuxHugepageLimit) -> Self { + crate::oci::LinuxHugepageLimit { + Pagesize: from.page_size, + Limit: from.limit, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxInterfacePriority { + fn from(from: LinuxInterfacePriority) -> Self { + crate::oci::LinuxInterfacePriority { + Name: from.name, + Priority: from.priority, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxNetwork { + fn from(from: LinuxNetwork) -> Self { + crate::oci::LinuxNetwork { + ClassID: from.class_id.map_or(0, |t| t), + Priorities: from_vec(from.priorities), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxResources { + fn from(from: LinuxResources) -> Self { + crate::oci::LinuxResources { + Devices: from_vec(from.devices), + Memory: from_option(from.memory), + CPU: from_option(from.cpu), + Pids: from_option(from.pids), + BlockIO: from_option(from.block_io), + HugepageLimits: from_vec(from.hugepage_limits), + Network: from_option(from.network), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Root { + fn from(from: Root) -> Self { + crate::oci::Root { + Path: from.path, + Readonly: from.readonly, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Mount { + fn from(from: Mount) -> Self { + crate::oci::Mount { + destination: from.destination, + source: from.source, + field_type: from.r#type, + options: from_vec(from.options), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Hook { + fn from(from: Hook) -> Self { + let mut timeout: i64 = 0; + if let Some(v) = from.timeout { + timeout = v as i64; + } + crate::oci::Hook { + Path: from.path, + Args: from_vec(from.args), + Env: from_vec(from.env), + Timeout: timeout, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Hooks { + fn from(from: Hooks) -> Self { + crate::oci::Hooks { + Prestart: from_vec(from.prestart), + Poststart: from_vec(from.poststart), + Poststop: from_vec(from.poststop), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxIDMapping { + fn from(from: LinuxIdMapping) -> Self { + crate::oci::LinuxIDMapping { + HostID: from.host_id, + ContainerID: from.container_id, + Size: from.size, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxNamespace { + fn from(from: LinuxNamespace) -> Self { + crate::oci::LinuxNamespace { + Type: from.r#type, + Path: from.path, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxDevice { + fn from(from: LinuxDevice) -> Self { + crate::oci::LinuxDevice { + Path: from.path, + Type: from.r#type, + Major: from.major, + Minor: from.minor, + FileMode: from.file_mode.map_or(0, |v| v as u32), + UID: from.uid.map_or(0, |v| v), + GID: from.gid.map_or(0, |v| v), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxSeccompArg { + fn from(from: LinuxSeccompArg) -> Self { + crate::oci::LinuxSeccompArg { + Index: from.index as u64, + Value: from.value, + ValueTwo: from.value_two, + Op: from.op, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxSyscall { + fn from(from: LinuxSyscall) -> Self { + crate::oci::LinuxSyscall { + Names: from_vec(from.names), + Action: from.action, + Args: from_vec(from.args), + ErrnoRet: Some(crate::oci::LinuxSyscall_oneof_ErrnoRet::errnoret( + from.errno_ret, + )), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxSeccomp { + fn from(from: LinuxSeccomp) -> Self { + crate::oci::LinuxSeccomp { + DefaultAction: from.default_action, + Architectures: from_vec(from.architectures), + Syscalls: from_vec(from.syscalls), + Flags: from_vec(from.flags), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::LinuxIntelRdt { + fn from(from: LinuxIntelRdt) -> Self { + crate::oci::LinuxIntelRdt { + L3CacheSchema: from.l3_cache_schema, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Linux { + fn from(from: Linux) -> Self { + crate::oci::Linux { + UIDMappings: from_vec(from.uid_mappings), + GIDMappings: from_vec(from.gid_mappings), + Sysctl: from.sysctl, + Resources: from_option(from.resources), + CgroupsPath: from.cgroups_path, + Namespaces: from_vec(from.namespaces), + Devices: from_vec(from.devices), + Seccomp: from_option(from.seccomp), + RootfsPropagation: from.rootfs_propagation, + MaskedPaths: from_vec(from.masked_paths), + ReadonlyPaths: from_vec(from.readonly_paths), + MountLabel: from.mount_label, + IntelRdt: from_option(from.intel_rdt), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for crate::oci::Spec { + fn from(from: Spec) -> Self { + crate::oci::Spec { + Version: from.version, + Process: from_option(from.process), + Root: from_option(from.root), + Hostname: from.hostname, + Mounts: from_vec(from.mounts), + Hooks: from_option(from.hooks), + Annotations: from.annotations, + Linux: from_option(from.linux), + Solaris: Default::default(), + Windows: Default::default(), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for oci::Root { + fn from(from: crate::oci::Root) -> Self { + Self { + path: from.Path, + readonly: from.Readonly, + } + } +} + +impl From for oci::Mount { + fn from(mut from: crate::oci::Mount) -> Self { + let options = from.take_options().to_vec(); + Self { + r#type: from.take_field_type(), + destination: from.take_destination(), + source: from.take_source(), + options, + } + } +} + +impl From for oci::LinuxIdMapping { + fn from(from: crate::oci::LinuxIDMapping) -> Self { + LinuxIdMapping { + container_id: from.get_ContainerID(), + host_id: from.get_HostID(), + size: from.get_Size(), + } + } +} + +impl From for oci::LinuxDeviceCgroup { + fn from(mut from: crate::oci::LinuxDeviceCgroup) -> Self { + let mut major = None; + if from.get_Major() > 0 { + major = Some(from.get_Major() as i64); + } + + let mut minor = None; + if from.get_Minor() > 0 { + minor = Some(from.get_Minor() as i64) + } + + oci::LinuxDeviceCgroup { + allow: from.get_Allow(), + r#type: from.take_Type(), + major, + minor, + access: from.take_Access(), + } + } +} + +impl From for oci::LinuxMemory { + fn from(from: crate::oci::LinuxMemory) -> Self { + let mut limit = None; + if from.get_Limit() > 0 { + limit = Some(from.get_Limit()); + } + + let mut reservation = None; + if from.get_Reservation() > 0 { + reservation = Some(from.get_Reservation()); + } + + let mut swap = None; + if from.get_Swap() > 0 { + swap = Some(from.get_Swap()); + } + + let mut kernel = None; + if from.get_Kernel() > 0 { + kernel = Some(from.get_Kernel()); + } + + let mut kernel_tcp = None; + if from.get_KernelTCP() > 0 { + kernel_tcp = Some(from.get_KernelTCP()); + } + + let mut swappiness = None; + if from.get_Swappiness() > 0 { + swappiness = Some(from.get_Swappiness() as i64); + } + + let disable_oom_killer = Some(from.get_DisableOOMKiller()); + + oci::LinuxMemory { + limit, + reservation, + swap, + kernel, + kernel_tcp, + swappiness, + disable_oom_killer, + } + } +} + +impl From for oci::LinuxCpu { + fn from(mut from: crate::oci::LinuxCPU) -> Self { + let mut shares = None; + if from.get_Shares() > 0 { + shares = Some(from.get_Shares()); + } + + let mut quota = None; + if from.get_Quota() > 0 { + quota = Some(from.get_Quota()); + } + + let mut period = None; + if from.get_Period() > 0 { + period = Some(from.get_Period()); + } + + let mut realtime_runtime = None; + if from.get_RealtimeRuntime() > 0 { + realtime_runtime = Some(from.get_RealtimeRuntime()); + } + + let mut realtime_period = None; + if from.get_RealtimePeriod() > 0 { + realtime_period = Some(from.get_RealtimePeriod()); + } + + let cpus = from.take_Cpus(); + let mems = from.take_Mems(); + + oci::LinuxCpu { + shares, + quota, + period, + realtime_runtime, + realtime_period, + cpus, + mems, + } + } +} + +impl From for oci::LinuxPids { + fn from(from: crate::oci::LinuxPids) -> Self { + oci::LinuxPids { + limit: from.get_Limit(), + } + } +} + +impl From for oci::LinuxBlockIo { + fn from(from: crate::oci::LinuxBlockIO) -> Self { + let mut weight = None; + if from.get_Weight() > 0 { + weight = Some(from.get_Weight() as u16); + } + let mut leaf_weight = None; + if from.get_LeafWeight() > 0 { + leaf_weight = Some(from.get_LeafWeight() as u16); + } + let mut weight_device = Vec::new(); + for wd in from.get_WeightDevice() { + weight_device.push(wd.clone().into()); + } + + let mut throttle_read_bps_device = Vec::new(); + for td in from.get_ThrottleReadBpsDevice() { + throttle_read_bps_device.push(td.clone().into()); + } + + let mut throttle_write_bps_device = Vec::new(); + for td in from.get_ThrottleWriteBpsDevice() { + throttle_write_bps_device.push(td.clone().into()); + } + + let mut throttle_read_iops_device = Vec::new(); + for td in from.get_ThrottleReadIOPSDevice() { + throttle_read_iops_device.push(td.clone().into()); + } + + let mut throttle_write_iops_device = Vec::new(); + for td in from.get_ThrottleWriteIOPSDevice() { + throttle_write_iops_device.push(td.clone().into()); + } + + oci::LinuxBlockIo { + weight, + leaf_weight, + weight_device, + throttle_read_bps_device, + throttle_write_bps_device, + throttle_read_iops_device, + throttle_write_iops_device, + } + } +} + +impl From for oci::LinuxThrottleDevice { + fn from(from: crate::oci::LinuxThrottleDevice) -> Self { + oci::LinuxThrottleDevice { + blk: oci::LinuxBlockIoDevice { + major: from.Major, + minor: from.Minor, + }, + rate: from.Rate, + } + } +} + +impl From for oci::LinuxWeightDevice { + fn from(from: crate::oci::LinuxWeightDevice) -> Self { + oci::LinuxWeightDevice { + blk: oci::LinuxBlockIoDevice { + major: from.Major, + minor: from.Minor, + }, + weight: Some(from.Weight as u16), + leaf_weight: Some(from.LeafWeight as u16), + } + } +} + +impl From for oci::LinuxInterfacePriority { + fn from(mut from: crate::oci::LinuxInterfacePriority) -> Self { + oci::LinuxInterfacePriority { + name: from.take_Name(), + priority: from.get_Priority(), + } + } +} + +impl From for oci::LinuxNetwork { + fn from(mut from: crate::oci::LinuxNetwork) -> Self { + let mut class_id = None; + if from.get_ClassID() > 0 { + class_id = Some(from.get_ClassID()); + } + let mut priorities = Vec::new(); + for p in from.take_Priorities().to_vec() { + priorities.push(p.into()) + } + + oci::LinuxNetwork { + class_id, + priorities, + } + } +} + +impl From for oci::LinuxHugepageLimit { + fn from(mut from: crate::oci::LinuxHugepageLimit) -> Self { + oci::LinuxHugepageLimit { + page_size: from.take_Pagesize(), + limit: from.get_Limit(), + } + } +} + +impl From for oci::LinuxResources { + fn from(mut from: crate::oci::LinuxResources) -> Self { + let mut devices = Vec::new(); + for d in from.take_Devices().to_vec() { + devices.push(d.into()); + } + + let mut memory = None; + if from.has_Memory() { + memory = Some(from.take_Memory().into()); + } + + let mut cpu = None; + if from.has_CPU() { + cpu = Some(from.take_CPU().into()); + } + + let mut pids = None; + if from.has_Pids() { + pids = Some(from.get_Pids().clone().into()) + } + + let mut block_io = None; + if from.has_BlockIO() { + block_io = Some(from.get_BlockIO().clone().into()); + } + + let mut hugepage_limits = Vec::new(); + for hl in from.get_HugepageLimits() { + hugepage_limits.push(hl.clone().into()); + } + + let mut network = None; + if from.has_Network() { + network = Some(from.take_Network().into()); + } + + let rdma = HashMap::new(); + + LinuxResources { + devices, + memory, + cpu, + pids, + block_io, + hugepage_limits, + network, + rdma, + } + } +} + +impl From for oci::LinuxDevice { + fn from(mut from: crate::oci::LinuxDevice) -> Self { + oci::LinuxDevice { + path: from.take_Path(), + r#type: from.take_Type(), + major: from.get_Major(), + minor: from.get_Minor(), + file_mode: Some(from.get_FileMode()), + uid: Some(from.get_UID()), + gid: Some(from.get_GID()), + } + } +} + +impl From for oci::LinuxSeccompArg { + fn from(mut from: crate::oci::LinuxSeccompArg) -> Self { + oci::LinuxSeccompArg { + index: from.get_Index() as u32, + value: from.get_Value(), + value_two: from.get_ValueTwo(), + op: from.take_Op(), + } + } +} + +impl From for oci::LinuxSyscall { + fn from(mut from: crate::oci::LinuxSyscall) -> Self { + let mut args = Vec::new(); + for ag in from.take_Args().to_vec() { + args.push(ag.into()); + } + oci::LinuxSyscall { + names: from.take_Names().to_vec(), + action: from.take_Action(), + args, + errno_ret: from.get_errnoret(), + } + } +} + +impl From for oci::LinuxSeccomp { + fn from(mut from: crate::oci::LinuxSeccomp) -> Self { + let mut syscalls = Vec::new(); + for s in from.take_Syscalls().to_vec() { + syscalls.push(s.into()); + } + + oci::LinuxSeccomp { + default_action: from.take_DefaultAction(), + architectures: from.take_Architectures().to_vec(), + syscalls, + flags: from.take_Flags().to_vec(), + } + } +} + +impl From for oci::LinuxNamespace { + fn from(mut from: crate::oci::LinuxNamespace) -> Self { + oci::LinuxNamespace { + r#type: from.take_Type(), + path: from.take_Path(), + } + } +} + +impl From for oci::Linux { + fn from(mut from: crate::oci::Linux) -> Self { + let mut uid_mappings = Vec::new(); + for id_map in from.take_UIDMappings().to_vec() { + uid_mappings.push(id_map.into()) + } + + let mut gid_mappings = Vec::new(); + for id_map in from.take_GIDMappings().to_vec() { + gid_mappings.push(id_map.into()) + } + + let sysctl = from.get_Sysctl().clone(); + let mut resources = None; + if from.has_Resources() { + resources = Some(from.take_Resources().into()); + } + + let cgroups_path = from.take_CgroupsPath(); + let mut namespaces = Vec::new(); + for ns in from.take_Namespaces().to_vec() { + namespaces.push(ns.into()) + } + + let mut devices = Vec::new(); + for d in from.take_Devices().to_vec() { + devices.push(d.into()); + } + + let mut seccomp = None; + if from.has_Seccomp() { + seccomp = Some(from.take_Seccomp().into()); + } + + let rootfs_propagation = from.take_RootfsPropagation(); + let masked_paths = from.take_MaskedPaths().to_vec(); + + let readonly_paths = from.take_ReadonlyPaths().to_vec(); + + let mount_label = from.take_MountLabel(); + let intel_rdt = None; + + oci::Linux { + uid_mappings, + gid_mappings, + sysctl, + resources, + cgroups_path, + namespaces, + devices, + seccomp, + rootfs_propagation, + masked_paths, + readonly_paths, + mount_label, + intel_rdt, + } + } +} + +impl From for oci::PosixRlimit { + fn from(mut from: crate::oci::POSIXRlimit) -> Self { + oci::PosixRlimit { + r#type: from.take_Type(), + hard: from.get_Hard(), + soft: from.get_Soft(), + } + } +} + +impl From for oci::LinuxCapabilities { + fn from(mut from: crate::oci::LinuxCapabilities) -> Self { + oci::LinuxCapabilities { + bounding: from.take_Bounding().to_vec(), + effective: from.take_Effective().to_vec(), + inheritable: from.take_Inheritable().to_vec(), + permitted: from.take_Permitted().to_vec(), + ambient: from.take_Ambient().to_vec(), + } + } +} + +impl From for oci::User { + fn from(mut from: crate::oci::User) -> Self { + oci::User { + uid: from.get_UID(), + gid: from.get_GID(), + additional_gids: from.take_AdditionalGids().to_vec(), + username: from.take_Username(), + } + } +} + +impl From for oci::Box { + fn from(from: crate::oci::Box) -> Self { + oci::Box { + height: from.get_Height(), + width: from.get_Width(), + } + } +} + +impl From for oci::Process { + fn from(mut from: crate::oci::Process) -> Self { + let mut console_size = None; + if from.has_ConsoleSize() { + console_size = Some(from.take_ConsoleSize().into()); + } + + let user = from.take_User().into(); + let args = from.take_Args().into_vec(); + let env = from.take_Env().into_vec(); + let cwd = from.take_Cwd(); + let mut capabilities = None; + if from.has_Capabilities() { + capabilities = Some(from.take_Capabilities().into()); + } + let mut rlimits = Vec::new(); + for rl in from.take_Rlimits().to_vec() { + rlimits.push(rl.into()); + } + let no_new_privileges = from.get_NoNewPrivileges(); + let apparmor_profile = from.take_ApparmorProfile(); + let mut oom_score_adj = None; + if from.get_OOMScoreAdj() != 0 { + oom_score_adj = Some(from.get_OOMScoreAdj() as i32); + } + let selinux_label = from.take_SelinuxLabel(); + + oci::Process { + terminal: from.Terminal, + console_size, + user, + args, + env, + cwd, + capabilities, + rlimits, + no_new_privileges, + apparmor_profile, + oom_score_adj, + selinux_label, + } + } +} + +impl From for oci::Hook { + fn from(mut from: crate::oci::Hook) -> Self { + let mut timeout = None; + if from.get_Timeout() > 0 { + timeout = Some(from.get_Timeout() as i32); + } + oci::Hook { + path: from.take_Path(), + args: from.take_Args().to_vec(), + env: from.take_Env().to_vec(), + timeout, + } + } +} + +impl From for oci::Hooks { + fn from(mut from: crate::oci::Hooks) -> Self { + let mut prestart = Vec::new(); + for hook in from.take_Prestart().to_vec() { + prestart.push(hook.into()) + } + let mut poststart = Vec::new(); + for hook in from.take_Poststart().to_vec() { + poststart.push(hook.into()); + } + let mut poststop = Vec::new(); + for hook in from.take_Poststop().to_vec() { + poststop.push(hook.into()); + } + oci::Hooks { + prestart, + poststart, + poststop, + } + } +} + +impl From for oci::Spec { + fn from(mut from: crate::oci::Spec) -> Self { + let mut process = None; + if from.has_Process() { + process = Some(from.take_Process().into()); + } + + let mut root = None; + if from.has_Root() { + root = Some(from.take_Root().into()); + } + + let mut mounts = Vec::new(); + for m in from.take_Mounts().into_vec() { + mounts.push(m.into()) + } + + let mut hooks: Option = None; + if from.has_Hooks() { + hooks = Some(from.take_Hooks().into()); + } + + let annotations = from.take_Annotations(); + + let mut linux = None; + if from.has_Linux() { + linux = Some(from.take_Linux().into()); + } + + oci::Spec { + version: from.take_Version(), + process, + root, + hostname: from.take_Hostname(), + mounts, + hooks, + annotations, + linux, + solaris: None, + windows: None, + vm: None, + } + } +} + +#[cfg(test)] +mod tests { + use crate::trans::from_vec; + + #[derive(Clone)] + struct TestA { + pub from: String, + } + + #[derive(Clone)] + struct TestB { + pub to: String, + } + + impl From for TestB { + fn from(from: TestA) -> Self { + TestB { to: from.from } + } + } + + #[test] + fn test_from() { + let from = TestA { + from: "a".to_string(), + }; + let to: TestB = TestB::from(from.clone()); + + assert_eq!(from.from, to.to); + } + + #[test] + fn test_from_vec_len_0() { + let from: Vec = vec![]; + let to: ::protobuf::RepeatedField = from_vec(from.clone()); + assert_eq!(from.len(), to.len()); + } + + #[test] + fn test_from_vec_len_1() { + let from: Vec = vec![TestA { + from: "a".to_string(), + }]; + let to: ::protobuf::RepeatedField = from_vec(from.clone()); + + assert_eq!(from.len(), to.len()); + assert_eq!(from[0].from, to[0].to); + } +} diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index d82e11e6e8c1..ec036924e07f 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -72,16 +72,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.1.0" @@ -129,7 +119,7 @@ checksum = "1b827f9d9f6c2fff719d25f5d44cbc8d2ef6df1ef00d055c5c14d5dc25529579" dependencies = [ "libc", "log", - "nix 0.23.1", + "nix", "regex", ] @@ -398,15 +388,6 @@ dependencies = [ "libc", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "itertools" version = "0.10.3" @@ -434,7 +415,7 @@ dependencies = [ "lazy_static", "libc", "logging", - "nix 0.23.1", + "nix", "oci", "protobuf", "protocols", @@ -522,19 +503,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nix" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e06129fb611568ef4e868c14b326274959aa70ff7776e9d55323531c374945" -dependencies = [ - "bitflags", - "cc", - "cfg-if", - "libc", - "memoffset", -] - [[package]] name = "nix" version = "0.23.1" @@ -666,7 +634,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ - "bytes 1.1.0", + "bytes", "prost-derive", ] @@ -676,7 +644,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" dependencies = [ - "bytes 1.1.0", + "bytes", "heck", "itertools", "log", @@ -707,15 +675,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ - "bytes 1.1.0", + "bytes", "prost", ] [[package]] name = "protobuf" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" dependencies = [ "serde", "serde_derive", @@ -723,18 +691,18 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c" +checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" dependencies = [ "protobuf", ] [[package]] name = "protobuf-codegen-pure" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440" +checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" dependencies = [ "protobuf", "protobuf-codegen", @@ -744,7 +712,7 @@ dependencies = [ name = "protocols" version = "0.1.0" dependencies = [ - "async-trait", + "oci", "protobuf", "serde", "serde_json", @@ -858,7 +826,7 @@ dependencies = [ "inotify", "lazy_static", "libc", - "nix 0.23.1", + "nix", "oci", "path-absolutize", "protobuf", @@ -1080,7 +1048,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" dependencies = [ - "bytes 1.1.0", + "bytes", "libc", "memchr", "mio", @@ -1102,36 +1070,19 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-vsock" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6" -dependencies = [ - "bytes 0.4.12", - "futures", - "libc", - "tokio", - "vsock", -] - [[package]] name = "ttrpc" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a973ce6d5eaa20c173635b29ffb660dafbc7ef109172c0015ba44e47a23711" +checksum = "0c7d6c992964a013c17814c08d31708d577b0aae44ebadb58755659dd824c2d1" dependencies = [ - "async-trait", "byteorder", - "futures", "libc", "log", - "nix 0.20.2", + "nix", "protobuf", "protobuf-codegen-pure", "thiserror", - "tokio", - "tokio-vsock", ] [[package]] @@ -1185,16 +1136,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" -[[package]] -name = "vsock" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" -dependencies = [ - "libc", - "nix 0.23.1", -] - [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/src/tools/agent-ctl/Cargo.toml b/src/tools/agent-ctl/Cargo.toml index 8847f734f431..4d5845f1c31a 100644 --- a/src/tools/agent-ctl/Cargo.toml +++ b/src/tools/agent-ctl/Cargo.toml @@ -31,7 +31,7 @@ protobuf = "2.14.0" nix = "0.23.0" libc = "0.2.112" # XXX: Must be the same as the version used by the agent -ttrpc = { version = "0.5.2" } +ttrpc = { version = "0.6.0" } # For parsing timeouts humantime = "2.1.0" diff --git a/src/tools/agent-ctl/src/client.rs b/src/tools/agent-ctl/src/client.rs index 315969b87be2..802bc7966959 100644 --- a/src/tools/agent-ctl/src/client.rs +++ b/src/tools/agent-ctl/src/client.rs @@ -561,7 +561,7 @@ fn create_ttrpc_client( } }; - Ok(ttrpc::client::Client::new(fd)) + Ok(ttrpc::Client::new(fd)) } fn kata_service_agent( diff --git a/src/tools/agent-ctl/src/main.rs b/src/tools/agent-ctl/src/main.rs index 93d913ba86a9..fe5722b07945 100644 --- a/src/tools/agent-ctl/src/main.rs +++ b/src/tools/agent-ctl/src/main.rs @@ -181,11 +181,11 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { let cfg = Config { server_address, bundle_dir, - interactive, - ignore_errors, timeout_nano, hybrid_vsock_port, + interactive, hybrid_vsock, + ignore_errors, no_auto_values, }; diff --git a/src/tools/agent-ctl/src/utils.rs b/src/tools/agent-ctl/src/utils.rs index 064b54486ad3..bd094df170ab 100644 --- a/src/tools/agent-ctl/src/utils.rs +++ b/src/tools/agent-ctl/src/utils.rs @@ -203,11 +203,7 @@ pub fn get_option(name: &str, options: &mut Options, args: &str) -> Result { msg = "derived"; - match options.get("cid") { - Some(value) => value, - None => "", - } - .into() + options.get("cid").unwrap_or(&"".to_string()).into() } _ => "".into(), }; From 69ba1ae9e4a66e8a964191023626d4d801032889 Mon Sep 17 00:00:00 2001 From: Fupan Li Date: Fri, 10 Jun 2022 19:46:25 +0800 Subject: [PATCH 0035/1953] trans: fix the issue of wrong swapness type Signed-off-by: Fupan Li --- src/libs/protocols/src/trans.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/protocols/src/trans.rs b/src/libs/protocols/src/trans.rs index 52fb0d99ea4b..e9ecfe785970 100644 --- a/src/libs/protocols/src/trans.rs +++ b/src/libs/protocols/src/trans.rs @@ -515,7 +515,7 @@ impl From for oci::LinuxMemory { let mut swappiness = None; if from.get_Swappiness() > 0 { - swappiness = Some(from.get_Swappiness() as i64); + swappiness = Some(from.get_Swappiness()); } let disable_oom_killer = Some(from.get_DisableOOMKiller()); From 641b736106d124f5c808e459558fd1e6382bc9c8 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Thu, 3 Mar 2022 14:24:08 +0800 Subject: [PATCH 0036/1953] libs: enhance kata-sys-util 1. move verify_cid from agent to libs/kata-sys-util 2. enhance kata-sys-util/k8s Signed-off-by: Quanwei Zhou --- src/agent/Cargo.lock | 76 ++++++- src/agent/Cargo.toml | 1 + src/agent/src/rpc.rs | 253 +---------------------- src/libs/kata-sys-util/src/fs.rs | 14 +- src/libs/kata-sys-util/src/lib.rs | 1 + src/libs/kata-sys-util/src/validate.rs | 267 +++++++++++++++++++++++++ src/libs/kata-types/.gitignore | 1 + src/libs/kata-types/src/container.rs | 18 +- src/libs/kata-types/src/k8s.rs | 139 +++++++++++-- src/libs/protocols/.gitignore | 2 + 10 files changed, 495 insertions(+), 277 deletions(-) create mode 100644 src/libs/kata-sys-util/src/validate.rs create mode 100644 src/libs/kata-types/.gitignore diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 58bac5207df7..3f1b4df85dc2 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -214,6 +214,12 @@ dependencies = [ "syn", ] +[[package]] +name = "common-path" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" + [[package]] name = "core-foundation-sys" version = "0.8.3" @@ -311,6 +317,17 @@ dependencies = [ "libc", ] +[[package]] +name = "fail" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" +dependencies = [ + "lazy_static", + "log", + "rand", +] + [[package]] name = "fixedbitset" version = "0.2.0" @@ -440,6 +457,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -571,6 +594,7 @@ dependencies = [ "clap", "futures", "ipnetwork", + "kata-sys-util", "lazy_static", "libc", "log", @@ -608,6 +632,44 @@ dependencies = [ "vsock-exporter", ] +[[package]] +name = "kata-sys-util" +version = "0.1.0" +dependencies = [ + "cgroups-rs", + "chrono", + "common-path", + "fail", + "kata-types", + "lazy_static", + "libc", + "nix 0.23.1", + "oci", + "once_cell", + "serde_json", + "slog", + "slog-scope", + "subprocess", + "thiserror", +] + +[[package]] +name = "kata-types" +version = "0.1.0" +dependencies = [ + "glob", + "lazy_static", + "num_cpus", + "oci", + "regex", + "serde", + "serde_json", + "slog", + "slog-scope", + "thiserror", + "toml", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -863,9 +925,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -1552,6 +1614,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subprocess" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "syn" version = "1.0.82" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 93ecde93c631..041fcdb5d95e 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -20,6 +20,7 @@ scopeguard = "1.0.0" thiserror = "1.0.26" regex = "1.5.4" serial_test = "0.5.1" +kata-sys-util = { path = "../libs/kata-sys-util" } sysinfo = "0.23.0" # Async helpers diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 6d14da1d98fa..5e9bc7696f94 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -134,30 +134,6 @@ pub struct AgentService { sandbox: Arc>, } -// A container ID must match this regex: -// -// ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$ -// -fn verify_cid(id: &str) -> Result<()> { - let mut chars = id.chars(); - - let valid = match chars.next() { - Some(first) - if first.is_alphanumeric() - && id.len() > 1 - && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)) => - { - true - } - _ => false, - }; - - match valid { - true => Ok(()), - false => Err(anyhow!("invalid container ID: {:?}", id)), - } -} - impl AgentService { #[instrument] async fn do_create_container( @@ -166,7 +142,7 @@ impl AgentService { ) -> Result<()> { let cid = req.container_id.clone(); - verify_cid(&cid)?; + kata_sys_util::validate::verify_cid(&cid)?; let mut oci_spec = req.OCI.clone(); let use_sandbox_pidns = req.get_sandbox_pidns(); @@ -2674,233 +2650,6 @@ OtherField:other } } - #[tokio::test] - async fn test_verify_cid() { - #[derive(Debug)] - struct TestData<'a> { - id: &'a str, - expect_error: bool, - } - - let tests = &[ - TestData { - // Cannot be blank - id: "", - expect_error: true, - }, - TestData { - // Cannot be a space - id: " ", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: ".", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: "-", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: "_", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: " a", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: ".a", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: "-a", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: "_a", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: "..", - expect_error: true, - }, - TestData { - // Too short - id: "a", - expect_error: true, - }, - TestData { - // Too short - id: "z", - expect_error: true, - }, - TestData { - // Too short - id: "A", - expect_error: true, - }, - TestData { - // Too short - id: "Z", - expect_error: true, - }, - TestData { - // Too short - id: "0", - expect_error: true, - }, - TestData { - // Too short - id: "9", - expect_error: true, - }, - TestData { - // Must start with an alphanumeric - id: "-1", - expect_error: true, - }, - TestData { - id: "/", - expect_error: true, - }, - TestData { - id: "a/", - expect_error: true, - }, - TestData { - id: "a/../", - expect_error: true, - }, - TestData { - id: "../a", - expect_error: true, - }, - TestData { - id: "../../a", - expect_error: true, - }, - TestData { - id: "../../../a", - expect_error: true, - }, - TestData { - id: "foo/../bar", - expect_error: true, - }, - TestData { - id: "foo bar", - expect_error: true, - }, - TestData { - id: "a.", - expect_error: false, - }, - TestData { - id: "a..", - expect_error: false, - }, - TestData { - id: "aa", - expect_error: false, - }, - TestData { - id: "aa.", - expect_error: false, - }, - TestData { - id: "hello..world", - expect_error: false, - }, - TestData { - id: "hello/../world", - expect_error: true, - }, - TestData { - id: "aa1245124sadfasdfgasdga.", - expect_error: false, - }, - TestData { - id: "aAzZ0123456789_.-", - expect_error: false, - }, - TestData { - id: "abcdefghijklmnopqrstuvwxyz0123456789.-_", - expect_error: false, - }, - TestData { - id: "0123456789abcdefghijklmnopqrstuvwxyz.-_", - expect_error: false, - }, - TestData { - id: " abcdefghijklmnopqrstuvwxyz0123456789.-_", - expect_error: true, - }, - TestData { - id: ".abcdefghijklmnopqrstuvwxyz0123456789.-_", - expect_error: true, - }, - TestData { - id: "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_", - expect_error: false, - }, - TestData { - id: "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ.-_", - expect_error: false, - }, - TestData { - id: " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_", - expect_error: true, - }, - TestData { - id: ".ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_", - expect_error: true, - }, - TestData { - id: "/a/b/c", - expect_error: true, - }, - TestData { - id: "a/b/c", - expect_error: true, - }, - TestData { - id: "foo/../../../etc/passwd", - expect_error: true, - }, - TestData { - id: "../../../../../../etc/motd", - expect_error: true, - }, - TestData { - id: "/etc/passwd", - expect_error: true, - }, - ]; - - for (i, d) in tests.iter().enumerate() { - let msg = format!("test[{}]: {:?}", i, d); - - let result = verify_cid(d.id); - - let msg = format!("{}, result: {:?}", msg, result); - - if result.is_ok() { - assert!(!d.expect_error, "{}", msg); - } else { - assert!(d.expect_error, "{}", msg); - } - } - } - #[tokio::test] async fn test_volume_capacity_stats() { skip_if_not_root!(); diff --git a/src/libs/kata-sys-util/src/fs.rs b/src/libs/kata-sys-util/src/fs.rs index a875b832d432..fb056e1324f9 100644 --- a/src/libs/kata-sys-util/src/fs.rs +++ b/src/libs/kata-sys-util/src/fs.rs @@ -13,8 +13,16 @@ use std::process::Command; use crate::{eother, sl}; +// nix filesystem_type for different target_os +#[cfg(all(target_os = "linux", target_env = "musl"))] +type FsType = libc::c_ulong; +#[cfg(all(target_os = "linux", not(any(target_env = "musl"))))] +type FsType = libc::__fsword_t; + // from linux.git/fs/fuse/inode.c: #define FUSE_SUPER_MAGIC 0x65735546 -const FUSE_SUPER_MAGIC: u32 = 0x65735546; +const FUSE_SUPER_MAGIC: FsType = 0x65735546; +// from linux.git/include/uapi/linux/magic.h +const OVERLAYFS_SUPER_MAGIC: FsType = 0x794c7630; /// Get bundle path (current working directory). pub fn get_bundle_path() -> Result { @@ -35,7 +43,7 @@ pub fn get_base_name>(src: P) -> Result { /// Check whether `path` is on a fuse filesystem. pub fn is_fuse_fs>(path: P) -> bool { if let Ok(st) = nix::sys::statfs::statfs(path.as_ref()) { - if st.filesystem_type().0 == FUSE_SUPER_MAGIC as i64 { + if st.filesystem_type().0 == FUSE_SUPER_MAGIC { return true; } } @@ -45,7 +53,7 @@ pub fn is_fuse_fs>(path: P) -> bool { /// Check whether `path` is on a overlay filesystem. pub fn is_overlay_fs>(path: P) -> bool { if let Ok(st) = nix::sys::statfs::statfs(path.as_ref()) { - if st.filesystem_type() == nix::sys::statfs::OVERLAYFS_SUPER_MAGIC { + if st.filesystem_type().0 == OVERLAYFS_SUPER_MAGIC { return true; } } diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 4efc4fb8cde2..606ebf6bde29 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -13,6 +13,7 @@ pub mod hooks; pub mod k8s; pub mod mount; pub mod numa; +pub mod validate; // Convenience macro to obtain the scoped logger #[macro_export] diff --git a/src/libs/kata-sys-util/src/validate.rs b/src/libs/kata-sys-util/src/validate.rs new file mode 100644 index 000000000000..a58b19289ad6 --- /dev/null +++ b/src/libs/kata-sys-util/src/validate.rs @@ -0,0 +1,267 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("invalid container ID {0}")] + InvalidContainerID(String), +} + +// A container ID must match this regex: +// +// ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$ +// +pub fn verify_cid(id: &str) -> Result<(), Error> { + let mut chars = id.chars(); + + let valid = match chars.next() { + Some(first) + if first.is_alphanumeric() + && id.len() > 1 + && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)) => + { + true + } + _ => false, + }; + + match valid { + true => Ok(()), + false => Err(Error::InvalidContainerID(id.to_string())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_verify_cid() { + #[derive(Debug)] + struct TestData<'a> { + id: &'a str, + expect_error: bool, + } + + let tests = &[ + TestData { + // Cannot be blank + id: "", + expect_error: true, + }, + TestData { + // Cannot be a space + id: " ", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: ".", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: "-", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: "_", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: " a", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: ".a", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: "-a", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: "_a", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: "..", + expect_error: true, + }, + TestData { + // Too short + id: "a", + expect_error: true, + }, + TestData { + // Too short + id: "z", + expect_error: true, + }, + TestData { + // Too short + id: "A", + expect_error: true, + }, + TestData { + // Too short + id: "Z", + expect_error: true, + }, + TestData { + // Too short + id: "0", + expect_error: true, + }, + TestData { + // Too short + id: "9", + expect_error: true, + }, + TestData { + // Must start with an alphanumeric + id: "-1", + expect_error: true, + }, + TestData { + id: "/", + expect_error: true, + }, + TestData { + id: "a/", + expect_error: true, + }, + TestData { + id: "a/../", + expect_error: true, + }, + TestData { + id: "../a", + expect_error: true, + }, + TestData { + id: "../../a", + expect_error: true, + }, + TestData { + id: "../../../a", + expect_error: true, + }, + TestData { + id: "foo/../bar", + expect_error: true, + }, + TestData { + id: "foo bar", + expect_error: true, + }, + TestData { + id: "a.", + expect_error: false, + }, + TestData { + id: "a..", + expect_error: false, + }, + TestData { + id: "aa", + expect_error: false, + }, + TestData { + id: "aa.", + expect_error: false, + }, + TestData { + id: "hello..world", + expect_error: false, + }, + TestData { + id: "hello/../world", + expect_error: true, + }, + TestData { + id: "aa1245124sadfasdfgasdga.", + expect_error: false, + }, + TestData { + id: "aAzZ0123456789_.-", + expect_error: false, + }, + TestData { + id: "abcdefghijklmnopqrstuvwxyz0123456789.-_", + expect_error: false, + }, + TestData { + id: "0123456789abcdefghijklmnopqrstuvwxyz.-_", + expect_error: false, + }, + TestData { + id: " abcdefghijklmnopqrstuvwxyz0123456789.-_", + expect_error: true, + }, + TestData { + id: ".abcdefghijklmnopqrstuvwxyz0123456789.-_", + expect_error: true, + }, + TestData { + id: "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_", + expect_error: false, + }, + TestData { + id: "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ.-_", + expect_error: false, + }, + TestData { + id: " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_", + expect_error: true, + }, + TestData { + id: ".ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_", + expect_error: true, + }, + TestData { + id: "/a/b/c", + expect_error: true, + }, + TestData { + id: "a/b/c", + expect_error: true, + }, + TestData { + id: "foo/../../../etc/passwd", + expect_error: true, + }, + TestData { + id: "../../../../../../etc/motd", + expect_error: true, + }, + TestData { + id: "/etc/passwd", + expect_error: true, + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let result = verify_cid(d.id); + + let msg = format!("{}, result: {:?}", msg, result); + + if result.is_ok() { + assert!(!d.expect_error, "{}", msg); + } else { + assert!(d.expect_error, "{}", msg); + } + } + } +} diff --git a/src/libs/kata-types/.gitignore b/src/libs/kata-types/.gitignore new file mode 100644 index 000000000000..03314f77b5aa --- /dev/null +++ b/src/libs/kata-types/.gitignore @@ -0,0 +1 @@ +Cargo.lock diff --git a/src/libs/kata-types/src/container.rs b/src/libs/kata-types/src/container.rs index aee94f95d37c..3a64a4dd7fae 100644 --- a/src/libs/kata-types/src/container.rs +++ b/src/libs/kata-types/src/container.rs @@ -7,11 +7,17 @@ use std::fmt::{Display, Formatter}; use std::str::FromStr; -const CONTAINER: &str = "container"; -const SANDBOX: &str = "sandbox"; -const POD_CONTAINER: &str = "pod_container"; -const POD_SANDBOX: &str = "pod_sandbox"; -const POD_SANDBOX2: &str = "podsandbox"; +// a container running within a pod +pub(crate) const POD_CONTAINER: &str = "pod_container"; +// cri containerd/crio/docker: a container running within a pod +pub(crate) const CONTAINER: &str = "container"; + +// a pod sandbox container +pub(crate) const POD_SANDBOX: &str = "pod_sandbox"; +// cri containerd/crio: a pod sandbox container +pub(crate) const SANDBOX: &str = "sandbox"; +// docker: a sandbox sandbox container +pub(crate) const PODSANDBOX: &str = "podsandbox"; const STATE_READY: &str = "ready"; const STATE_RUNNING: &str = "running"; @@ -68,7 +74,7 @@ impl FromStr for ContainerType { fn from_str(value: &str) -> Result { match value { POD_CONTAINER | CONTAINER => Ok(ContainerType::PodContainer), - POD_SANDBOX | POD_SANDBOX2 | SANDBOX => Ok(ContainerType::PodSandbox), + POD_SANDBOX | PODSANDBOX | SANDBOX => Ok(ContainerType::PodSandbox), _ => Err(Error::InvalidContainerType(value.to_owned())), } } diff --git a/src/libs/kata-types/src/k8s.rs b/src/libs/kata-types/src/k8s.rs index 43f5ba839d24..7e53601b882f 100644 --- a/src/libs/kata-types/src/k8s.rs +++ b/src/libs/kata-types/src/k8s.rs @@ -56,28 +56,31 @@ pub fn container_type(spec: &oci::Spec) -> ContainerType { /// Determine the k8s sandbox ID from OCI annotations. /// /// This function is expected to be called only when the container type is "PodContainer". -pub fn sandbox_id(spec: &oci::Spec) -> Result, String> { - if container_type(spec) != ContainerType::PodSandbox { - return Err("Not a sandbox container".to_string()); - } - for k in [ - annotations::crio::SANDBOX_ID_LABEL_KEY, - annotations::cri_containerd::SANDBOX_ID_LABEL_KEY, - annotations::dockershim::SANDBOX_ID_LABEL_KEY, - ] - .iter() - { - if let Some(id) = spec.annotations.get(k.to_owned()) { - return Ok(Some(id.to_string())); +pub fn container_type_with_id(spec: &oci::Spec) -> (ContainerType, Option) { + let container_type = container_type(spec); + let mut sid = None; + if container_type == ContainerType::PodContainer { + for k in [ + annotations::crio::SANDBOX_ID_LABEL_KEY, + annotations::cri_containerd::SANDBOX_ID_LABEL_KEY, + annotations::dockershim::SANDBOX_ID_LABEL_KEY, + ] + .iter() + { + if let Some(id) = spec.annotations.get(k.to_owned()) { + sid = Some(id.to_string()); + break; + } } } - Ok(None) + (container_type, sid) } #[cfg(test)] mod tests { use super::*; + use crate::{annotations, container}; #[test] fn test_is_empty_dir() { @@ -99,4 +102,112 @@ mod tests { let empty_dir = "/kubernetes.io~empty-dir/shm"; assert!(is_empty_dir(empty_dir)); } + + #[test] + fn test_container_type() { + let sid = "sid".to_string(); + let mut spec = oci::Spec::default(); + + // default + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodSandbox, None) + ); + + // crio sandbox + spec.annotations = [( + annotations::crio::CONTAINER_TYPE_LABEL_KEY.to_string(), + container::SANDBOX.to_string(), + )] + .iter() + .cloned() + .collect(); + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodSandbox, None) + ); + + // cri containerd sandbox + spec.annotations = [( + annotations::crio::CONTAINER_TYPE_LABEL_KEY.to_string(), + container::POD_SANDBOX.to_string(), + )] + .iter() + .cloned() + .collect(); + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodSandbox, None) + ); + + // docker shim sandbox + spec.annotations = [( + annotations::crio::CONTAINER_TYPE_LABEL_KEY.to_string(), + container::PODSANDBOX.to_string(), + )] + .iter() + .cloned() + .collect(); + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodSandbox, None) + ); + + // crio container + spec.annotations = [ + ( + annotations::crio::CONTAINER_TYPE_LABEL_KEY.to_string(), + container::CONTAINER.to_string(), + ), + ( + annotations::crio::SANDBOX_ID_LABEL_KEY.to_string(), + sid.clone(), + ), + ] + .iter() + .cloned() + .collect(); + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodContainer, Some(sid.clone())) + ); + + // cri containerd container + spec.annotations = [ + ( + annotations::cri_containerd::CONTAINER_TYPE_LABEL_KEY.to_string(), + container::POD_CONTAINER.to_string(), + ), + ( + annotations::cri_containerd::SANDBOX_ID_LABEL_KEY.to_string(), + sid.clone(), + ), + ] + .iter() + .cloned() + .collect(); + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodContainer, Some(sid.clone())) + ); + + // docker shim container + spec.annotations = [ + ( + annotations::dockershim::CONTAINER_TYPE_LABEL_KEY.to_string(), + container::CONTAINER.to_string(), + ), + ( + annotations::dockershim::SANDBOX_ID_LABEL_KEY.to_string(), + sid.clone(), + ), + ] + .iter() + .cloned() + .collect(); + assert_eq!( + container_type_with_id(&spec), + (ContainerType::PodContainer, Some(sid)) + ); + } } diff --git a/src/libs/protocols/.gitignore b/src/libs/protocols/.gitignore index ce4964c4f0df..0a83b1689a97 100644 --- a/src/libs/protocols/.gitignore +++ b/src/libs/protocols/.gitignore @@ -1,9 +1,11 @@ Cargo.lock src/agent.rs src/agent_ttrpc.rs +src/agent_ttrpc_async.rs src/csi.rs src/empty.rs src/health.rs src/health_ttrpc.rs +src/health_ttrpc_async.rs src/oci.rs src/types.rs From 278f843f9256e50306543d8c2b576147a80b2812 Mon Sep 17 00:00:00 2001 From: Zack Date: Tue, 22 Feb 2022 15:39:56 +0800 Subject: [PATCH 0037/1953] runtime-rs: shim implements for runtime-rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Responsible for processing shim related commands: start, delete. This patch is extracted from Alibaba Cloud's internal repository *runD* Thanks to all contributors! Fixes: #3785 Signed-off-by: acetang Signed-off-by: Bin Liu Signed-off-by: Chao Wu Signed-off-by: Eryu Guan Signed-off-by: Fupan Li  Signed-off-by: gexuyang Signed-off-by: Helin Guo Signed-off-by: He Rongguang Signed-off-by: Hui Zhu Signed-off-by: Issac Hai Signed-off-by: Jiahuan Chao Signed-off-by: lichenglong9 Signed-off-by: mengze Signed-off-by: Qingyuan Hou Signed-off-by: Quanwei Zhou Signed-off-by: shiqiangzhang Signed-off-by: Simon Guo Signed-off-by: Tim Zhang Signed-off-by: wanglei01 Signed-off-by: Wei Yang Signed-off-by: yanlei Signed-off-by: Yiqun Leng Signed-off-by: yuchang.xu Signed-off-by: Yves Chan Signed-off-by: Zack Signed-off-by: Zhiheng Tao Signed-off-by: Zhongtao Hu Signed-off-by: Zizheng Bian --- Makefile | 1 + src/runtime-rs/.gitignore | 2 + src/runtime-rs/Cargo.lock | 1423 +++++++++++++++++ src/runtime-rs/Cargo.toml | 4 + src/runtime-rs/Makefile | 171 ++ src/runtime-rs/VERSION | 1 + src/runtime-rs/crates/shim/Cargo.toml | 46 + src/runtime-rs/crates/shim/build.rs | 12 + src/runtime-rs/crates/shim/src/args.rs | 325 ++++ src/runtime-rs/crates/shim/src/bin/main.rs | 192 +++ src/runtime-rs/crates/shim/src/config.rs.in | 19 + src/runtime-rs/crates/shim/src/error.rs | 52 + src/runtime-rs/crates/shim/src/lib.rs | 28 + src/runtime-rs/crates/shim/src/logger.rs | 41 + src/runtime-rs/crates/shim/src/panic_hook.rs | 41 + src/runtime-rs/crates/shim/src/shim.rs | 116 ++ src/runtime-rs/crates/shim/src/shim_delete.rs | 71 + src/runtime-rs/crates/shim/src/shim_run.rs | 54 + src/runtime-rs/crates/shim/src/shim_start.rs | 238 +++ .../tests/texture/image-bundle/config.json | 395 +++++ .../kata-containers-configuration.toml | 11 + src/runtime-rs/tests/utils/Cargo.toml | 10 + src/runtime-rs/tests/utils/src/lib.rs | 35 + 23 files changed, 3288 insertions(+) create mode 100644 src/runtime-rs/.gitignore create mode 100644 src/runtime-rs/Cargo.lock create mode 100644 src/runtime-rs/Cargo.toml create mode 100644 src/runtime-rs/Makefile create mode 120000 src/runtime-rs/VERSION create mode 100644 src/runtime-rs/crates/shim/Cargo.toml create mode 100644 src/runtime-rs/crates/shim/build.rs create mode 100644 src/runtime-rs/crates/shim/src/args.rs create mode 100644 src/runtime-rs/crates/shim/src/bin/main.rs create mode 100644 src/runtime-rs/crates/shim/src/config.rs.in create mode 100644 src/runtime-rs/crates/shim/src/error.rs create mode 100644 src/runtime-rs/crates/shim/src/lib.rs create mode 100644 src/runtime-rs/crates/shim/src/logger.rs create mode 100644 src/runtime-rs/crates/shim/src/panic_hook.rs create mode 100644 src/runtime-rs/crates/shim/src/shim.rs create mode 100644 src/runtime-rs/crates/shim/src/shim_delete.rs create mode 100644 src/runtime-rs/crates/shim/src/shim_run.rs create mode 100644 src/runtime-rs/crates/shim/src/shim_start.rs create mode 100644 src/runtime-rs/tests/texture/image-bundle/config.json create mode 100644 src/runtime-rs/tests/texture/kata-containers-configuration.toml create mode 100644 src/runtime-rs/tests/utils/Cargo.toml create mode 100644 src/runtime-rs/tests/utils/src/lib.rs diff --git a/Makefile b/Makefile index 7411a4c29707..4d2be6b4d829 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ COMPONENTS = COMPONENTS += libs COMPONENTS += agent COMPONENTS += runtime +COMPONENTS += runtime-rs # List of available tools TOOLS = diff --git a/src/runtime-rs/.gitignore b/src/runtime-rs/.gitignore new file mode 100644 index 000000000000..82cc0b64ca97 --- /dev/null +++ b/src/runtime-rs/.gitignore @@ -0,0 +1,2 @@ +target +crates/shim/src/config.rs diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock new file mode 100644 index 000000000000..c55b571a666f --- /dev/null +++ b/src/runtime-rs/Cargo.lock @@ -0,0 +1,1423 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd" + +[[package]] +name = "arc-swap" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" + +[[package]] +name = "async-trait" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cgroups-rs" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdae996d9638ba03253ffa1c93345a585974a97abbdeab9176c77922f3efc1e8" +dependencies = [ + "libc", + "log", + "nix 0.23.1", + "regex", +] + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time 0.1.43", + "winapi", +] + +[[package]] +name = "common-path" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" + +[[package]] +name = "containerd-shim-protos" +version = "0.1.2" +source = "git+https://github.com/containerd/rust-extensions.git?rev=c0baac598fc3ad62f651e8aae8de15db2ce5695c#c0baac598fc3ad62f651e8aae8de15db2ce5695c" +dependencies = [ + "async-trait", + "protobuf", + "ttrpc", +] + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crossbeam-channel" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "enum-iterator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fail" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" +dependencies = [ + "lazy_static", + "log", + "rand", +] + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "getset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "gimli" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" + +[[package]] +name = "git2" +version = "0.13.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "go-flag" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4a40c9ca507513f573aabaf6a8558173a1ac9aa1363d8de30c7f89b34f8d2b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + +[[package]] +name = "kata-sys-util" +version = "0.1.0" +dependencies = [ + "cgroups-rs", + "chrono", + "common-path", + "fail", + "kata-types", + "lazy_static", + "libc", + "nix 0.23.1", + "oci", + "once_cell", + "serde_json", + "slog", + "slog-scope", + "subprocess", + "thiserror", +] + +[[package]] +name = "kata-types" +version = "0.1.0" +dependencies = [ + "glob", + "lazy_static", + "num_cpus", + "oci", + "regex", + "serde", + "serde_json", + "slog", + "slog-scope", + "thiserror", + "toml", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" + +[[package]] +name = "libgit2-sys" +version = "0.12.26+1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "logging" +version = "0.1.0" +dependencies = [ + "serde_json", + "slog", + "slog-async", + "slog-json", + "slog-scope", +] + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +dependencies = [ + "memchr", +] + +[[package]] +name = "oci" +version = "0.1.0" +dependencies = [ + "libc", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "protobuf" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + +[[package]] +name = "protobuf-codegen" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" +dependencies = [ + "protobuf", +] + +[[package]] +name = "protobuf-codegen-pure" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" +dependencies = [ + "protobuf", + "protobuf-codegen", +] + +[[package]] +name = "quote" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" + +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serial_test" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d" +dependencies = [ + "lazy_static", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool", + "digest", + "opaque-debug", +] + +[[package]] +name = "shim" +version = "0.1.0" +dependencies = [ + "anyhow", + "backtrace", + "containerd-shim-protos", + "go-flag", + "kata-sys-util", + "kata-types", + "libc", + "log", + "logging", + "nix 0.16.1", + "oci", + "protobuf", + "serial_test", + "sha2", + "slog", + "slog-async", + "slog-scope", + "slog-stdlog", + "tempfile", + "tests_utils", + "thiserror", + "tokio", + "unix_socket2", + "vergen", +] + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-json" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f825ce7346f40aa318111df5d3a94945a7fdca9081584cb9b05692fb3dfcb4" +dependencies = [ + "serde", + "serde_json", + "slog", + "time 0.3.7", +] + +[[package]] +name = "slog-scope" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" +dependencies = [ + "arc-swap", + "lazy_static", + "slog", +] + +[[package]] +name = "slog-stdlog" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8228ab7302adbf4fcb37e66f3cda78003feb521e7fd9e3847ec117a7784d0f5a" +dependencies = [ + "log", + "slog", + "slog-scope", +] + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "subprocess" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "tests_utils" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" +dependencies = [ + "itoa", + "libc", + "num_threads", +] + +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +dependencies = [ + "bytes 1.1.0", + "libc", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-vsock" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6" +dependencies = [ + "bytes 0.4.12", + "futures", + "libc", + "tokio", + "vsock", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "ttrpc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7d6c992964a013c17814c08d31708d577b0aae44ebadb58755659dd824c2d1" +dependencies = [ + "async-trait", + "byteorder", + "futures", + "libc", + "log", + "nix 0.23.1", + "protobuf", + "protobuf-codegen-pure", + "thiserror", + "tokio", + "tokio-vsock", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unix_socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b57c6eace16c00eccb98a28e85db3370eab0685bdd5e13831d59e2bcb49a1d8a" +dependencies = [ + "libc", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vergen" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3893329bee75c101278e0234b646fa72221547d63f97fb66ac112a0569acd110" +dependencies = [ + "anyhow", + "cfg-if 1.0.0", + "chrono", + "enum-iterator", + "getset", + "git2", + "rustc_version", + "rustversion", + "thiserror", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "vsock" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" +dependencies = [ + "libc", + "nix 0.23.1", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml new file mode 100644 index 000000000000..414b707258dd --- /dev/null +++ b/src/runtime-rs/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = [ + "crates/shim", +] diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile new file mode 100644 index 000000000000..5585e844a49a --- /dev/null +++ b/src/runtime-rs/Makefile @@ -0,0 +1,171 @@ +# Copyright (c) 2019-2022 Alibaba Cloud +# Copyright (c) 2019-2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +# To show variables or targets help on `make help` +# Use the following format: +# '##VAR VARIABLE_NAME: help about variable' +# '##TARGET TARGET_NAME: help about target' + +PROJECT_NAME = Kata Containers +PROJECT_URL = https://github.com/kata-containers +PROJECT_COMPONENT = containerd-shim-kata-v2 +CONTAINERD_RUNTIME_NAME = io.containerd.kata.v2 + +TARGET = $(PROJECT_COMPONENT) + +SOURCES := \ + $(shell find . 2>&1 | grep -E '.*\.rs$$') \ + Cargo.toml + +VERSION_FILE := ./VERSION +VERSION := $(shell grep -v ^\# $(VERSION_FILE)) +COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true) +COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2>/dev/null || true),${COMMIT_NO}-dirty,${COMMIT_NO}) +COMMIT_MSG = $(if $(COMMIT),$(COMMIT),unknown) + +# Exported to allow cargo to see it +export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) + +EXTRA_RUSTFEATURES := + +ifneq ($(EXTRA_RUSTFEATURES),) + override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES) +endif + +include ../../utils.mk + +TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) + +##VAR DESTDIR= is a directory prepended to each installed target file +DESTDIR := +##VAR BINDIR= is a directory for installing executable programs +BINDIR := /usr/bin + +GENERATED_CODE = crates/shim/src/config.rs + +RUNTIME_NAME=$(TARGET) +RUNTIME_VERSION=$(VERSION) + +GENERATED_REPLACEMENTS= \ + PROJECT_NAME \ + RUNTIME_NAME \ + CONTAINERD_RUNTIME_NAME \ + RUNTIME_VERSION \ + BINDIR \ + COMMIT \ + VERSION_COMMIT +GENERATED_FILES := + +GENERATED_FILES += $(GENERATED_CODE) + +# Display name of command and it's version (or a message if not available). +# +# Arguments: +# +# 1: Name of command +define get_command_version +$(shell printf "%s: %s\\n" $(1) "$(or $(shell $(1) --version 2>/dev/null), (not available))") +endef + +define get_toolchain_version +$(shell printf "%s: %s\\n" "toolchain" "$(or $(shell rustup show active-toolchain 2>/dev/null), (unknown))") +endef + +define INSTALL_FILE + install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1; +endef + +.DEFAULT_GOAL := default + +##TARGET default: build code +default: $(TARGET) show-header + +$(TARGET): $(GENERATED_CODE) $(TARGET_PATH) + +$(TARGET_PATH): $(SOURCES) | show-summary + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) + +$(GENERATED_FILES): %: %.in + @sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@" + +##TARGET optimize: optimized build +optimize: $(SOURCES) | show-summary show-header + @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) + +##TARGET clean: clean build +clean: + @cargo clean + @rm -f $(GENERATED_FILES) + @rm -f tarpaulin-report.html + +vendor: + @cargo vendor + +#TARGET test: run cargo tests +test: + @cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture + +##TARGET check: run test +check: $(GENERATED_FILES) standard_rust_check + +##TARGET run: build and run agent +run: + @cargo run --target $(TRIPLE) + +show-header: + @printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)" + +show-summary: show-header + @printf "project:\n" + @printf " name: $(PROJECT_NAME)\n" + @printf " url: $(PROJECT_URL)\n" + @printf " component: $(PROJECT_COMPONENT)\n" + @printf "target: $(TARGET)\n" + @printf "architecture:\n" + @printf " host: $(ARCH)\n" + @printf "rust:\n" + @printf " %s\n" "$(call get_command_version,cargo)" + @printf " %s\n" "$(call get_command_version,rustc)" + @printf " %s\n" "$(call get_command_version,rustup)" + @printf " %s\n" "$(call get_toolchain_version)" + @printf "\n" + +## help: Show help comments that start with `##VAR` and `##TARGET` +help: Makefile show-summary + @echo "========================== Help =============================" + @echo "Variables:" + @sed -n 's/^##VAR//p' $< | sort + @echo "" + @echo "Targets:" + @sed -n 's/^##TARGET//p' $< | sort + +TARPAULIN_ARGS:=-v --workspace +install-tarpaulin: + cargo install cargo-tarpaulin + +# Check if cargo tarpaulin is installed +HAS_TARPAULIN:= $(shell cargo --list | grep tarpaulin 2>/dev/null) +check_tarpaulin: +ifndef HAS_TARPAULIN + $(error "tarpaulin is not available please: run make install-tarpaulin ") +else + $(info OK: tarpaulin installed) +endif + +##TARGET codecov: Generate code coverage report +codecov: check_tarpaulin + cargo tarpaulin $(TARPAULIN_ARGS) + +##TARGET codecov-html: Generate code coverage html report +codecov-html: check_tarpaulin + cargo tarpaulin $(TARPAULIN_ARGS) -o Html + +.PHONY: \ + help \ + optimize \ + show-header \ + show-summary \ + vendor diff --git a/src/runtime-rs/VERSION b/src/runtime-rs/VERSION new file mode 120000 index 000000000000..558194c5a5a5 --- /dev/null +++ b/src/runtime-rs/VERSION @@ -0,0 +1 @@ +../../VERSION \ No newline at end of file diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml new file mode 100644 index 000000000000..c208eb3187d6 --- /dev/null +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "shim" +version = "0.1.0" +authors = ["The Kata Containers community "] +description = "Containerd shim runtime for Kata Containers" +keywords = ["kata-containers", "shim"] +repository = "https://github.com/kata-containers/kata-containers.git" +license = "Apache-2.0" +edition = "2018" + +[[bin]] +name = "containerd-shim-kata-v2" +path = "src/bin/main.rs" + +[dependencies] +anyhow = "^1.0" +backtrace = {version = ">=0.3.35", features = ["libunwind", "libbacktrace", "std"], default-features = false} +# TODO: change to version after release +# issue: https://github.com/kata-containers/kata-containers/issues/3866 +containerd-shim-protos = { git="https://github.com/containerd/rust-extensions.git", rev = "c0baac598fc3ad62f651e8aae8de15db2ce5695c", features = ["async"]} +go-flag = "0.1.0" +libc = "0.2.108" +log = "0.4.14" +nix = "0.16.0" +protobuf = "2.23.0" +sha2 = "=0.9.3" +slog = {version = "2.7.0", features = ["std", "release_max_level_trace", "max_level_trace"]} +slog-async = "2.7.0" +slog-scope = "4.4.0" +slog-stdlog = "4.1.0" +thiserror = "1.0.30" +tokio = { version = "1.8.0", features = [ "rt", "rt-multi-thread" ] } +unix_socket2 = "0.5.4" + +kata-types = { path = "../../../libs/kata-types"} +kata-sys-util = { path = "../../../libs/kata-sys-util"} +logging = { path = "../../../libs/logging"} +oci = { path = "../../../libs/oci" } + +[build-dependencies] +vergen = { version = "6", default-features = false, features = ["build", "git", "rustc"] } + +[dev-dependencies] +tempfile = "3.2.0" +serial_test = "0.5.1" +tests_utils = { path = "../../tests/utils"} diff --git a/src/runtime-rs/crates/shim/build.rs b/src/runtime-rs/crates/shim/build.rs new file mode 100644 index 000000000000..6fd7ff9a9942 --- /dev/null +++ b/src/runtime-rs/crates/shim/build.rs @@ -0,0 +1,12 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use vergen::{vergen, Config}; + +fn main() { + // Generate the default 'cargo:' instruction output + vergen(Config::default()).unwrap(); +} diff --git a/src/runtime-rs/crates/shim/src/args.rs b/src/runtime-rs/crates/shim/src/args.rs new file mode 100644 index 000000000000..62b9653cca32 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/args.rs @@ -0,0 +1,325 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{os::unix::fs::FileTypeExt, path::PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use kata_sys_util::validate; + +use crate::Error; + +/// Received command-line arguments or environment arguments +/// from a shimv2 container manager such as containerd. +/// +/// For detailed information, please refer to the +/// [shim spec](https://github.com/containerd/containerd/blob/main/runtime/v2/README.md). +#[derive(Debug, Default, Clone)] +pub struct Args { + /// the id of the container + pub id: String, + /// the namespace for the container + pub namespace: String, + /// the address of the containerd's main socket + pub address: String, + /// the binary path to publish events back to containerd + pub publish_binary: String, + /// Abstract socket path to serve. + pub socket: String, + /// the path to the bundle to delete + pub bundle: String, + /// Whether or not to enable debug + pub debug: bool, +} + +impl Args { + /// Check the shim argument object is vaild or not. + /// + /// The id, namespace, address and publish_binary are mandatory for START, RUN and DELETE. + /// And bundle is mandatory for DELETE. + pub fn validate(&mut self, should_check_bundle: bool) -> Result<()> { + if self.id.is_empty() + || self.namespace.is_empty() + || self.address.is_empty() + || self.publish_binary.is_empty() + { + return Err(anyhow!(Error::ArgumentIsEmpty(format!( + "id: {} namespace: {} address: {} publish_binary: {}", + &self.id, &self.namespace, &self.address, &self.publish_binary + )))); + } + + validate::verify_cid(&self.id).context("verify cid")?; + validate::verify_cid(&self.namespace).context("verify namespace")?; + + // Ensure `address` is a valid path. + let path = PathBuf::from(self.address.clone()) + .canonicalize() + .context(Error::InvalidPath(self.address.clone()))?; + let md = path + .metadata() + .context(Error::FileGetMetadata(format!("{:?}", path)))?; + if !md.file_type().is_socket() { + return Err(Error::InvalidArgument).context("address is not socket"); + } + self.address = path + .to_str() + .map(|v| v.to_owned()) + .ok_or(Error::InvalidArgument)?; + + // Ensure `bundle` is a valid path. + if should_check_bundle { + if self.bundle.is_empty() { + return Err(anyhow!(Error::ArgumentIsEmpty("bundle".to_string()))); + } + + let path = PathBuf::from(self.bundle.clone()) + .canonicalize() + .map_err(|_| Error::InvalidArgument)?; + let md = path + .metadata() + .map_err(|_| Error::InvalidArgument) + .context("get address metadata")?; + if !md.is_dir() { + return Err(Error::InvalidArgument).context("medata is dir"); + } + self.bundle = path + .to_str() + .map(|v| v.to_owned()) + .ok_or(Error::InvalidArgument) + .context("path to string")?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::net::UnixListener; + + use anyhow::anyhow; + use kata_sys_util::validate; + + #[test] + fn test_args_is_valid() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_path_buf(); + let path = path.to_str().unwrap(); + let bind_address = &format!("{}/socket1", path); + UnixListener::bind(bind_address).unwrap(); + + #[derive(Debug)] + struct TestData { + arg: Args, + should_check_bundle: bool, + result: Result<()>, + } + + let default_id = "1dfc0567".to_string(); + let default_namespace = "ns1".to_string(); + let default_address = bind_address.to_string(); + let default_publish_binary = "containerd".to_string(); + let default_socket = "socket".to_string(); + let default_bundle = path.to_string(); + let default_debug = false; + + let mut arg = Args { + id: default_id.clone(), + namespace: default_namespace.clone(), + address: default_address.clone(), + publish_binary: default_publish_binary.clone(), + socket: default_socket, + bundle: default_bundle.clone(), + debug: default_debug, + }; + + let tests = &[ + TestData { + arg: arg.clone(), + should_check_bundle: false, + result: Ok(()), + }, + TestData { + arg: { + arg.namespace = "".to_string(); + arg.clone() + }, + should_check_bundle: false, + result: Err(anyhow!(Error::ArgumentIsEmpty(format!( + "id: {} namespace: {} address: {} publish_binary: {}", + &arg.id, &arg.namespace, &arg.address, &arg.publish_binary + )))), + }, + TestData { + arg: { + arg.namespace = default_namespace.clone(); + arg.clone() + }, + should_check_bundle: false, + result: Ok(()), + }, + TestData { + arg: { + arg.id = "".to_string(); + arg.clone() + }, + should_check_bundle: false, + result: Err(anyhow!(Error::ArgumentIsEmpty(format!( + "id: {} namespace: {} address: {} publish_binary: {}", + &arg.id, &arg.namespace, &arg.address, &arg.publish_binary + )))), + }, + TestData { + arg: { + arg.id = default_id; + arg.clone() + }, + should_check_bundle: false, + result: Ok(()), + }, + TestData { + arg: { + arg.address = "".to_string(); + arg.clone() + }, + should_check_bundle: false, + result: Err(anyhow!(Error::ArgumentIsEmpty(format!( + "id: {} namespace: {} address: {} publish_binary: {}", + &arg.id, &arg.namespace, &arg.address, &arg.publish_binary + )))), + }, + TestData { + arg: { + arg.address = default_address.clone(); + arg.clone() + }, + should_check_bundle: false, + result: Ok(()), + }, + TestData { + arg: { + arg.publish_binary = "".to_string(); + arg.clone() + }, + should_check_bundle: false, + result: Err(anyhow!(Error::ArgumentIsEmpty(format!( + "id: {} namespace: {} address: {} publish_binary: {}", + &arg.id, &arg.namespace, &arg.address, &arg.publish_binary + )))), + }, + TestData { + arg: { + arg.publish_binary = default_publish_binary; + arg.clone() + }, + should_check_bundle: false, + result: Ok(()), + }, + TestData { + arg: { + arg.bundle = "".to_string(); + arg.clone() + }, + should_check_bundle: false, + result: Ok(()), + }, + TestData { + arg: arg.clone(), + should_check_bundle: true, + result: Err(anyhow!(Error::ArgumentIsEmpty("bundle".to_string()))), + }, + TestData { + arg: { + arg.bundle = default_bundle; + arg.clone() + }, + should_check_bundle: true, + result: Ok(()), + }, + TestData { + arg: { + arg.namespace = "id1/id2".to_string(); + arg.clone() + }, + should_check_bundle: true, + result: Err( + anyhow!(validate::Error::InvalidContainerID("id/id2".to_string())) + .context("verify namespace"), + ), + }, + TestData { + arg: { + arg.namespace = default_namespace.clone() + "id1 id2"; + arg.clone() + }, + should_check_bundle: true, + result: Err(anyhow!(validate::Error::InvalidContainerID( + default_namespace.clone() + "id1 id2", + )) + .context("verify namespace")), + }, + TestData { + arg: { + arg.namespace = default_namespace.clone() + "id2\tid2"; + arg.clone() + }, + should_check_bundle: true, + result: Err(anyhow!(validate::Error::InvalidContainerID( + default_namespace.clone() + "id1\tid2", + )) + .context("verify namespace")), + }, + TestData { + arg: { + arg.namespace = default_namespace; + arg.clone() + }, + should_check_bundle: true, + result: Ok(()), + }, + TestData { + arg: { + arg.address = default_address.clone() + "/.."; + arg.clone() + }, + should_check_bundle: true, + result: Err(anyhow!(Error::InvalidPath(arg.address.clone()))), + }, + TestData { + arg: { + arg.address = default_address.clone() + "/.."; + arg.clone() + }, + should_check_bundle: true, + result: Err(anyhow!(Error::InvalidPath(arg.address.clone()))), + }, + TestData { + arg: { + arg.address = default_address; + arg + }, + should_check_bundle: true, + result: Ok(()), + }, + ]; + + for (i, t) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, t); + let should_check_bundle = t.should_check_bundle; + let result = t.arg.clone().validate(should_check_bundle); + let msg = format!("{}, result: {:?}", msg, result); + + if t.result.is_ok() { + assert!(result.is_ok(), "{}", msg); + } else { + let expected_error = format!("{}", t.result.as_ref().unwrap_err()); + let actual_error = format!("{}", result.unwrap_err()); + assert!(actual_error == expected_error, "{}", msg); + } + } + } +} diff --git a/src/runtime-rs/crates/shim/src/bin/main.rs b/src/runtime-rs/crates/shim/src/bin/main.rs new file mode 100644 index 000000000000..1526a89cd376 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/bin/main.rs @@ -0,0 +1,192 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + ffi::{OsStr, OsString}, + path::PathBuf, +}; + +use anyhow::{anyhow, Context, Result}; +use nix::{ + mount::{mount, MsFlags}, + sched::{self, CloneFlags}, +}; +use shim::{config, Args, Error, ShimExecutor}; + +const DEFAULT_RUNTIME_WORKER_THREADS: usize = 2; +const ENV_RUNTIME_WORKER_THREADS: &str = "RUNTIME_WORKER_THREADS"; + +#[derive(Debug)] +enum Action { + Run(Args), + Start(Args), + Delete(Args), + Help, + Version, +} + +fn parse_args(args: &[OsString]) -> Result { + let mut help = false; + let mut version = false; + let mut shim_args = Args::default(); + + // Crate `go_flag` is used to keep compatible with go/flag package. + let rest_args = go_flag::parse_args_with_warnings::(&args[1..], None, |flags| { + flags.add_flag("address", &mut shim_args.address); + flags.add_flag("bundle", &mut shim_args.bundle); + flags.add_flag("debug", &mut shim_args.debug); + flags.add_flag("id", &mut shim_args.id); + flags.add_flag("namespace", &mut shim_args.namespace); + flags.add_flag("publish-binary", &mut shim_args.publish_binary); + flags.add_flag("socket", &mut shim_args.socket); + flags.add_flag("help", &mut help); + flags.add_flag("version", &mut version); + }) + .context(Error::ParseArgument(format!("{:?}", args)))?; + + if help { + Ok(Action::Help) + } else if version { + Ok(Action::Version) + } else if rest_args.is_empty() { + Ok(Action::Run(shim_args)) + } else if rest_args[0] == "start" { + Ok(Action::Start(shim_args)) + } else if rest_args[0] == "delete" { + Ok(Action::Delete(shim_args)) + } else { + Err(anyhow!(Error::InvalidArgument)) + } +} + +fn show_help(cmd: &OsStr) { + let path = PathBuf::from(cmd); + let name = match path.file_name() { + Some(v) => v.to_str(), + None => None, + }; + + let name = name.unwrap_or(config::RUNTIME_NAME); + + println!( + r#"Usage of {}: + -address string + grpc address back to main containerd + -bundle string + path to the bundle if not workdir + -debug + enable debug output in logs + -id string + id of the task + -namespace string + namespace that owns the shim + -publish-binary string + path to publish binary (used for publishing events) (default "containerd") + -socket string + socket path to serve + --version + show the runtime version detail and exit +"#, + name + ); +} + +fn show_version(err: Option) { + let data = format!( + r#"{} containerd shim: id: {}, version: {}, commit: {}"#, + config::PROJECT_NAME, + config::CONTAINERD_RUNTIME_NAME, + config::RUNTIME_VERSION, + config::RUNTIME_VERSION_COMMIT, + ); + + if let Some(err) = err { + eprintln!( + "{}\r\nERROR: {} failed: {:?}", + data, + config::RUNTIME_NAME, + err + ); + } else { + println!("{}", data) + } +} + +fn get_tokio_runtime() -> Result { + let worker_threads = std::env::var(ENV_RUNTIME_WORKER_THREADS) + .unwrap_or_default() + .parse() + .unwrap_or(DEFAULT_RUNTIME_WORKER_THREADS); + + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(worker_threads) + .enable_all() + .build() + .context("prepare tokio runtime")?; + Ok(rt) +} + +fn real_main() -> Result<()> { + let args = std::env::args_os().collect::>(); + if args.is_empty() { + return Err(anyhow!(Error::ArgumentIsEmpty( + "command-line arguments".to_string() + ))); + } + + let action = parse_args(&args).context("parse args")?; + match action { + Action::Start(args) => ShimExecutor::new(args).start().context("shim start")?, + Action::Delete(args) => ShimExecutor::new(args).delete().context("shim delete")?, + Action::Run(args) => { + // set mnt namespace + // need setup before other async call + setup_mnt().context("setup mnt")?; + + let mut shim = ShimExecutor::new(args); + let rt = get_tokio_runtime().context("get tokio runtime")?; + rt.block_on(shim.run())? + } + Action::Help => show_help(&args[0]), + Action::Version => show_version(None), + } + Ok(()) +} +fn main() { + if let Err(err) = real_main() { + show_version(Some(err)); + } +} + +fn setup_mnt() -> Result<()> { + // Unshare the mount namespace, so that the calling process has a private copy of its namespace + // which is not shared with any other process. + sched::unshare(CloneFlags::CLONE_NEWNS).context("unshare clone newns")?; + + // Mount and unmount events propagate into this mount from the (master) shared peer group of + // which it was formerly a member. Mount and unmount events under this mount do not propagate + // to any peer. + mount( + Some("none"), + "/", + Some(""), + MsFlags::MS_REC | MsFlags::MS_SLAVE, + Some(""), + ) + .context("mount with slave")?; + + // Mount and unmount events immediately under this mount will propagate to the other mounts + // that are members of this mount's peer group. + mount( + Some("none"), + "/", + Some(""), + MsFlags::MS_REC | MsFlags::MS_SHARED, + Some(""), + ) + .context("mount with shared")?; + Ok(()) +} diff --git a/src/runtime-rs/crates/shim/src/config.rs.in b/src/runtime-rs/crates/shim/src/config.rs.in new file mode 100644 index 000000000000..e1a181ec4a75 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/config.rs.in @@ -0,0 +1,19 @@ +// Copyright (c) 2020 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// +// WARNING: This file is auto-generated - DO NOT EDIT! +// + +#![allow(dead_code)] + +pub const PROJECT_NAME: &str = "@PROJECT_NAME@"; +pub const RUNTIME_VERSION: &str = "@RUNTIME_VERSION@"; +pub const RUNTIME_VERSION_COMMIT: &str = "@VERSION_COMMIT@"; +pub const RUNTIME_GIT_COMMIT: &str = "@COMMIT@"; +pub const RUNTIME_NAME: &str = "@RUNTIME_NAME@"; +pub const CONTAINERD_RUNTIME_NAME: &str = "@CONTAINERD_RUNTIME_NAME@"; +pub const RUNTIME_DIR: &str = "@BINDIR@"; +pub const RUNTIME_PATH: &str = "@BINDIR@/@RUNTIME_NAME@"; diff --git a/src/runtime-rs/crates/shim/src/error.rs b/src/runtime-rs/crates/shim/src/error.rs new file mode 100644 index 000000000000..3867963fbc26 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/error.rs @@ -0,0 +1,52 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::PathBuf; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("failed to parse argument {0}")] + ParseArgument(String), + #[error("failed to get bundle path")] + GetBundlePath, + #[error("invalid argument")] + InvalidArgument, + #[error("argument is empty {0}")] + ArgumentIsEmpty(String), + #[error("invalid path {0}")] + InvalidPath(String), + + // File + #[error("failed to open file {0}")] + FileOpen(String), + #[error("failed to get file metadata {0}")] + FileGetMetadata(String), + #[error("failed to read file {0}")] + FileRead(String), + #[error("failed to write file {0}")] + FileWrite(String), + + #[error("empty sandbox id")] + EmptySandboxId, + #[error("failed to get self exec: {0}")] + SelfExec(#[source] std::io::Error), + #[error("failed to bind socket at {1} with error: {0}")] + BindSocket(#[source] std::io::Error, PathBuf), + #[error("failed to spawn child: {0}")] + SpawnChild(#[source] std::io::Error), + #[error("failed to clean container {0}")] + CleanUpContainer(String), + #[error("failed to get env variable: {0}")] + EnvVar(#[source] std::env::VarError), + #[error("failed to parse server fd environment variable {0}")] + ServerFd(String), + #[error("failed to wait ttrpc server when {0}")] + WaitServer(String), + #[error("failed to get system time: {0}")] + SystemTime(#[source] std::time::SystemTimeError), + #[error("failed to parse pid")] + ParsePid, +} diff --git a/src/runtime-rs/crates/shim/src/lib.rs b/src/runtime-rs/crates/shim/src/lib.rs new file mode 100644 index 000000000000..1130cb7e4597 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/lib.rs @@ -0,0 +1,28 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[macro_use] +extern crate slog; + +macro_rules! sl { + () => { + slog_scope::logger().new(slog::o!("subsystem" => "shim")) + }; +} + +mod args; +pub use args::Args; +mod error; +pub use error::Error; +mod logger; +mod panic_hook; +mod shim; +pub use shim::ShimExecutor; +#[rustfmt::skip] +pub mod config; +mod shim_delete; +mod shim_run; +mod shim_start; diff --git a/src/runtime-rs/crates/shim/src/logger.rs b/src/runtime-rs/crates/shim/src/logger.rs new file mode 100644 index 000000000000..fc82df73d43e --- /dev/null +++ b/src/runtime-rs/crates/shim/src/logger.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::os::unix::fs::OpenOptionsExt; + +use anyhow::{Context, Result}; + +use crate::Error; + +pub(crate) fn set_logger(path: &str, sid: &str, is_debug: bool) -> Result { + let fifo = std::fs::OpenOptions::new() + .custom_flags(libc::O_NONBLOCK) + .create(true) + .write(true) + .append(true) + .open(path) + .context(Error::FileOpen(path.to_string()))?; + + let level = if is_debug { + slog::Level::Debug + } else { + slog::Level::Info + }; + + let (logger, async_guard) = logging::create_logger("kata-runtime", sid, level, fifo); + + // not reset global logger when drop + slog_scope::set_global_logger(logger).cancel_reset(); + + let level = if is_debug { + log::Level::Debug + } else { + log::Level::Info + }; + let _ = slog_stdlog::init_with_level(level).context(format!("init with level {}", level))?; + + Ok(async_guard) +} diff --git a/src/runtime-rs/crates/shim/src/panic_hook.rs b/src/runtime-rs/crates/shim/src/panic_hook.rs new file mode 100644 index 000000000000..0b0f4e1db8f8 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/panic_hook.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{boxed::Box, ops::Deref}; + +use backtrace::Backtrace; + +// TODO: the Kata 1.x runtime had a SIGUSR1 handler that would log a formatted backtrace on +// receiving that signal. It could be useful to re-add that feature. +pub(crate) fn set_panic_hook() { + std::panic::set_hook(Box::new(move |panic_info| { + let (filename, line) = panic_info + .location() + .map(|loc| (loc.file(), loc.line())) + .unwrap_or(("", 0)); + + let cause = panic_info + .payload() + .downcast_ref::() + .map(std::string::String::deref); + + let cause = cause.unwrap_or_else(|| { + panic_info + .payload() + .downcast_ref::<&str>() + .copied() + .unwrap_or("") + }); + let bt = Backtrace::new(); + let bt_data = format!("{:?}", bt); + error!( + sl!(), + "A panic occurred at {}:{}: {}\r\n{:?}", filename, line, cause, bt_data + ); + + std::process::abort(); + })); +} diff --git a/src/runtime-rs/crates/shim/src/shim.rs b/src/runtime-rs/crates/shim/src/shim.rs new file mode 100644 index 000000000000..298f976dcf72 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/shim.rs @@ -0,0 +1,116 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + os::unix::ffi::OsStrExt, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context, Result}; +use sha2::Digest; + +use crate::{Args, Error}; + +const SOCKET_ROOT: &str = "/run/containerd"; +const SHIM_PID_FILE: &str = "shim.pid"; + +pub(crate) const ENV_KATA_RUNTIME_BIND_FD: &str = "KATA_RUNTIME_BIND_FD"; + +/// Command executor for shim. +pub struct ShimExecutor { + pub(crate) args: Args, +} + +impl ShimExecutor { + /// Create a new instance of [`Shim`]. + pub fn new(args: Args) -> Self { + ShimExecutor { args } + } + + pub(crate) fn load_oci_spec(&self) -> Result { + let bundle_path = self.get_bundle_path()?; + let spec_file = bundle_path.join("config.json"); + + oci::Spec::load(spec_file.to_str().unwrap_or_default()).context("load spec") + } + + pub(crate) fn write_address(&self, address: &Path) -> Result<()> { + let dir = self.get_bundle_path()?; + let file_path = &dir.join("address"); + std::fs::write(file_path, address.as_os_str().as_bytes()) + .context(Error::FileWrite(format!("{:?}", &file_path))) + } + + pub(crate) fn write_pid_file(&self, pid: u32) -> Result<()> { + let dir = self.get_bundle_path()?; + let file_path = &dir.join(SHIM_PID_FILE); + std::fs::write(file_path, format!("{}", pid)) + .context(Error::FileWrite(format!("{:?}", &file_path))) + } + + pub(crate) fn read_pid_file(&self, bundle_path: &Path) -> Result { + let file_path = bundle_path.join(SHIM_PID_FILE); + let data = std::fs::read_to_string(&file_path) + .context(Error::FileOpen(format!("{:?}", file_path)))?; + + data.parse::().context(Error::ParsePid) + } + + pub(crate) fn get_bundle_path(&self) -> Result { + std::env::current_dir().context(Error::GetBundlePath) + } + + pub(crate) fn socket_address(&self, id: &str) -> Result { + if id.is_empty() { + return Err(anyhow!(Error::EmptySandboxId)); + } + + let data = [&self.args.address, &self.args.namespace, id].join("/"); + let mut hasher = sha2::Sha256::new(); + hasher.update(data); + Ok(PathBuf::from(format!( + "unix://{}/s/{:X}", + SOCKET_ROOT, + hasher.finalize() + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serial_test::serial; + + #[test] + #[serial] + fn test_shim_executor() { + let dir = tempfile::tempdir().unwrap(); + let bundle_path = dir.path(); + std::env::set_current_dir(bundle_path).unwrap(); + + let args = Args { + id: "1dfc0567".to_string(), + namespace: "test_namespace".into(), + address: "containerd_socket".into(), + publish_binary: "containerd".into(), + socket: "socket".into(), + bundle: bundle_path.to_str().unwrap().into(), + debug: false, + }; + + let executor = ShimExecutor::new(args); + + executor.write_address(Path::new("12345")).unwrap(); + let dir = executor.get_bundle_path().unwrap(); + let file_path = &dir.join("address"); + let buf = std::fs::read_to_string(file_path).unwrap(); + assert_eq!(&buf, "12345"); + + executor.write_pid_file(1267).unwrap(); + let read_pid = executor.read_pid_file(&dir).unwrap(); + assert_eq!(read_pid, 1267); + } +} diff --git a/src/runtime-rs/crates/shim/src/shim_delete.rs b/src/runtime-rs/crates/shim/src/shim_delete.rs new file mode 100644 index 000000000000..4d9890d13822 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/shim_delete.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{Context, Result}; +use containerd_shim_protos::shim::shim::DeleteResponse; +use protobuf::Message; + +use crate::{shim::ShimExecutor, Error}; + +impl ShimExecutor { + pub fn delete(&mut self) -> Result<()> { + self.args.validate(true).context("validate")?; + let rsp = self.do_cleanup().context("do cleanup")?; + rsp.write_to_writer(&mut std::io::stdout()) + .context(Error::FileWrite(format!("write {:?} to stdout", rsp)))?; + Ok(()) + } + + fn do_cleanup(&self) -> Result { + let mut rsp = DeleteResponse::new(); + rsp.set_exit_status(128 + libc::SIGKILL as u32); + let mut exited_time = protobuf::well_known_types::Timestamp::new(); + let seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(Error::SystemTime)? + .as_secs() as i64; + exited_time.set_seconds(seconds); + rsp.set_exited_at(exited_time); + + // TODO: implement cleanup + Ok(rsp) + } +} + +#[cfg(test)] +mod tests { + use serial_test::serial; + use tests_utils::gen_id; + + use super::*; + use crate::Args; + + #[test] + #[serial] + fn test_shim_delete() { + let dir = tempfile::tempdir().unwrap(); + let bundle_path = dir.path(); + std::env::set_current_dir(bundle_path).unwrap(); + + let id = gen_id(16); + let namespace = gen_id(16); + let args = Args { + id, + namespace, + address: "containerd_socket".into(), + publish_binary: "containerd".into(), + socket: "socket".into(), + bundle: bundle_path.to_str().unwrap().into(), + debug: false, + }; + + let executor = ShimExecutor::new(args); + + let resp = executor.do_cleanup().unwrap(); + assert_eq!(resp.exit_status, 128 + libc::SIGKILL as u32); + assert!(resp.exited_at.as_ref().unwrap().seconds > 0); + } +} diff --git a/src/runtime-rs/crates/shim/src/shim_run.rs b/src/runtime-rs/crates/shim/src/shim_run.rs new file mode 100644 index 000000000000..76058964bd8d --- /dev/null +++ b/src/runtime-rs/crates/shim/src/shim_run.rs @@ -0,0 +1,54 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::os::unix::io::RawFd; + +use anyhow::{Context, Result}; + +use crate::{ + logger, + shim::{ShimExecutor, ENV_KATA_RUNTIME_BIND_FD}, + Error, +}; + +impl ShimExecutor { + pub async fn run(&mut self) -> Result<()> { + crate::panic_hook::set_panic_hook(); + let sid = self.args.id.clone(); + let bundle_path = self.get_bundle_path().context("get bundle")?; + let path = bundle_path.join("log"); + let _logger_guard = + logger::set_logger(path.to_str().unwrap(), &sid, self.args.debug).context("set logger"); + + self.do_run() + .await + .map_err(|err| { + error!(sl!(), "failed run shim {:?}", err); + err + }) + .context("run shim")?; + + Ok(()) + } + + async fn do_run(&mut self) -> Result<()> { + info!(sl!(), "start to run"); + self.args.validate(false).context("validata")?; + + let _server_fd = get_server_fd().context("get server fd")?; + // TODO: implement run + + Ok(()) + } +} + +fn get_server_fd() -> Result { + let env_fd = std::env::var(ENV_KATA_RUNTIME_BIND_FD).map_err(Error::EnvVar)?; + let fd = env_fd + .parse::() + .map_err(|_| Error::ServerFd(env_fd))?; + Ok(fd) +} diff --git a/src/runtime-rs/crates/shim/src/shim_start.rs b/src/runtime-rs/crates/shim/src/shim_start.rs new file mode 100644 index 000000000000..b5b136607249 --- /dev/null +++ b/src/runtime-rs/crates/shim/src/shim_start.rs @@ -0,0 +1,238 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + fs, + io::Write, + os::unix::{io::IntoRawFd, prelude::OsStrExt}, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context, Result}; +use kata_types::{container::ContainerType, k8s}; +use unix_socket::UnixListener; + +use crate::{ + shim::{ShimExecutor, ENV_KATA_RUNTIME_BIND_FD}, + Error, +}; + +impl ShimExecutor { + pub fn start(&mut self) -> Result<()> { + self.args.validate(false).context("validate")?; + + let address = self.do_start().context("do start")?; + std::io::stdout() + .write_all(address.as_os_str().as_bytes()) + .context("failed to write stdout")?; + Ok(()) + } + + fn do_start(&mut self) -> Result { + let spec = self.load_oci_spec()?; + let (container_type, id) = k8s::container_type_with_id(&spec); + + match container_type { + ContainerType::PodSandbox => { + let address = self.socket_address(&self.args.id)?; + let socket = new_listener(&address)?; + let child_pid = self.create_shim_process(socket)?; + self.write_pid_file(child_pid)?; + self.write_address(&address)?; + Ok(address) + } + ContainerType::PodContainer => { + let sid = id + .ok_or(Error::InvalidArgument) + .context("get sid for container")?; + let (address, pid) = self.get_shim_info_from_sandbox(&sid)?; + self.write_pid_file(pid)?; + self.write_address(&address)?; + Ok(address) + } + } + } + + fn new_command(&self) -> Result { + if self.args.id.is_empty() + || self.args.namespace.is_empty() + || self.args.address.is_empty() + || self.args.publish_binary.is_empty() + { + return Err(anyhow!("invalid param")); + } + + let bundle_path = self.get_bundle_path().context("get bundle path")?; + let self_exec = std::env::current_exe().map_err(Error::SelfExec)?; + let mut command = std::process::Command::new(self_exec); + + command + .current_dir(bundle_path) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .arg("-id") + .arg(&self.args.id) + .arg("-namespace") + .arg(&self.args.namespace) + .arg("-address") + .arg(&self.args.address) + .arg("-publish-binary") + .arg(&self.args.publish_binary) + .env("RUST_BACKTRACE", "1"); + + if self.args.debug { + command.arg("-debug"); + } + + Ok(command) + } + + fn create_shim_process(&self, socket: T) -> Result { + let mut cmd = self.new_command().context("new command")?; + cmd.env( + ENV_KATA_RUNTIME_BIND_FD, + format!("{}", socket.into_raw_fd()), + ); + let child = cmd + .spawn() + .map_err(Error::SpawnChild) + .context("spawn child")?; + + Ok(child.id()) + } + + fn get_shim_info_from_sandbox(&self, sandbox_id: &str) -> Result<(PathBuf, u32)> { + // All containers of a pod share the same pod socket address. + let address = self.socket_address(sandbox_id).context("socket address")?; + let bundle_path = self.get_bundle_path().context("get bundle path")?; + let parent_bundle_path = Path::new(&bundle_path) + .parent() + .unwrap_or_else(|| Path::new("")); + let sandbox_bundle_path = parent_bundle_path + .join(sandbox_id) + .canonicalize() + .context(Error::GetBundlePath)?; + let pid = self.read_pid_file(&sandbox_bundle_path)?; + + Ok((address, pid)) + } +} + +fn new_listener(address: &Path) -> Result { + let trim_path = address.strip_prefix("unix:").context("trim path")?; + let file_path = Path::new("/").join(trim_path); + let file_path = file_path.as_path(); + if let Some(parent_dir) = file_path.parent() { + fs::create_dir_all(parent_dir).context("create parent dir")?; + } + + UnixListener::bind(file_path).context("bind address") +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serial_test::serial; + use tests_utils::gen_id; + + use super::*; + use crate::Args; + + #[test] + #[serial] + fn test_new_command() { + let dir = tempfile::tempdir().unwrap(); + let bundle_path = dir.path(); + std::env::set_current_dir(bundle_path).unwrap(); + + let args = Args { + id: "sandbox1".into(), + namespace: "ns".into(), + address: "address".into(), + publish_binary: "containerd".into(), + socket: "socket".into(), + bundle: bundle_path.to_str().unwrap().into(), + debug: false, + }; + let mut executor = ShimExecutor::new(args); + + let cmd = executor.new_command().unwrap(); + assert_eq!(cmd.get_args().len(), 8); + assert_eq!(cmd.get_envs().len(), 1); + assert_eq!( + cmd.get_current_dir().unwrap(), + executor.get_bundle_path().unwrap() + ); + + executor.args.debug = true; + let cmd = executor.new_command().unwrap(); + assert_eq!(cmd.get_args().len(), 9); + assert_eq!(cmd.get_envs().len(), 1); + assert_eq!( + cmd.get_current_dir().unwrap(), + executor.get_bundle_path().unwrap() + ); + } + + #[test] + #[serial] + fn test_get_info_from_sandbox() { + let dir = tempfile::tempdir().unwrap(); + let sandbox_id = gen_id(16); + let bundle_path = &dir.path().join(&sandbox_id); + std::fs::create_dir(bundle_path).unwrap(); + std::env::set_current_dir(bundle_path).unwrap(); + + let args = Args { + id: sandbox_id.to_owned(), + namespace: "ns1".into(), + address: "containerd_socket".into(), + publish_binary: "containerd".into(), + socket: "socket".into(), + bundle: bundle_path.to_str().unwrap().into(), + debug: false, + }; + let executor = ShimExecutor::new(args); + + let addr = executor.socket_address(&executor.args.id).unwrap(); + executor.write_address(&addr).unwrap(); + executor.write_pid_file(1267).unwrap(); + + let container_id = gen_id(16); + let bundle_path2 = &dir.path().join(&container_id); + std::fs::create_dir(bundle_path2).unwrap(); + std::env::set_current_dir(bundle_path2).unwrap(); + + let args = Args { + id: container_id, + namespace: "ns1".into(), + address: "containerd_socket".into(), + publish_binary: "containerd".into(), + socket: "socket".into(), + bundle: bundle_path2.to_str().unwrap().into(), + debug: false, + }; + let executor2 = ShimExecutor::new(args); + + let (address, pid) = executor2.get_shim_info_from_sandbox(&sandbox_id).unwrap(); + + assert_eq!(pid, 1267); + assert_eq!(&address, &addr); + } + + #[test] + #[serial] + fn test_new_listener() { + let path = "/tmp/aaabbbccc"; + let uds_path = format!("unix://{}", path); + std::fs::remove_file(path).ok(); + + let _ = new_listener(Path::new(&uds_path)).unwrap(); + std::fs::remove_file(path).ok(); + } +} diff --git a/src/runtime-rs/tests/texture/image-bundle/config.json b/src/runtime-rs/tests/texture/image-bundle/config.json new file mode 100644 index 000000000000..0b6665a2eb75 --- /dev/null +++ b/src/runtime-rs/tests/texture/image-bundle/config.json @@ -0,0 +1,395 @@ +{ + "ociVersion": "0.5.0-dev", + "process": { + "terminal": true, + "user": { + "uid": 1, + "gid": 1, + "additionalGids": [ + 5, + 6 + ] + }, + "args": [ + "sh" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "TERM=xterm" + ], + "cwd": "/", + "capabilities": { + "bounding": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "permitted": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "inheritable": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "effective": [ + "CAP_AUDIT_WRITE", + "CAP_KILL" + ], + "ambient": [ + "CAP_NET_BIND_SERVICE" + ] + }, + "rlimits": [ + { + "type": "RLIMIT_CORE", + "hard": 1024, + "soft": 1024 + }, + { + "type": "RLIMIT_NOFILE", + "hard": 1024, + "soft": 1024 + } + ], + "apparmorProfile": "acme_secure_profile", + "selinuxLabel": "system_u:system_r:svirt_lxc_net_t:s0:c124,c675", + "noNewPrivileges": true + }, + "root": { + "path": "rootfs", + "readonly": true + }, + "hostname": "slartibartfast", + "mounts": [ + { + "destination": "/proc", + "type": "proc", + "source": "proc" + }, + { + "destination": "/dev", + "type": "tmpfs", + "source": "tmpfs", + "options": [ + "nosuid", + "strictatime", + "mode=755", + "size=65536k" + ] + }, + { + "destination": "/dev/pts", + "type": "devpts", + "source": "devpts", + "options": [ + "nosuid", + "noexec", + "newinstance", + "ptmxmode=0666", + "mode=0620", + "gid=5" + ] + }, + { + "destination": "/dev/shm", + "type": "tmpfs", + "source": "shm", + "options": [ + "nosuid", + "noexec", + "nodev", + "mode=1777", + "size=65536k" + ] + }, + { + "destination": "/dev/mqueue", + "type": "mqueue", + "source": "mqueue", + "options": [ + "nosuid", + "noexec", + "nodev" + ] + }, + { + "destination": "/sys", + "type": "sysfs", + "source": "sysfs", + "options": [ + "nosuid", + "noexec", + "nodev" + ] + }, + { + "destination": "/sys/fs/cgroup", + "type": "cgroup", + "source": "cgroup", + "options": [ + "nosuid", + "noexec", + "nodev", + "relatime", + "ro" + ] + } + ], + "hooks": { + "prestart": [ + { + "path": "/usr/bin/fix-mounts", + "args": [ + "fix-mounts", + "arg1", + "arg2" + ], + "env": [ + "key1=value1" + ] + }, + { + "path": "/usr/bin/setup-network" + } + ], + "createRuntime": [ + { + "path": "/usr/bin/fix-mounts", + "args": ["fix-mounts", "arg1", "arg2"], + "env": [ "key1=value1"] + }, + { + "path": "/usr/bin/setup-network" + } + ], + "createContainer": [ + { + "path": "/usr/bin/mount-hook", + "args": ["-mount", "arg1", "arg2"], + "env": [ "key1=value1"] + } + ], + "startContainer": [ + { + "path": "/usr/bin/refresh-ldcache" + } + ], + "poststart": [ + { + "path": "/usr/bin/notify-start", + "timeout": 5 + } + ], + "poststop": [ + { + "path": "/usr/sbin/cleanup.sh", + "args": [ + "cleanup.sh", + "-f" + ] + } + ] + }, + "linux": { + "devices": [ + { + "path": "/dev/fuse", + "type": "c", + "major": 10, + "minor": 229, + "fileMode": 438, + "uid": 0, + "gid": 0 + }, + { + "path": "/dev/sda", + "type": "b", + "major": 8, + "minor": 0, + "fileMode": 432, + "uid": 0, + "gid": 0 + } + ], + "uidMappings": [ + { + "containerID": 0, + "hostID": 1000, + "size": 32000 + } + ], + "gidMappings": [ + { + "containerID": 0, + "hostID": 1000, + "size": 32000 + } + ], + "sysctl": { + "net.ipv4.ip_forward": "1", + "net.core.somaxconn": "256" + }, + "cgroupsPath": "/myRuntime/myContainer", + "resources": { + "network": { + "classID": 1048577, + "priorities": [ + { + "name": "eth0", + "priority": 500 + }, + { + "name": "eth1", + "priority": 1000 + } + ] + }, + "pids": { + "limit": 32771 + }, + "hugepageLimits": [ + { + "pageSize": "2MB", + "limit": 9223372036854772000 + }, + { + "pageSize": "64KB", + "limit": 1000000 + } + ], + "oomScoreAdj": 100, + "memory": { + "limit": 536870912, + "reservation": 536870912, + "swap": 536870912, + "kernel": -1, + "kernelTCP": -1, + "swappiness": 0, + "disableOOMKiller": false, + "useHierarchy": false + }, + "cpu": { + "shares": 1024, + "quota": 1000000, + "period": 500000, + "realtimeRuntime": 950000, + "realtimePeriod": 1000000, + "cpus": "2-3", + "mems": "0-7" + }, + "devices": [ + { + "allow": false, + "access": "rwm" + }, + { + "allow": true, + "type": "c", + "major": 10, + "minor": 229, + "access": "rw" + }, + { + "allow": true, + "type": "b", + "major": 8, + "minor": 0, + "access": "r" + } + ], + "blockIO": { + "weight": 10, + "leafWeight": 10, + "weightDevice": [ + { + "major": 8, + "minor": 0, + "weight": 500, + "leafWeight": 300 + }, + { + "major": 8, + "minor": 16, + "weight": 500 + } + ], + "throttleReadBpsDevice": [ + { + "major": 8, + "minor": 0, + "rate": 600 + } + ], + "throttleWriteIOPSDevice": [ + { + "major": 8, + "minor": 16, + "rate": 300 + } + ] + } + }, + "rootfsPropagation": "slave", + "seccomp": { + "defaultAction": "SCMP_ACT_ALLOW", + "architectures": [ + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" + ], + "syscalls": [ + { + "names": [ + "getcwd", + "chmod" + ], + "action": "SCMP_ACT_ERRNO" + } + ] + }, + "namespaces": [ + { + "type": "pid" + }, + { + "type": "network" + }, + { + "type": "ipc" + }, + { + "type": "uts" + }, + { + "type": "mount" + }, + { + "type": "user" + }, + { + "type": "cgroup" + } + ], + "maskedPaths": [ + "/proc/kcore", + "/proc/latency_stats", + "/proc/timer_stats", + "/proc/sched_debug" + ], + "readonlyPaths": [ + "/proc/asound", + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger" + ], + "mountLabel": "system_u:object_r:svirt_sandbox_file_t:s0:c715,c811" + }, + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file diff --git a/src/runtime-rs/tests/texture/kata-containers-configuration.toml b/src/runtime-rs/tests/texture/kata-containers-configuration.toml new file mode 100644 index 000000000000..9116080ddf1f --- /dev/null +++ b/src/runtime-rs/tests/texture/kata-containers-configuration.toml @@ -0,0 +1,11 @@ +[runtime] +enable_debug = true + +[hypervisor] + +[hypervisor.dragonball] +default_vcpus = 2 + +[hypervisor.qemu] +default_vcpus = 4 + diff --git a/src/runtime-rs/tests/utils/Cargo.toml b/src/runtime-rs/tests/utils/Cargo.toml new file mode 100644 index 000000000000..7317b7f0ffd0 --- /dev/null +++ b/src/runtime-rs/tests/utils/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "tests_utils" +version = "0.1.0" +edition = "2018" +description = "This crate is used to share code among tests" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rand = "0.8.4" diff --git a/src/runtime-rs/tests/utils/src/lib.rs b/src/runtime-rs/tests/utils/src/lib.rs new file mode 100644 index 000000000000..b3a4b35174d5 --- /dev/null +++ b/src/runtime-rs/tests/utils/src/lib.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +// This crate is used to share code among tests + +use std::path::PathBuf; + +use rand::{ + distributions::Alphanumeric, + {thread_rng, Rng}, +}; + +pub fn get_kata_config_file() -> PathBuf { + let target = format!( + "{}/../texture/kata-containers-configuration.toml", + env!("CARGO_MANIFEST_DIR") + ); + std::fs::canonicalize(target).unwrap() +} + +pub fn get_image_bundle_path() -> PathBuf { + let target = format!("{}/../texture/image-bundle", env!("CARGO_MANIFEST_DIR")); + std::fs::canonicalize(target).unwrap() +} + +pub fn gen_id(len: usize) -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} From 8c0a60e191437f74fb8454e7f3e24cc55cafe768 Mon Sep 17 00:00:00 2001 From: "quanwei.zqw" Date: Sat, 12 Mar 2022 16:33:08 +0800 Subject: [PATCH 0038/1953] runtime-rs: modify the review suggestion Fixes: #3876 Signed-off-by: quanwei.zqw --- src/runtime-rs/crates/shim/src/bin/main.rs | 10 +++--- src/runtime-rs/crates/shim/src/shim.rs | 35 ++++++++++++-------- src/runtime-rs/crates/shim/src/shim_start.rs | 15 +++++---- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/runtime-rs/crates/shim/src/bin/main.rs b/src/runtime-rs/crates/shim/src/bin/main.rs index 1526a89cd376..262f9e238521 100644 --- a/src/runtime-rs/crates/shim/src/bin/main.rs +++ b/src/runtime-rs/crates/shim/src/bin/main.rs @@ -16,8 +16,10 @@ use nix::{ }; use shim::{config, Args, Error, ShimExecutor}; -const DEFAULT_RUNTIME_WORKER_THREADS: usize = 2; -const ENV_RUNTIME_WORKER_THREADS: &str = "RUNTIME_WORKER_THREADS"; +// default tokio runtime worker threads +const DEFAULT_TOKIO_RUNTIME_WORKER_THREADS: usize = 2; +// env to config tokio runtime worker threads +const ENV_TOKIO_RUNTIME_WORKER_THREADS: &str = "TOKIO_RUNTIME_WORKER_THREADS"; #[derive(Debug)] enum Action { @@ -116,10 +118,10 @@ fn show_version(err: Option) { } fn get_tokio_runtime() -> Result { - let worker_threads = std::env::var(ENV_RUNTIME_WORKER_THREADS) + let worker_threads = std::env::var(ENV_TOKIO_RUNTIME_WORKER_THREADS) .unwrap_or_default() .parse() - .unwrap_or(DEFAULT_RUNTIME_WORKER_THREADS); + .unwrap_or(DEFAULT_TOKIO_RUNTIME_WORKER_THREADS); let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(worker_threads) diff --git a/src/runtime-rs/crates/shim/src/shim.rs b/src/runtime-rs/crates/shim/src/shim.rs index 298f976dcf72..164e6c6f60d3 100644 --- a/src/runtime-rs/crates/shim/src/shim.rs +++ b/src/runtime-rs/crates/shim/src/shim.rs @@ -30,29 +30,30 @@ impl ShimExecutor { ShimExecutor { args } } - pub(crate) fn load_oci_spec(&self) -> Result { - let bundle_path = self.get_bundle_path()?; - let spec_file = bundle_path.join("config.json"); - + pub(crate) fn load_oci_spec(&self, path: &Path) -> Result { + let spec_file = path.join("config.json"); oci::Spec::load(spec_file.to_str().unwrap_or_default()).context("load spec") } - pub(crate) fn write_address(&self, address: &Path) -> Result<()> { - let dir = self.get_bundle_path()?; - let file_path = &dir.join("address"); + pub(crate) fn write_address(&self, path: &Path, address: &Path) -> Result<()> { + let file_path = &path.join("address"); std::fs::write(file_path, address.as_os_str().as_bytes()) .context(Error::FileWrite(format!("{:?}", &file_path))) } - pub(crate) fn write_pid_file(&self, pid: u32) -> Result<()> { - let dir = self.get_bundle_path()?; - let file_path = &dir.join(SHIM_PID_FILE); + pub(crate) fn write_pid_file(&self, path: &Path, pid: u32) -> Result<()> { + let file_path = &path.join(SHIM_PID_FILE); std::fs::write(file_path, format!("{}", pid)) .context(Error::FileWrite(format!("{:?}", &file_path))) } - pub(crate) fn read_pid_file(&self, bundle_path: &Path) -> Result { - let file_path = bundle_path.join(SHIM_PID_FILE); + // There may be a multi-container for a Pod, each container has a bundle path, we need to write + // the PID to the file for each container in their own bundle path, so we can directly get the + // `bundle_path()` and write the PID. + // While the real runtime process's PID is stored in the file in the sandbox container's bundle + // path, so needs to read from the sandbox container's bundle path. + pub(crate) fn read_pid_file(&self, path: &Path) -> Result { + let file_path = path.join(SHIM_PID_FILE); let data = std::fs::read_to_string(&file_path) .context(Error::FileOpen(format!("{:?}", file_path)))?; @@ -71,6 +72,10 @@ impl ShimExecutor { let data = [&self.args.address, &self.args.namespace, id].join("/"); let mut hasher = sha2::Sha256::new(); hasher.update(data); + + // Follow + // https://github.com/containerd/containerd/blob/main/runtime/v2/shim/util_unix.go#L68 to + // generate a shim socket path. Ok(PathBuf::from(format!( "unix://{}/s/{:X}", SOCKET_ROOT, @@ -103,13 +108,15 @@ mod tests { let executor = ShimExecutor::new(args); - executor.write_address(Path::new("12345")).unwrap(); + executor + .write_address(bundle_path, Path::new("12345")) + .unwrap(); let dir = executor.get_bundle_path().unwrap(); let file_path = &dir.join("address"); let buf = std::fs::read_to_string(file_path).unwrap(); assert_eq!(&buf, "12345"); - executor.write_pid_file(1267).unwrap(); + executor.write_pid_file(&dir, 1267).unwrap(); let read_pid = executor.read_pid_file(&dir).unwrap(); assert_eq!(read_pid, 1267); } diff --git a/src/runtime-rs/crates/shim/src/shim_start.rs b/src/runtime-rs/crates/shim/src/shim_start.rs index b5b136607249..a84053209151 100644 --- a/src/runtime-rs/crates/shim/src/shim_start.rs +++ b/src/runtime-rs/crates/shim/src/shim_start.rs @@ -32,7 +32,8 @@ impl ShimExecutor { } fn do_start(&mut self) -> Result { - let spec = self.load_oci_spec()?; + let bundle_path = self.get_bundle_path().context("get bundle path")?; + let spec = self.load_oci_spec(&bundle_path)?; let (container_type, id) = k8s::container_type_with_id(&spec); match container_type { @@ -40,8 +41,8 @@ impl ShimExecutor { let address = self.socket_address(&self.args.id)?; let socket = new_listener(&address)?; let child_pid = self.create_shim_process(socket)?; - self.write_pid_file(child_pid)?; - self.write_address(&address)?; + self.write_pid_file(&bundle_path, child_pid)?; + self.write_address(&bundle_path, &address)?; Ok(address) } ContainerType::PodContainer => { @@ -49,8 +50,8 @@ impl ShimExecutor { .ok_or(Error::InvalidArgument) .context("get sid for container")?; let (address, pid) = self.get_shim_info_from_sandbox(&sid)?; - self.write_pid_file(pid)?; - self.write_address(&address)?; + self.write_pid_file(&bundle_path, pid)?; + self.write_address(&bundle_path, &address)?; Ok(address) } } @@ -200,8 +201,8 @@ mod tests { let executor = ShimExecutor::new(args); let addr = executor.socket_address(&executor.args.id).unwrap(); - executor.write_address(&addr).unwrap(); - executor.write_pid_file(1267).unwrap(); + executor.write_address(bundle_path, &addr).unwrap(); + executor.write_pid_file(bundle_path, 1267).unwrap(); let container_id = gen_id(16); let bundle_path2 = &dir.path().join(&container_id); From e705ee07c571c0cca867c1d3323a680fcc799030 Mon Sep 17 00:00:00 2001 From: "quanwei.zqw" Date: Sun, 13 Mar 2022 11:09:30 +0800 Subject: [PATCH 0039/1953] runtime-rs: update containerd-shim-protos to 0.2.0 Fixes: #3866 Signed-off-by: quanwei.zqw --- src/runtime-rs/Cargo.lock | 5 +++-- src/runtime-rs/crates/shim/Cargo.toml | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index c55b571a666f..352d21cd6e35 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -161,8 +161,9 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" [[package]] name = "containerd-shim-protos" -version = "0.1.2" -source = "git+https://github.com/containerd/rust-extensions.git?rev=c0baac598fc3ad62f651e8aae8de15db2ce5695c#c0baac598fc3ad62f651e8aae8de15db2ce5695c" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "077ec778a0835d9d85502e8535362130187759b69eddabe2bdb3a68ffb575bd0" dependencies = [ "async-trait", "protobuf", diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml index c208eb3187d6..e2c07e4e2878 100644 --- a/src/runtime-rs/crates/shim/Cargo.toml +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -15,9 +15,7 @@ path = "src/bin/main.rs" [dependencies] anyhow = "^1.0" backtrace = {version = ">=0.3.35", features = ["libunwind", "libbacktrace", "std"], default-features = false} -# TODO: change to version after release -# issue: https://github.com/kata-containers/kata-containers/issues/3866 -containerd-shim-protos = { git="https://github.com/containerd/rust-extensions.git", rev = "c0baac598fc3ad62f651e8aae8de15db2ce5695c", features = ["async"]} +containerd-shim-protos = { version = "0.2.0", features = ["async"]} go-flag = "0.1.0" libc = "0.2.108" log = "0.4.14" From d3da156eeafac3f9080ff94c029f3ba9d04ea70e Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 14 Mar 2022 17:35:46 +0100 Subject: [PATCH 0040/1953] runtime-rs: uint FsType for s390x statfs type on s390x should be c_uint, not __fsword_t Fixes: #3888 Signed-off-by: Jakob Naucke --- src/libs/kata-sys-util/src/fs.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libs/kata-sys-util/src/fs.rs b/src/libs/kata-sys-util/src/fs.rs index fb056e1324f9..1d85fa61cd20 100644 --- a/src/libs/kata-sys-util/src/fs.rs +++ b/src/libs/kata-sys-util/src/fs.rs @@ -13,11 +13,16 @@ use std::process::Command; use crate::{eother, sl}; -// nix filesystem_type for different target_os +// nix filesystem_type for different libc and architectures #[cfg(all(target_os = "linux", target_env = "musl"))] type FsType = libc::c_ulong; -#[cfg(all(target_os = "linux", not(any(target_env = "musl"))))] +#[cfg(all( + target_os = "linux", + not(any(target_env = "musl", target_arch = "s390x")) +))] type FsType = libc::__fsword_t; +#[cfg(all(target_os = "linux", not(target_env = "musl"), target_arch = "s390x"))] +type FsType = libc::c_uint; // from linux.git/fs/fuse/inode.c: #define FUSE_SUPER_MAGIC 0x65735546 const FUSE_SUPER_MAGIC: FsType = 0x65735546; From 4296e3069f4041f5b9f3375d2cb193e24152deb6 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Fri, 3 Dec 2021 18:53:48 +0800 Subject: [PATCH 0041/1953] runtime-rs: agent implements Responsible for communicating with the agent, such as kata-agent in the VM Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/libs/kata-types/src/config/agent.rs | 44 +- src/libs/kata-types/src/config/mod.rs | 2 +- src/runtime-rs/Cargo.lock | 279 +++++- src/runtime-rs/Cargo.toml | 2 + src/runtime-rs/crates/agent/Cargo.toml | 28 + src/runtime-rs/crates/agent/src/kata/agent.rs | 110 +++ src/runtime-rs/crates/agent/src/kata/mod.rs | 123 +++ src/runtime-rs/crates/agent/src/kata/trans.rs | 794 ++++++++++++++++++ src/runtime-rs/crates/agent/src/lib.rs | 84 ++ .../crates/agent/src/log_forwarder.rs | 159 ++++ .../crates/agent/src/sock/hybrid_vsock.rs | 81 ++ src/runtime-rs/crates/agent/src/sock/mod.rs | 159 ++++ src/runtime-rs/crates/agent/src/sock/vsock.rs | 32 + src/runtime-rs/crates/agent/src/types.rs | 454 ++++++++++ 14 files changed, 2317 insertions(+), 34 deletions(-) create mode 100644 src/runtime-rs/crates/agent/Cargo.toml create mode 100644 src/runtime-rs/crates/agent/src/kata/agent.rs create mode 100644 src/runtime-rs/crates/agent/src/kata/mod.rs create mode 100644 src/runtime-rs/crates/agent/src/kata/trans.rs create mode 100644 src/runtime-rs/crates/agent/src/lib.rs create mode 100644 src/runtime-rs/crates/agent/src/log_forwarder.rs create mode 100644 src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs create mode 100644 src/runtime-rs/crates/agent/src/sock/mod.rs create mode 100644 src/runtime-rs/crates/agent/src/sock/vsock.rs create mode 100644 src/runtime-rs/crates/agent/src/types.rs diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index ef9546ca9d47..b5bcf5b3b5e0 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -33,9 +33,29 @@ pub struct Agent { #[serde(default)] pub debug_console_enabled: bool, - /// Agent connection dialing timeout value in seconds + /// Agent server port #[serde(default)] - pub dial_timeout: u32, + pub server_port: u32, + + /// Agent log port + #[serde(default)] + pub log_port: u32, + + /// Agent connection dialing timeout value in millisecond + #[serde(default = "default_dial_timeout")] + pub dial_timeout_ms: u32, + + /// Agent reconnect timeout value in millisecond + #[serde(default = "default_reconnect_timeout")] + pub reconnect_timeout_ms: u32, + + /// Agent request timeout value in millisecond + #[serde(default = "default_request_timeout")] + pub request_timeout_ms: u32, + + /// Agent health check request timeout value in millisecond + #[serde(default = "default_health_check_timeout")] + pub health_check_request_timeout_ms: u32, /// Comma separated list of kernel modules and their parameters. /// @@ -55,6 +75,26 @@ pub struct Agent { pub container_pipe_size: u32, } +fn default_dial_timeout() -> u32 { + // 10ms + 10 +} + +fn default_reconnect_timeout() -> u32 { + // 3s + 3_000 +} + +fn default_request_timeout() -> u32 { + // 30s + 30_000 +} + +fn default_health_check_timeout() -> u32 { + // 90s + 90_000 +} + impl ConfigOps for Agent { fn adjust_config(conf: &mut TomlConfig) -> Result<()> { AgentVendor::adjust_config(conf)?; diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 435b8d1f0663..52c9a0e3c06c 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -21,7 +21,7 @@ pub mod default; mod agent; pub mod hypervisor; -use self::agent::Agent; +pub use self::agent::Agent; pub use self::hypervisor::{ BootInfo, DragonballConfig, Hypervisor, QemuConfig, HYPERVISOR_NAME_DRAGONBALL, HYPERVISOR_NAME_QEMU, diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 352d21cd6e35..7f3db194da5b 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -17,6 +17,27 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "agent" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "futures 0.1.31", + "kata-types", + "log", + "oci", + "protobuf", + "protocols", + "serde", + "serde_json", + "slog", + "slog-scope", + "tokio", + "ttrpc", + "url", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -28,9 +49,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd" +checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" [[package]] name = "arc-swap" @@ -178,9 +199,9 @@ checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -188,14 +209,25 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ "cfg-if 1.0.0", "lazy_static", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.9.0" @@ -205,6 +237,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "enum-iterator" version = "0.7.0" @@ -245,6 +283,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -255,6 +299,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.21" @@ -356,13 +406,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" +checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -385,9 +435,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" [[package]] name = "git2" -version = "0.13.25" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "3826a6e0e2215d7a41c2bfc7c9244123969273f3476b939a226aac0ab56e9e3c" dependencies = [ "bitflags", "libc", @@ -411,6 +461,21 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -431,6 +496,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "instant" version = "0.1.12" @@ -449,6 +524,15 @@ dependencies = [ "libc", ] +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.1" @@ -510,15 +594,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.119" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.13.2+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "3a42de9a51a5c12e00fc0e4ca6bc2ea43582fc6418488e8f615e905d886f258b" dependencies = [ "cc", "libc", @@ -528,9 +612,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" dependencies = [ "cc", "libc", @@ -600,14 +684,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" dependencies = [ "libc", "log", "miow", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi", ] @@ -620,6 +705,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "nix" version = "0.16.1" @@ -686,9 +777,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" dependencies = [ "libc", ] @@ -755,6 +846,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project-lite" version = "0.2.8" @@ -812,11 +913,66 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prost" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" +dependencies = [ + "bytes 1.1.0", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" +dependencies = [ + "bytes 1.1.0", + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" +dependencies = [ + "bytes 1.1.0", + "prost", +] + [[package]] name = "protobuf" version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" +dependencies = [ + "serde", + "serde_derive", +] [[package]] name = "protobuf-codegen" @@ -837,11 +993,22 @@ dependencies = [ "protobuf-codegen", ] +[[package]] +name = "protocols" +version = "0.1.0" +dependencies = [ + "async-trait", + "oci", + "protobuf", + "ttrpc", + "ttrpc-codegen", +] + [[package]] name = "quote" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" dependencies = [ "proc-macro2", ] @@ -878,18 +1045,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ "aho-corasick", "memchr", @@ -1132,9 +1299,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" dependencies = [ "proc-macro2", "quote", @@ -1268,7 +1435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.3.21", "libc", "tokio", "vsock", @@ -1291,7 +1458,7 @@ checksum = "0c7d6c992964a013c17814c08d31708d577b0aae44ebadb58755659dd824c2d1" dependencies = [ "async-trait", "byteorder", - "futures", + "futures 0.3.21", "libc", "log", "nix 0.23.1", @@ -1302,6 +1469,33 @@ dependencies = [ "tokio-vsock", ] +[[package]] +name = "ttrpc-codegen" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809eda4e459820237104e4b61d6b41bbe6c9e1ce6adf4057955e6e6722a90408" +dependencies = [ + "protobuf", + "protobuf-codegen", + "protobuf-codegen-pure", + "ttrpc-compiler", +] + +[[package]] +name = "ttrpc-compiler" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2978ed3fa047d8fd55cbeb4d4a61d461fb3021a90c9618519c73ce7e5bb66c15" +dependencies = [ + "derive-new", + "prost", + "prost-build", + "prost-types", + "protobuf", + "protobuf-codegen", + "tempfile", +] + [[package]] name = "typenum" version = "1.15.0" @@ -1323,6 +1517,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -1401,6 +1601,23 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index 414b707258dd..b359f9d1fb2f 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -1,4 +1,6 @@ [workspace] members = [ "crates/shim", + # TODO: current only for check, delete after use the agent crate + "crates/agent", ] diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml new file mode 100644 index 000000000000..bd17f82d0053 --- /dev/null +++ b/src/runtime-rs/crates/agent/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "agent" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dev-dependencies] +futures = "0.1.27" + +[dependencies] +anyhow = "1.0.26" +async-trait = "0.1.48" +log = "0.4.14" +protobuf = "2.23.0" +serde = { version = "^1.0", features = ["derive"] } +serde_json = ">=1.0.9" +slog = "2.5.2" +slog-scope = "4.4.0" +ttrpc = { version = "0.6.0" } +tokio = { version = "1.8.0", features = ["fs", "rt"] } +url = "2.2.2" + +kata-types = { path = "../../../libs/kata-types"} +oci = { path = "../../../libs/oci" } +protocols = { path = "../../../libs/protocols", features=["async"] } + +[features] +default = [] diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs new file mode 100644 index 000000000000..9f8b4304d3e2 --- /dev/null +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -0,0 +1,110 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use ttrpc::context as ttrpc_ctx; + +use crate::{kata::KataAgent, Agent, AgentManager, HealthService}; + +/// millisecond to nanosecond +const MILLISECOND_TO_NANOSECOND: i64 = 1_000_000; + +/// new ttrpc context with timeout +fn new_ttrpc_ctx(timeout: i64) -> ttrpc_ctx::Context { + ttrpc_ctx::with_timeout(timeout) +} + +#[async_trait] +impl AgentManager for KataAgent { + async fn set_socket_address(&self, address: &str) -> Result<()> { + let mut inner = self.inner.lock().await; + inner.socket_address = address.to_string(); + Ok(()) + } + + async fn start(&self) -> Result<()> { + info!(sl!(), "begin to connect agent"); + self.connect_agent_server() + .await + .context("connect agent server")?; + self.start_log_forwarder() + .await + .context("connect log forwarder")?; + Ok(()) + } + + async fn stop(&self) { + self.stop_log_forwarder().await; + } +} + +// implement for health service +macro_rules! impl_health_service { + ($($name: tt | $req: ty | $resp: ty),*) => { + #[async_trait] + impl HealthService for KataAgent { + $(async fn $name(&self, req: $req) -> Result<$resp> { + let r = req.into(); + let (mut client, timeout, _) = self.get_health_client().await.context("get health client")?; + let resp = client.$name(new_ttrpc_ctx(timeout * MILLISECOND_TO_NANOSECOND), &r).await?; + Ok(resp.into()) + })* + } + }; +} + +impl_health_service!( + check | crate::CheckRequest | crate::HealthCheckResponse, + version | crate::CheckRequest | crate::VersionCheckResponse +); + +macro_rules! impl_agent { + ($($name: tt | $req: ty | $resp: ty | $new_timeout: expr),*) => { + #[async_trait] + impl Agent for KataAgent { + $(async fn $name(&self, req: $req) -> Result<$resp> { + let r = req.into(); + let (mut client, mut timeout, _) = self.get_agent_client().await.context("get client")?; + + // update new timeout + if let Some(v) = $new_timeout { + timeout = v; + } + + let resp = client.$name(new_ttrpc_ctx(timeout * MILLISECOND_TO_NANOSECOND), &r).await?; + Ok(resp.into()) + })* + } + }; +} + +impl_agent!( + create_container | crate::CreateContainerRequest | crate::Empty | None, + start_container | crate::ContainerID | crate::Empty | None, + remove_container | crate::RemoveContainerRequest | crate::Empty | None, + exec_process | crate::ExecProcessRequest | crate::Empty | None, + signal_process | crate::SignalProcessRequest | crate::Empty | None, + wait_process | crate::WaitProcessRequest | crate::WaitProcessResponse | Some(0), + update_container | crate::UpdateContainerRequest | crate::Empty | None, + stats_container | crate::ContainerID | crate::StatsContainerResponse | None, + pause_container | crate::ContainerID | crate::Empty | None, + resume_container | crate::ContainerID | crate::Empty | None, + write_stdin | crate::WriteStreamRequest | crate::WriteStreamResponse | None, + read_stdout | crate::ReadStreamRequest | crate::ReadStreamResponse | None, + read_stderr | crate::ReadStreamRequest | crate::ReadStreamResponse | None, + close_stdin | crate::CloseStdinRequest | crate::Empty | None, + tty_win_resize | crate::TtyWinResizeRequest | crate::Empty | None, + update_interface | crate::UpdateInterfaceRequest | crate::Interface | None, + update_routes | crate::UpdateRoutesRequest | crate::Routes | None, + add_arp_neighbors | crate::AddArpNeighborRequest | crate::Empty | None, + list_interfaces | crate::Empty | crate::Interfaces | None, + list_routes | crate::Empty | crate::Routes | None, + create_sandbox | crate::CreateSandboxRequest | crate::Empty | None, + destroy_sandbox | crate::Empty | crate::Empty | None, + copy_file | crate::CopyFileRequest | crate::Empty | None, + get_oom_event | crate::Empty | crate::OomEventResponse | Some(0) +); diff --git a/src/runtime-rs/crates/agent/src/kata/mod.rs b/src/runtime-rs/crates/agent/src/kata/mod.rs new file mode 100644 index 000000000000..043b9aa14e5b --- /dev/null +++ b/src/runtime-rs/crates/agent/src/kata/mod.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod agent; +mod trans; + +use std::os::unix::io::{IntoRawFd, RawFd}; + +use anyhow::{Context, Result}; +use kata_types::config::Agent as AgentConfig; +use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc}; +use tokio::sync::Mutex; +use ttrpc::asynchronous::Client; + +use crate::{log_forwarder::LogForwarder, sock}; + +// https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md +#[derive(Debug, Default)] +pub struct Vsock { + pub context_id: u64, + pub port: u32, +} + +pub(crate) struct KataAgentInner { + /// TTRPC client + pub client: Option, + + /// Client fd + pub client_fd: RawFd, + + /// Unix domain socket address + pub socket_address: String, + + /// Agent config + config: AgentConfig, + + /// Log forwarder + log_forwarder: LogForwarder, +} + +unsafe impl Send for KataAgent {} +unsafe impl Sync for KataAgent {} +pub struct KataAgent { + pub(crate) inner: Mutex, +} + +impl KataAgent { + pub fn new(config: AgentConfig) -> Self { + KataAgent { + inner: Mutex::new(KataAgentInner { + client: None, + client_fd: -1, + socket_address: "".to_string(), + config, + log_forwarder: LogForwarder::new(), + }), + } + } + + pub async fn get_health_client(&self) -> Option<(health_ttrpc::HealthClient, i64, RawFd)> { + let inner = self.inner.lock().await; + inner.client.as_ref().map(|c| { + ( + health_ttrpc::HealthClient::new(c.clone()), + inner.config.health_check_request_timeout_ms as i64, + inner.client_fd, + ) + }) + } + + pub async fn get_agent_client(&self) -> Option<(agent_ttrpc::AgentServiceClient, i64, RawFd)> { + let inner = self.inner.lock().await; + inner.client.as_ref().map(|c| { + ( + agent_ttrpc::AgentServiceClient::new(c.clone()), + inner.config.request_timeout_ms as i64, + inner.client_fd, + ) + }) + } + + pub(crate) async fn connect_agent_server(&self) -> Result<()> { + let mut inner = self.inner.lock().await; + + let config = sock::ConnectConfig::new( + inner.config.dial_timeout_ms as u64, + inner.config.reconnect_timeout_ms as u64, + ); + let sock = + sock::new(&inner.socket_address, inner.config.server_port).context("new sock")?; + let stream = sock.connect(&config).await.context("connect")?; + let fd = stream.into_raw_fd(); + info!(sl!(), "get stream raw fd {:?}", fd); + let c = Client::new(fd); + inner.client = Some(c); + inner.client_fd = fd; + Ok(()) + } + + pub(crate) async fn start_log_forwarder(&self) -> Result<()> { + let mut inner = self.inner.lock().await; + let config = sock::ConnectConfig::new( + inner.config.dial_timeout_ms as u64, + inner.config.reconnect_timeout_ms as u64, + ); + let address = inner.socket_address.clone(); + let port = inner.config.log_port; + inner + .log_forwarder + .start(&address, port, config) + .await + .context("start log forwarder")?; + Ok(()) + } + + pub(crate) async fn stop_log_forwarder(&self) { + let mut inner = self.inner.lock().await; + inner.log_forwarder.stop(); + } +} diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs new file mode 100644 index 000000000000..c8e4dbca7c5a --- /dev/null +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -0,0 +1,794 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::convert::Into; + +use protocols::{ + agent::{self, OOMEvent}, + empty, health, types, +}; + +use crate::{ + types::{ + ARPNeighbor, ARPNeighbors, AddArpNeighborRequest, AgentDetails, BlkioStats, + BlkioStatsEntry, CgroupStats, CheckRequest, CloseStdinRequest, ContainerID, + CopyFileRequest, CpuStats, CpuUsage, CreateContainerRequest, CreateSandboxRequest, Device, + Empty, ExecProcessRequest, GuestDetailsResponse, HealthCheckResponse, HugetlbStats, + IPAddress, IPFamily, Interface, Interfaces, KernelModule, MemHotplugByProbeRequest, + MemoryData, MemoryStats, NetworkStats, OnlineCPUMemRequest, PidsStats, ReadStreamRequest, + ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, + SetGuestDateTimeRequest, SignalProcessRequest, StatsContainerResponse, Storage, StringUser, + ThrottlingData, TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, + UpdateRoutesRequest, VersionCheckResponse, WaitProcessRequest, WriteStreamRequest, + }, + OomEventResponse, WaitProcessResponse, WriteStreamResponse, +}; + +fn from_vec, T: Sized>(from: Vec) -> ::protobuf::RepeatedField { + let mut to: Vec = vec![]; + for data in from { + to.push(data.into()); + } + ::protobuf::RepeatedField::from_vec(to) +} + +fn into_vec>(from: ::protobuf::RepeatedField) -> Vec { + let mut to: Vec = vec![]; + for data in from.to_vec() { + to.push(data.into()); + } + to +} + +fn from_option>(from: Option) -> ::protobuf::SingularPtrField { + match from { + Some(f) => ::protobuf::SingularPtrField::from_option(Some(T::from(f))), + None => ::protobuf::SingularPtrField::none(), + } +} + +fn into_option, T: Sized>(from: ::protobuf::SingularPtrField) -> Option { + from.into_option().map(|f| f.into()) +} + +fn into_hash_map, T>( + from: std::collections::HashMap, +) -> std::collections::HashMap { + let mut to: std::collections::HashMap = Default::default(); + + for (key, value) in from { + to.insert(key, value.into()); + } + + to +} + +impl From for Empty { + fn from(_: empty::Empty) -> Self { + Self {} + } +} + +impl From for agent::StringUser { + fn from(from: StringUser) -> Self { + Self { + uid: from.uid, + gid: from.gid, + additionalGids: ::protobuf::RepeatedField::from_vec(from.additional_gids), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::Device { + fn from(from: Device) -> Self { + Self { + id: from.id, + field_type: from.field_type, + vm_path: from.vm_path, + container_path: from.container_path, + options: from_vec(from.options), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::Storage { + fn from(from: Storage) -> Self { + Self { + driver: from.driver, + driver_options: from_vec(from.driver_options), + source: from.source, + fstype: from.fs_type, + options: from_vec(from.options), + mount_point: from.mount_point, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::KernelModule { + fn from(from: KernelModule) -> Self { + Self { + name: from.name, + parameters: from_vec(from.parameters), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for types::IPFamily { + fn from(from: IPFamily) -> Self { + if from == IPFamily::V4 { + types::IPFamily::v4 + } else { + types::IPFamily::v6 + } + } +} + +impl From for IPFamily { + fn from(src: types::IPFamily) -> Self { + match src { + types::IPFamily::v4 => IPFamily::V4, + types::IPFamily::v6 => IPFamily::V6, + } + } +} + +impl From for types::IPAddress { + fn from(from: IPAddress) -> Self { + Self { + family: from.family.into(), + address: from.address, + mask: from.mask, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for IPAddress { + fn from(src: types::IPAddress) -> Self { + Self { + family: src.family.into(), + address: "".to_string(), + mask: "".to_string(), + } + } +} + +impl From for types::Interface { + fn from(from: Interface) -> Self { + Self { + device: from.device, + name: from.name, + IPAddresses: from_vec(from.ip_addresses), + mtu: from.mtu, + hwAddr: from.hw_addr, + pciPath: from.pci_addr, + field_type: from.field_type, + raw_flags: from.raw_flags, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for Interface { + fn from(src: types::Interface) -> Self { + Self { + device: src.device, + name: src.name, + ip_addresses: into_vec(src.IPAddresses), + mtu: src.mtu, + hw_addr: src.hwAddr, + pci_addr: src.pciPath, + field_type: src.field_type, + raw_flags: src.raw_flags, + } + } +} + +impl From for Interfaces { + fn from(src: agent::Interfaces) -> Self { + Self { + interfaces: into_vec(src.Interfaces), + } + } +} + +impl From for types::Route { + fn from(from: Route) -> Self { + Self { + dest: from.dest, + gateway: from.gateway, + device: from.device, + source: from.source, + scope: from.scope, + family: from.family.into(), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for Route { + fn from(src: types::Route) -> Self { + Self { + dest: src.dest, + gateway: src.gateway, + device: src.device, + source: src.source, + scope: src.scope, + family: src.family.into(), + } + } +} + +impl From for agent::Routes { + fn from(from: Routes) -> Self { + Self { + Routes: from_vec(from.routes), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for Routes { + fn from(src: agent::Routes) -> Self { + Self { + routes: into_vec(src.Routes), + } + } +} + +impl From for agent::CreateContainerRequest { + fn from(from: CreateContainerRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + string_user: from_option(from.string_user), + devices: from_vec(from.devices), + storages: from_vec(from.storages), + OCI: from_option(from.oci), + sandbox_pidns: from.sandbox_pidns, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::RemoveContainerRequest { + fn from(from: RemoveContainerRequest) -> Self { + Self { + container_id: from.container_id, + timeout: from.timeout, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::StartContainerRequest { + fn from(from: ContainerID) -> Self { + Self { + container_id: from.container_id, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::StatsContainerRequest { + fn from(from: ContainerID) -> Self { + Self { + container_id: from.container_id, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::PauseContainerRequest { + fn from(from: ContainerID) -> Self { + Self { + container_id: from.container_id, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::ResumeContainerRequest { + fn from(from: ContainerID) -> Self { + Self { + container_id: from.container_id, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::SignalProcessRequest { + fn from(from: SignalProcessRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + signal: from.signal, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::WaitProcessRequest { + fn from(from: WaitProcessRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::UpdateContainerRequest { + fn from(from: UpdateContainerRequest) -> Self { + Self { + container_id: from.container_id, + resources: from_option(Some(from.resources)), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::WriteStreamRequest { + fn from(from: WriteStreamRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + data: from.data, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for WriteStreamResponse { + fn from(from: agent::WriteStreamResponse) -> Self { + Self { length: from.len } + } +} + +impl From for agent::ExecProcessRequest { + fn from(from: ExecProcessRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + string_user: from_option(from.string_user), + process: from_option(from.process), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for CpuUsage { + fn from(src: agent::CpuUsage) -> Self { + Self { + total_usage: src.total_usage, + percpu_usage: src.percpu_usage, + usage_in_kernelmode: src.usage_in_kernelmode, + usage_in_usermode: src.usage_in_usermode, + } + } +} + +impl From for ThrottlingData { + fn from(src: agent::ThrottlingData) -> Self { + Self { + periods: src.periods, + throttled_periods: src.throttled_periods, + throttled_time: src.throttled_time, + } + } +} + +impl From for CpuStats { + fn from(src: agent::CpuStats) -> Self { + Self { + cpu_usage: into_option(src.cpu_usage), + throttling_data: into_option(src.throttling_data), + } + } +} + +impl From for MemoryData { + fn from(src: agent::MemoryData) -> Self { + Self { + usage: src.usage, + max_usage: src.max_usage, + failcnt: src.failcnt, + limit: src.limit, + } + } +} + +impl From for MemoryStats { + fn from(src: agent::MemoryStats) -> Self { + Self { + cache: src.cache, + usage: into_option(src.usage), + swap_usage: into_option(src.swap_usage), + kernel_usage: into_option(src.kernel_usage), + use_hierarchy: src.use_hierarchy, + stats: into_hash_map(src.stats), + } + } +} + +impl From for PidsStats { + fn from(src: agent::PidsStats) -> Self { + Self { + current: src.current, + limit: src.limit, + } + } +} + +impl From for BlkioStatsEntry { + fn from(src: agent::BlkioStatsEntry) -> Self { + Self { + major: src.major, + minor: src.minor, + op: src.op, + value: src.value, + } + } +} + +impl From for BlkioStats { + fn from(src: agent::BlkioStats) -> Self { + Self { + io_service_bytes_recursive: into_vec(src.io_service_bytes_recursive), + io_serviced_recursive: into_vec(src.io_serviced_recursive), + io_queued_recursive: into_vec(src.io_queued_recursive), + io_service_time_recursive: into_vec(src.io_service_time_recursive), + io_wait_time_recursive: into_vec(src.io_wait_time_recursive), + io_merged_recursive: into_vec(src.io_merged_recursive), + io_time_recursive: into_vec(src.io_time_recursive), + sectors_recursive: into_vec(src.sectors_recursive), + } + } +} + +impl From for HugetlbStats { + fn from(src: agent::HugetlbStats) -> Self { + Self { + usage: src.usage, + max_usage: src.max_usage, + failcnt: src.failcnt, + } + } +} + +impl From for CgroupStats { + fn from(src: agent::CgroupStats) -> Self { + Self { + cpu_stats: into_option(src.cpu_stats), + memory_stats: into_option(src.memory_stats), + pids_stats: into_option(src.pids_stats), + blkio_stats: into_option(src.blkio_stats), + hugetlb_stats: into_hash_map(src.hugetlb_stats), + } + } +} + +impl From for NetworkStats { + fn from(src: agent::NetworkStats) -> Self { + Self { + name: src.name, + rx_bytes: src.rx_bytes, + rx_packets: src.rx_packets, + rx_errors: src.rx_errors, + rx_dropped: src.rx_dropped, + tx_bytes: src.tx_bytes, + tx_packets: src.tx_packets, + tx_errors: src.tx_errors, + tx_dropped: src.tx_dropped, + } + } +} + +// translate ttrpc::agent response to interface::agent response +impl From for StatsContainerResponse { + fn from(src: agent::StatsContainerResponse) -> Self { + Self { + cgroup_stats: into_option(src.cgroup_stats), + network_stats: into_vec(src.network_stats), + } + } +} + +impl From for agent::ReadStreamRequest { + fn from(from: ReadStreamRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + len: from.len, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for ReadStreamResponse { + fn from(from: agent::ReadStreamResponse) -> Self { + Self { data: from.data } + } +} + +impl From for agent::CloseStdinRequest { + fn from(from: CloseStdinRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::TtyWinResizeRequest { + fn from(from: TtyWinResizeRequest) -> Self { + Self { + container_id: from.container_id, + exec_id: from.exec_id, + row: from.row, + column: from.column, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::UpdateInterfaceRequest { + fn from(from: UpdateInterfaceRequest) -> Self { + Self { + interface: from_option(from.interface), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::ListInterfacesRequest { + fn from(_: Empty) -> Self { + Self { + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::UpdateRoutesRequest { + fn from(from: UpdateRoutesRequest) -> Self { + Self { + routes: from_option(from.route), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::ListRoutesRequest { + fn from(_: Empty) -> Self { + Self { + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for types::ARPNeighbor { + fn from(from: ARPNeighbor) -> Self { + Self { + toIPAddress: from_option(from.to_ip_address), + device: from.device, + lladdr: from.ll_addr, + state: from.state, + flags: from.flags, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::ARPNeighbors { + fn from(from: ARPNeighbors) -> Self { + Self { + ARPNeighbors: from_vec(from.neighbors), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::AddARPNeighborsRequest { + fn from(from: AddArpNeighborRequest) -> Self { + Self { + neighbors: from_option(from.neighbors), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::CreateSandboxRequest { + fn from(from: CreateSandboxRequest) -> Self { + Self { + hostname: from.hostname, + dns: from_vec(from.dns), + storages: from_vec(from.storages), + sandbox_pidns: from.sandbox_pidns, + sandbox_id: from.sandbox_id, + guest_hook_path: from.guest_hook_path, + kernel_modules: from_vec(from.kernel_modules), + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::DestroySandboxRequest { + fn from(_: Empty) -> Self { + Self { + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::OnlineCPUMemRequest { + fn from(from: OnlineCPUMemRequest) -> Self { + Self { + wait: from.wait, + nb_cpus: from.nb_cpus, + cpu_only: from.cpu_only, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::ReseedRandomDevRequest { + fn from(from: ReseedRandomDevRequest) -> Self { + Self { + data: from.data, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::MemHotplugByProbeRequest { + fn from(from: MemHotplugByProbeRequest) -> Self { + Self { + memHotplugProbeAddr: from.mem_hotplug_probe_addr, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for agent::SetGuestDateTimeRequest { + fn from(from: SetGuestDateTimeRequest) -> Self { + Self { + Sec: from.sec, + Usec: from.usec, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for AgentDetails { + fn from(src: agent::AgentDetails) -> Self { + Self { + version: src.version, + init_daemon: src.init_daemon, + device_handlers: into_vec(src.device_handlers), + storage_handlers: into_vec(src.storage_handlers), + supports_seccomp: src.supports_seccomp, + } + } +} + +impl From for GuestDetailsResponse { + fn from(src: agent::GuestDetailsResponse) -> Self { + Self { + mem_block_size_bytes: src.mem_block_size_bytes, + agent_details: into_option(src.agent_details), + support_mem_hotplug_probe: src.support_mem_hotplug_probe, + } + } +} + +impl From for agent::CopyFileRequest { + fn from(from: CopyFileRequest) -> Self { + Self { + path: from.path, + file_size: from.file_size, + file_mode: from.file_mode, + dir_mode: from.dir_mode, + uid: from.uid, + gid: from.gid, + offset: from.offset, + data: from.data, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for WaitProcessResponse { + fn from(from: agent::WaitProcessResponse) -> Self { + Self { + status: from.status, + } + } +} + +impl From for agent::GetOOMEventRequest { + fn from(_: Empty) -> Self { + Self { + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for health::CheckRequest { + fn from(from: CheckRequest) -> Self { + Self { + service: from.service, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + +impl From for HealthCheckResponse { + fn from(from: health::HealthCheckResponse) -> Self { + Self { + status: from.status as u32, + } + } +} + +impl From for VersionCheckResponse { + fn from(from: health::VersionCheckResponse) -> Self { + Self { + grpc_version: from.grpc_version, + agent_version: from.agent_version, + } + } +} + +impl From for OomEventResponse { + fn from(from: OOMEvent) -> Self { + Self { + container_id: from.container_id, + } + } +} diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs new file mode 100644 index 000000000000..a9d8a15f923c --- /dev/null +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -0,0 +1,84 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[macro_use] +extern crate slog; + +macro_rules! sl { + () => { + slog_scope::logger().new(slog::o!("subsystem" => "agent")) + }; +} + +pub mod kata; +mod log_forwarder; +mod sock; +mod types; +pub use types::{ + ARPNeighbor, ARPNeighbors, AddArpNeighborRequest, BlkioStatsEntry, CheckRequest, + CloseStdinRequest, ContainerID, CopyFileRequest, CreateContainerRequest, CreateSandboxRequest, + Empty, ExecProcessRequest, GetGuestDetailsRequest, GuestDetailsResponse, HealthCheckResponse, + IPAddress, IPFamily, Interface, Interfaces, ListProcessesRequest, MemHotplugByProbeRequest, + OnlineCPUMemRequest, OomEventResponse, ReadStreamRequest, ReadStreamResponse, + RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, SetGuestDateTimeRequest, + SignalProcessRequest, StatsContainerResponse, Storage, TtyWinResizeRequest, + UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, + WaitProcessRequest, WaitProcessResponse, WriteStreamRequest, WriteStreamResponse, +}; + +use anyhow::Result; +use async_trait::async_trait; + +#[async_trait] +pub trait AgentManager: Send + Sync { + async fn set_socket_address(&self, address: &str) -> Result<()>; + async fn start(&self) -> Result<()>; + async fn stop(&self); +} + +#[async_trait] +pub trait HealthService: Send + Sync { + async fn check(&self, req: CheckRequest) -> Result; + async fn version(&self, req: CheckRequest) -> Result; +} + +#[async_trait] +pub trait Agent: AgentManager + HealthService + Send + Sync { + // sandbox + async fn create_sandbox(&self, req: CreateSandboxRequest) -> Result; + async fn destroy_sandbox(&self, req: Empty) -> Result; + + // network + async fn add_arp_neighbors(&self, req: AddArpNeighborRequest) -> Result; + async fn list_interfaces(&self, req: Empty) -> Result; + async fn list_routes(&self, req: Empty) -> Result; + async fn update_interface(&self, req: UpdateInterfaceRequest) -> Result; + async fn update_routes(&self, req: UpdateRoutesRequest) -> Result; + // container + async fn create_container(&self, req: CreateContainerRequest) -> Result; + async fn pause_container(&self, req: ContainerID) -> Result; + async fn remove_container(&self, req: RemoveContainerRequest) -> Result; + async fn resume_container(&self, req: ContainerID) -> Result; + async fn start_container(&self, req: ContainerID) -> Result; + async fn stats_container(&self, req: ContainerID) -> Result; + async fn update_container(&self, req: UpdateContainerRequest) -> Result; + + // process + async fn exec_process(&self, req: ExecProcessRequest) -> Result; + async fn signal_process(&self, req: SignalProcessRequest) -> Result; + async fn wait_process(&self, req: WaitProcessRequest) -> Result; + + // io and tty + async fn close_stdin(&self, req: CloseStdinRequest) -> Result; + async fn read_stderr(&self, req: ReadStreamRequest) -> Result; + async fn read_stdout(&self, req: ReadStreamRequest) -> Result; + async fn tty_win_resize(&self, req: TtyWinResizeRequest) -> Result; + async fn write_stdin(&self, req: WriteStreamRequest) -> Result; + + // utils + async fn copy_file(&self, req: CopyFileRequest) -> Result; + async fn get_oom_event(&self, req: Empty) -> Result; +} diff --git a/src/runtime-rs/crates/agent/src/log_forwarder.rs b/src/runtime-rs/crates/agent/src/log_forwarder.rs new file mode 100644 index 000000000000..73c668f2be36 --- /dev/null +++ b/src/runtime-rs/crates/agent/src/log_forwarder.rs @@ -0,0 +1,159 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; +use tokio::io::{AsyncBufReadExt, BufReader}; + +use crate::sock; + +// https://github.com/slog-rs/slog/blob/master/src/lib.rs#L2082 +const LOG_LEVEL_TRACE: &str = "TRCE"; +const LOG_LEVEL_DEBUG: &str = "DEBG"; +const LOG_LEVEL_INFO: &str = "INFO"; +const LOG_LEVEL_WARNING: &str = "WARN"; +const LOG_LEVEL_ERROR: &str = "ERRO"; +const LOG_LEVEL_CRITICAL: &str = "CRIT"; + +pub(crate) struct LogForwarder { + task_handler: Option>, +} + +impl LogForwarder { + pub(crate) fn new() -> Self { + Self { task_handler: None } + } + + pub(crate) fn stop(&mut self) { + let task_handler = self.task_handler.take(); + if let Some(handler) = task_handler { + handler.abort(); + info!(sl!(), "abort log forwarder thread"); + } + } + + // start connect kata-agent log vsock and copy data to hypervisor's log stream + pub(crate) async fn start( + &mut self, + address: &str, + port: u32, + config: sock::ConnectConfig, + ) -> Result<()> { + let logger = sl!().clone(); + let address = address.to_string(); + let task_handler = tokio::spawn(async move { + loop { + info!(logger, "try to connect to get agent log"); + let sock = match sock::new(&address, port) { + Ok(sock) => sock, + Err(err) => { + error!( + sl!(), + "failed to new sock for address {:?} port {} error {:?}", + address, + port, + err + ); + return; + } + }; + + match sock.connect(&config).await { + Ok(stream) => { + let stream = BufReader::new(stream); + let mut lines = stream.lines(); + while let Ok(line) = lines.next_line().await { + if let Some(l) = line { + match parse_agent_log_level(&l) { + LOG_LEVEL_TRACE => trace!(sl!(), "{}", l), + LOG_LEVEL_DEBUG => debug!(sl!(), "{}", l), + LOG_LEVEL_WARNING => warn!(sl!(), "{}", l), + LOG_LEVEL_ERROR => error!(sl!(), "{}", l), + LOG_LEVEL_CRITICAL => crit!(sl!(), "{}", l), + _ => info!(sl!(), "{}", l), + } + } + } + } + Err(err) => { + warn!(logger, "connect agent vsock failed: {:?}", err); + } + } + } + }); + self.task_handler = Some(task_handler); + Ok(()) + } +} + +pub fn parse_agent_log_level(s: &str) -> &str { + let v: serde_json::Result = serde_json::from_str(s); + match v { + Err(_err) => LOG_LEVEL_INFO, + Ok(val) => { + match &val["level"] { + serde_json::Value::String(s) => match s.as_str() { + LOG_LEVEL_TRACE => LOG_LEVEL_TRACE, + LOG_LEVEL_DEBUG => LOG_LEVEL_DEBUG, + LOG_LEVEL_WARNING => LOG_LEVEL_WARNING, + LOG_LEVEL_ERROR => LOG_LEVEL_ERROR, + LOG_LEVEL_CRITICAL => LOG_LEVEL_CRITICAL, + _ => LOG_LEVEL_INFO, // info or other values will return info, + }, + _ => LOG_LEVEL_INFO, // info or other values will return info, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::parse_agent_log_level; + + #[test] + fn test_parse_agent_log_level() { + let cases = vec![ + // normal cases + ( + r#"{"msg":"child exited unexpectedly","level":"TRCE"}"#, + super::LOG_LEVEL_TRACE, + ), + ( + r#"{"msg":"child exited unexpectedly","level":"DEBG"}"#, + super::LOG_LEVEL_DEBUG, + ), + ( + r#"{"msg":"child exited unexpectedly","level":"INFO"}"#, + super::LOG_LEVEL_INFO, + ), + ( + r#"{"msg":"child exited unexpectedly","level":"WARN"}"#, + super::LOG_LEVEL_WARNING, + ), + ( + r#"{"msg":"child exited unexpectedly","level":"ERRO"}"#, + super::LOG_LEVEL_ERROR, + ), + ( + r#"{"msg":"child exited unexpectedly","level":"CRIT"}"#, + super::LOG_LEVEL_CRITICAL, + ), + ( + r#"{"msg":"child exited unexpectedly","level":"abc"}"#, + super::LOG_LEVEL_INFO, + ), + // exception cases + (r#"{"not a valid json struct"}"#, super::LOG_LEVEL_INFO), + ("not a valid json struct", super::LOG_LEVEL_INFO), + ]; + + for case in cases.iter() { + let s = case.0; + let result = parse_agent_log_level(s); + let excepted = case.1; + assert_eq!(result, excepted); + } + } +} diff --git a/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs b/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs new file mode 100644 index 000000000000..7079ea392e4d --- /dev/null +++ b/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs @@ -0,0 +1,81 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::os::unix::prelude::AsRawFd; + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::UnixStream, +}; + +use super::{ConnectConfig, Sock, Stream}; + +unsafe impl Send for HybridVsock {} +unsafe impl Sync for HybridVsock {} + +#[derive(Debug, PartialEq)] +pub struct HybridVsock { + uds: String, + port: u32, +} + +impl HybridVsock { + pub fn new(uds: &str, port: u32) -> Self { + Self { + uds: uds.to_string(), + port, + } + } +} + +#[async_trait] +impl Sock for HybridVsock { + async fn connect(&self, config: &ConnectConfig) -> Result { + let retry_times = config.reconnect_timeout_ms / config.dial_timeout_ms; + for i in 0..retry_times { + match connect_helper(&self.uds, self.port).await { + Ok(stream) => { + info!( + sl!(), + "connect success on {} current client fd {}", + i, + stream.as_raw_fd() + ); + return Ok(Stream::Unix(stream)); + } + Err(err) => { + debug!(sl!(), "connect on {} err : {:?}", i, err); + tokio::time::sleep(std::time::Duration::from_millis(config.dial_timeout_ms)) + .await; + continue; + } + } + } + Err(anyhow!("cannot connect to agent ttrpc server")) + } +} + +async fn connect_helper(uds: &str, port: u32) -> Result { + info!(sl!(), "connect uds {:?} port {}", &uds, port); + let mut stream = UnixStream::connect(&uds).await.context("connect")?; + stream + .write_all(format!("connect {}\n", port).as_bytes()) + .await + .context("write all")?; + let mut reads = BufReader::new(&mut stream); + let mut response = String::new(); + reads.read_line(&mut response).await.context("read line")?; + //info!(sl!(), "get socket resp: {}", response); + if !response.contains("OK") { + return Err(anyhow!( + "handshake error: malformed response code: {:?}", + response + )); + } + Ok(stream) +} diff --git a/src/runtime-rs/crates/agent/src/sock/mod.rs b/src/runtime-rs/crates/agent/src/sock/mod.rs new file mode 100644 index 000000000000..52ea993b503b --- /dev/null +++ b/src/runtime-rs/crates/agent/src/sock/mod.rs @@ -0,0 +1,159 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod hybrid_vsock; +pub use hybrid_vsock::HybridVsock; +mod vsock; +pub use vsock::Vsock; + +use std::{ + pin::Pin, + task::{Context as TaskContext, Poll}, + { + os::unix::{io::IntoRawFd, prelude::RawFd}, + sync::Arc, + }, +}; + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use tokio::{ + io::{AsyncRead, ReadBuf}, + net::UnixStream, +}; +use url::Url; + +const VSOCK_SCHEME: &str = "vsock"; +const HYBRID_VSOCK_SCHEME: &str = "hvsock"; + +/// Socket stream +pub enum Stream { + // hvsock://:. Firecracker/Dragonball implements the virtio-vsock device + // model, and mediates communication between AF_UNIX sockets (on the host end) + // and AF_VSOCK sockets (on the guest end). + Unix(UnixStream), + // TODO: support vsock + // vsock://: +} + +impl Stream { + fn poll_read_priv( + &mut self, + cx: &mut TaskContext<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + // Safety: `UnixStream::read` correctly handles reads into uninitialized memory + match self { + Stream::Unix(stream) => Pin::new(stream).poll_read(cx, buf), + } + } +} + +impl IntoRawFd for Stream { + fn into_raw_fd(self) -> RawFd { + match self { + Stream::Unix(stream) => match stream.into_std() { + Ok(stream) => stream.into_raw_fd(), + Err(err) => { + error!(sl!(), "failed to into std unix stream {:?}", err); + -1 + } + }, + } + } +} + +impl AsyncRead for Stream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut TaskContext<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + // we know this is safe because doesn't moved + let me = unsafe { self.get_unchecked_mut() }; + me.poll_read_priv(cx, buf) + } +} + +/// Connect config +pub struct ConnectConfig { + dial_timeout_ms: u64, + reconnect_timeout_ms: u64, +} + +impl ConnectConfig { + pub fn new(dial_timeout_ms: u64, reconnect_timeout_ms: u64) -> Self { + Self { + dial_timeout_ms, + reconnect_timeout_ms, + } + } +} + +#[derive(Debug, PartialEq)] +enum SockType { + Vsock(Vsock), + HybridVsock(HybridVsock), +} + +#[async_trait] +pub trait Sock: Send + Sync { + async fn connect(&self, config: &ConnectConfig) -> Result; +} + +// Supported sock address formats are: +// - vsock://: +// - hvsock://:. Firecracker implements the virtio-vsock device +// model, and mediates communication between AF_UNIX sockets (on the host end) +// and AF_VSOCK sockets (on the guest end). +pub fn new(address: &str, port: u32) -> Result> { + match parse(address, port).context("parse url")? { + SockType::Vsock(sock) => Ok(Arc::new(sock)), + SockType::HybridVsock(sock) => Ok(Arc::new(sock)), + } +} + +fn parse(address: &str, port: u32) -> Result { + let url = Url::parse(address).context("parse url")?; + match url.scheme() { + VSOCK_SCHEME => { + let cid = url + .host_str() + .unwrap_or_default() + .parse::() + .context("parse cid")?; + Ok(SockType::Vsock(Vsock::new(cid, port))) + } + HYBRID_VSOCK_SCHEME => { + let path: Vec<&str> = url.path().split(':').collect(); + if path.len() != 1 { + return Err(anyhow!("invalid path {:?}", path)); + } + let uds = path[0]; + Ok(SockType::HybridVsock(HybridVsock::new(uds, port))) + } + _ => Err(anyhow!("Unsupported scheme")), + } +} + +#[cfg(test)] +mod test { + use super::{hybrid_vsock::HybridVsock, parse, vsock::Vsock, SockType}; + + #[test] + fn test_parse_url() { + // check vsock + let vsock = parse("vsock://123", 456).unwrap(); + assert_eq!(vsock, SockType::Vsock(Vsock::new(123, 456))); + + // check hybrid vsock + let hvsock = parse("hvsock:///tmp/test.hvsock", 456).unwrap(); + assert_eq!( + hvsock, + SockType::HybridVsock(HybridVsock::new("/tmp/test.hvsock", 456)) + ); + } +} diff --git a/src/runtime-rs/crates/agent/src/sock/vsock.rs b/src/runtime-rs/crates/agent/src/sock/vsock.rs new file mode 100644 index 000000000000..7f6a59d89a0d --- /dev/null +++ b/src/runtime-rs/crates/agent/src/sock/vsock.rs @@ -0,0 +1,32 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; +use async_trait::async_trait; + +use super::{ConnectConfig, Sock, Stream}; + +unsafe impl Send for Vsock {} +unsafe impl Sync for Vsock {} + +#[derive(Debug, PartialEq)] +pub struct Vsock { + cid: u32, + port: u32, +} + +impl Vsock { + pub fn new(cid: u32, port: u32) -> Self { + Self { cid, port } + } +} + +#[async_trait] +impl Sock for Vsock { + async fn connect(&self, _config: &ConnectConfig) -> Result { + todo!() + } +} diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs new file mode 100644 index 000000000000..41e0e3ee36ff --- /dev/null +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -0,0 +1,454 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use serde::Deserialize; + +#[derive(PartialEq, Clone, Default)] +pub struct Empty {} + +impl Empty { + pub fn new() -> Self { + Self::default() + } +} + +#[derive(PartialEq, Clone, Default)] +pub struct StringUser { + pub uid: String, + pub gid: String, + pub additional_gids: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct Device { + pub id: String, + pub field_type: String, + pub vm_path: String, + pub container_path: String, + pub options: Vec, +} + +#[derive(Debug, PartialEq, Clone, Default)] +pub struct Storage { + pub driver: String, + pub driver_options: Vec, + pub source: String, + pub fs_type: String, + pub options: Vec, + pub mount_point: String, +} + +#[derive(Deserialize, Clone, PartialEq, Eq, Debug, Hash)] +pub enum IPFamily { + V4 = 0, + V6 = 1, +} + +impl ::std::default::Default for IPFamily { + fn default() -> Self { + IPFamily::V4 + } +} + +#[derive(Deserialize, Debug, PartialEq, Clone, Default)] +pub struct IPAddress { + pub family: IPFamily, + pub address: String, + pub mask: String, +} + +#[derive(Deserialize, Debug, PartialEq, Clone, Default)] +pub struct Interface { + pub device: String, + pub name: String, + pub ip_addresses: Vec, + pub mtu: u64, + pub hw_addr: String, + #[serde(default)] + pub pci_addr: String, + #[serde(default)] + pub field_type: String, + #[serde(default)] + pub raw_flags: u32, +} + +#[derive(PartialEq, Clone, Default)] +pub struct Interfaces { + pub interfaces: Vec, +} + +#[derive(Deserialize, Debug, PartialEq, Clone, Default)] +pub struct Route { + pub dest: String, + pub gateway: String, + pub device: String, + pub source: String, + pub scope: u32, + pub family: IPFamily, +} + +#[derive(Deserialize, Debug, PartialEq, Clone, Default)] +pub struct Routes { + pub routes: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct CreateContainerRequest { + pub container_id: String, + pub exec_id: String, + pub string_user: Option, + pub devices: Vec, + pub storages: Vec, + pub oci: Option, + pub guest_hooks: Option, + pub sandbox_pidns: bool, + pub rootfs_mounts: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct ContainerID { + pub container_id: String, +} + +impl ContainerID { + pub fn new(id: &str) -> Self { + Self { + container_id: id.to_string(), + } + } +} + +#[derive(PartialEq, Clone, Debug, Default)] +pub struct RemoveContainerRequest { + pub container_id: String, + pub timeout: u32, +} + +impl RemoveContainerRequest { + pub fn new(id: &str, timeout: u32) -> Self { + Self { + container_id: id.to_string(), + timeout, + } + } +} + +#[derive(PartialEq, Clone, Default)] +pub struct SignalProcessRequest { + pub container_id: String, + pub exec_id: String, + pub signal: u32, +} + +#[derive(PartialEq, Clone, Default)] +pub struct WaitProcessRequest { + pub container_id: String, + pub exec_id: String, +} + +#[derive(PartialEq, Clone, Default)] +pub struct ListProcessesRequest { + pub container_id: String, + pub format: String, + pub args: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct UpdateContainerRequest { + pub container_id: String, + pub resources: oci::LinuxResources, + pub mounts: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct WriteStreamRequest { + pub container_id: String, + pub exec_id: String, + pub data: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct WriteStreamResponse { + pub length: u32, +} + +#[derive(PartialEq, Clone, Default)] +pub struct ExecProcessRequest { + pub container_id: String, + pub exec_id: String, + pub string_user: Option, + pub process: Option, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct CpuUsage { + pub total_usage: u64, + pub percpu_usage: ::std::vec::Vec, + pub usage_in_kernelmode: u64, + pub usage_in_usermode: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct ThrottlingData { + pub periods: u64, + pub throttled_periods: u64, + pub throttled_time: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct LoadData { + pub one: String, + pub five: String, + pub fifteen: String, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct CpuStats { + pub cpu_usage: Option, + pub throttling_data: Option, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct MemoryData { + pub usage: u64, + pub max_usage: u64, + pub failcnt: u64, + pub limit: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct MemoryStats { + pub cache: u64, + pub usage: Option, + pub swap_usage: Option, + pub kernel_usage: Option, + pub use_hierarchy: bool, + pub stats: ::std::collections::HashMap, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct PidsStats { + pub current: u64, + pub limit: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct BlkioStatsEntry { + pub major: u64, + pub minor: u64, + pub op: String, + pub value: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct BlkioStats { + pub io_service_bytes_recursive: Vec, + pub io_serviced_recursive: Vec, + pub io_queued_recursive: Vec, + pub io_service_time_recursive: Vec, + pub io_wait_time_recursive: Vec, + pub io_merged_recursive: Vec, + pub io_time_recursive: Vec, + pub sectors_recursive: Vec, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct HugetlbStats { + pub usage: u64, + pub max_usage: u64, + pub failcnt: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct CgroupStats { + pub cpu_stats: Option, + pub memory_stats: Option, + pub pids_stats: Option, + pub blkio_stats: Option, + pub hugetlb_stats: ::std::collections::HashMap, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct NetworkStats { + pub name: String, + pub rx_bytes: u64, + pub rx_packets: u64, + pub rx_errors: u64, + pub rx_dropped: u64, + pub tx_bytes: u64, + pub tx_packets: u64, + pub tx_errors: u64, + pub tx_dropped: u64, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct StatsContainerResponse { + pub cgroup_stats: Option, + pub network_stats: Vec, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct WaitProcessResponse { + pub status: i32, +} + +#[derive(PartialEq, Clone, Default)] +pub struct ReadStreamRequest { + pub container_id: String, + pub exec_id: String, + pub len: u32, +} + +#[derive(PartialEq, Clone, Default)] +pub struct ReadStreamResponse { + pub data: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct CloseStdinRequest { + pub container_id: String, + pub exec_id: String, +} + +#[derive(PartialEq, Clone, Default)] +pub struct TtyWinResizeRequest { + pub container_id: String, + pub exec_id: String, + pub row: u32, + pub column: u32, +} + +#[derive(Debug, PartialEq, Clone, Default)] +pub struct UpdateInterfaceRequest { + pub interface: Option, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct UpdateRoutesRequest { + pub route: Option, +} + +#[derive(Deserialize, PartialEq, Clone, Default, Debug)] +pub struct ARPNeighbor { + pub to_ip_address: Option, + pub device: String, + pub ll_addr: String, + pub state: i32, + pub flags: i32, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct ARPNeighbors { + pub neighbors: Vec, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct AddArpNeighborRequest { + pub neighbors: Option, +} + +#[derive(PartialEq, Clone, Default)] +pub struct KernelModule { + pub name: String, + pub parameters: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct CreateSandboxRequest { + pub hostname: String, + pub dns: Vec, + pub storages: Vec, + pub sandbox_pidns: bool, + pub sandbox_id: String, + pub guest_hook_path: String, + pub kernel_modules: Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct OnlineCPUMemRequest { + pub wait: bool, + pub nb_cpus: u32, + pub cpu_only: bool, +} + +#[derive(PartialEq, Clone, Default)] +pub struct ReseedRandomDevRequest { + pub data: ::std::vec::Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct GetGuestDetailsRequest { + pub mem_block_size: bool, + pub mem_hotplug_probe: bool, +} + +#[derive(PartialEq, Clone, Default)] +pub struct MemHotplugByProbeRequest { + pub mem_hotplug_probe_addr: ::std::vec::Vec, +} + +#[derive(PartialEq, Clone, Default)] +pub struct SetGuestDateTimeRequest { + pub sec: i64, + pub usec: i64, +} + +#[derive(PartialEq, Clone, Default)] +pub struct AgentDetails { + pub version: String, + pub init_daemon: bool, + pub device_handlers: Vec, + pub storage_handlers: Vec, + pub supports_seccomp: bool, +} + +#[derive(PartialEq, Clone, Default)] +pub struct GuestDetailsResponse { + pub mem_block_size_bytes: u64, + pub agent_details: Option, + pub support_mem_hotplug_probe: bool, +} + +#[derive(PartialEq, Clone, Default)] +pub struct CopyFileRequest { + pub path: String, + pub file_size: i64, + pub file_mode: u32, + pub dir_mode: u32, + pub uid: i32, + pub gid: i32, + pub offset: i64, + pub data: ::std::vec::Vec, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct CheckRequest { + pub service: String, +} + +impl CheckRequest { + pub fn new(service: &str) -> Self { + Self { + service: service.to_string(), + } + } +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct HealthCheckResponse { + pub status: u32, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct VersionCheckResponse { + pub grpc_version: String, + pub agent_version: String, +} + +#[derive(PartialEq, Clone, Default, Debug)] +pub struct OomEventResponse { + pub container_id: String, +} From bdfee005fa3a6b5114ae7da63bb5e7f0720d9ac7 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Fri, 3 Dec 2021 18:53:48 +0800 Subject: [PATCH 0042/1953] runtime-rs: service and runtime framework 1. service: Responsible for processing services, such as task service, image service 2. Responsible for implementing different runtimes, such as Virt-container, Linux-container, Wasm-container Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/agent/src/rpc.rs | 2 +- src/libs/kata-sys-util/src/validate.rs | 6 +- src/libs/kata-types/src/config/runtime.rs | 6 + src/libs/logging/src/lib.rs | 11 + src/libs/oci/src/lib.rs | 2 + src/runtime-rs/Cargo.lock | 155 +++++++++-- src/runtime-rs/crates/agent/src/sock/mod.rs | 6 +- src/runtime-rs/crates/agent/src/sock/vsock.rs | 6 +- src/runtime-rs/crates/runtimes/Cargo.toml | 28 ++ .../crates/runtimes/common/Cargo.toml | 26 ++ .../runtimes/common/src/container_manager.rs | 40 +++ .../crates/runtimes/common/src/error.rs | 17 ++ .../crates/runtimes/common/src/lib.rs | 15 ++ .../crates/runtimes/common/src/message.rs | 44 +++ .../runtimes/common/src/runtime_handler.rs | 38 +++ .../crates/runtimes/common/src/sandbox.rs | 16 ++ .../crates/runtimes/common/src/types/mod.rs | 219 +++++++++++++++ .../common/src/types/trans_from_shim.rs | 198 ++++++++++++++ .../common/src/types/trans_into_shim.rs | 242 +++++++++++++++++ .../runtimes/linux_container/Cargo.toml | 12 + .../runtimes/linux_container/src/lib.rs | 42 +++ src/runtime-rs/crates/runtimes/src/lib.rs | 13 + src/runtime-rs/crates/runtimes/src/manager.rs | 251 ++++++++++++++++++ .../crates/runtimes/virt_container/Cargo.toml | 13 + .../crates/runtimes/virt_container/src/lib.rs | 47 ++++ .../crates/runtimes/wasm_container/Cargo.toml | 12 + .../crates/runtimes/wasm_container/src/lib.rs | 42 +++ src/runtime-rs/crates/service/Cargo.toml | 18 ++ src/runtime-rs/crates/service/src/lib.rs | 14 + src/runtime-rs/crates/service/src/manager.rs | 107 ++++++++ .../crates/service/src/task_service.rs | 82 ++++++ src/runtime-rs/crates/shim/Cargo.toml | 4 +- src/runtime-rs/crates/shim/src/args.rs | 4 +- src/runtime-rs/crates/shim/src/lib.rs | 6 +- src/runtime-rs/crates/shim/src/shim.rs | 2 +- src/runtime-rs/crates/shim/src/shim_delete.rs | 43 +-- src/runtime-rs/crates/shim/src/shim_run.rs | 7 +- 37 files changed, 1713 insertions(+), 83 deletions(-) create mode 100644 src/runtime-rs/crates/runtimes/Cargo.toml create mode 100644 src/runtime-rs/crates/runtimes/common/Cargo.toml create mode 100644 src/runtime-rs/crates/runtimes/common/src/container_manager.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/error.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/lib.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/message.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/sandbox.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/types/mod.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs create mode 100644 src/runtime-rs/crates/runtimes/linux_container/Cargo.toml create mode 100644 src/runtime-rs/crates/runtimes/linux_container/src/lib.rs create mode 100644 src/runtime-rs/crates/runtimes/src/lib.rs create mode 100644 src/runtime-rs/crates/runtimes/src/manager.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/Cargo.toml create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/lib.rs create mode 100644 src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml create mode 100644 src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs create mode 100644 src/runtime-rs/crates/service/Cargo.toml create mode 100644 src/runtime-rs/crates/service/src/lib.rs create mode 100644 src/runtime-rs/crates/service/src/manager.rs create mode 100644 src/runtime-rs/crates/service/src/task_service.rs diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 5e9bc7696f94..e4c57c709335 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -142,7 +142,7 @@ impl AgentService { ) -> Result<()> { let cid = req.container_id.clone(); - kata_sys_util::validate::verify_cid(&cid)?; + kata_sys_util::validate::verify_id(&cid)?; let mut oci_spec = req.OCI.clone(); let use_sandbox_pidns = req.get_sandbox_pidns(); diff --git a/src/libs/kata-sys-util/src/validate.rs b/src/libs/kata-sys-util/src/validate.rs index a58b19289ad6..0847398cefe2 100644 --- a/src/libs/kata-sys-util/src/validate.rs +++ b/src/libs/kata-sys-util/src/validate.rs @@ -10,11 +10,11 @@ pub enum Error { InvalidContainerID(String), } -// A container ID must match this regex: +// A container ID or exec ID must match this regex: // // ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$ // -pub fn verify_cid(id: &str) -> Result<(), Error> { +pub fn verify_id(id: &str) -> Result<(), Error> { let mut chars = id.chars(); let valid = match chars.next() { @@ -253,7 +253,7 @@ mod tests { for (i, d) in tests.iter().enumerate() { let msg = format!("test[{}]: {:?}", i, d); - let result = verify_cid(d.id); + let result = verify_id(d.id); let msg = format!("{}, result: {:?}", msg, result); diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 7c417fdc2750..75b25f166d4c 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -13,6 +13,10 @@ use crate::{eother, resolve_path, validate_path}; /// Kata runtime configuration information. #[derive(Debug, Default, Deserialize, Serialize)] pub struct Runtime { + /// Runtime name: Plan to support virt-container, linux-container, wasm-container + #[serde(default)] + pub name: String, + /// If enabled, the runtime will log additional debug messages to the system log. #[serde(default, rename = "enable_debug")] pub debug: bool, @@ -238,6 +242,7 @@ vfio_mode = "guest_kernel" fn test_config() { let content = r#" [runtime] +name = "virt-container" enable_debug = true experimental = ["a", "b"] internetworking_model = "macvtap" @@ -255,6 +260,7 @@ field_should_be_ignored = true "#; let config: TomlConfig = TomlConfig::load(content).unwrap(); config.validate().unwrap(); + assert_eq!(&config.runtime.name, "virt-container"); assert!(config.runtime.debug); assert_eq!(config.runtime.experimental.len(), 2); assert_eq!(&config.runtime.experimental[0], "a"); diff --git a/src/libs/logging/src/lib.rs b/src/libs/logging/src/lib.rs index 33f9fee3ee2c..2c90b5bd0fe7 100644 --- a/src/libs/logging/src/lib.rs +++ b/src/libs/logging/src/lib.rs @@ -17,6 +17,17 @@ mod log_writer; pub use file_rotate::FileRotator; pub use log_writer::LogWriter; +#[macro_export] +macro_rules! logger_with_subsystem { + ($name: ident, $subsystem: expr) => { + macro_rules! $name { + () => { + slog_scope::logger().new(slog::o!("subsystem" => $subsystem)) + }; + } + }; +} + const LOG_LEVELS: &[(&str, slog::Level)] = &[ ("trace", slog::Level::Trace), ("debug", slog::Level::Debug), diff --git a/src/libs/oci/src/lib.rs b/src/libs/oci/src/lib.rs index f47f2df4be06..ace2238ef36f 100644 --- a/src/libs/oci/src/lib.rs +++ b/src/libs/oci/src/lib.rs @@ -14,6 +14,8 @@ use std::collections::HashMap; mod serialize; pub use serialize::{to_string, to_writer, Error, Result}; +pub const OCI_SPEC_CONFIG_FILE_NAME: &str = "config.json"; + #[allow(dead_code)] fn is_false(b: bool) -> bool { !b diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 7f3db194da5b..4dda25cd7c33 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -157,7 +157,7 @@ checksum = "cdae996d9638ba03253ffa1c93345a585974a97abbdeab9176c77922f3efc1e8" dependencies = [ "libc", "log", - "nix 0.23.1", + "nix", "regex", ] @@ -174,6 +174,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "common" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "containerd-shim-protos", + "kata-sys-util", + "kata-types", + "lazy_static", + "nix", + "oci", + "protobuf", + "serde_json", + "slog", + "slog-scope", + "strum", + "thiserror", + "tokio", + "ttrpc", +] + [[package]] name = "common-path" version = "1.0.0" @@ -476,6 +498,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -559,7 +587,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix 0.23.1", + "nix", "oci", "once_cell", "serde_json", @@ -622,6 +650,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux_container" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "common", + "tokio", +] + [[package]] name = "lock_api" version = "0.4.6" @@ -711,19 +749,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nix" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" -dependencies = [ - "bitflags", - "cc", - "cfg-if 0.1.10", - "libc", - "void", -] - [[package]] name = "nix" version = "0.23.1" @@ -930,7 +955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" dependencies = [ "bytes 1.1.0", - "heck", + "heck 0.3.3", "itertools", "log", "multimap", @@ -1078,6 +1103,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "runtimes" +version = "0.1.0" +dependencies = [ + "anyhow", + "common", + "kata-types", + "lazy_static", + "linux_container", + "logging", + "oci", + "slog", + "slog-scope", + "tokio", + "virt_container", + "wasm_container", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -1170,6 +1213,22 @@ dependencies = [ "syn", ] +[[package]] +name = "service" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "common", + "containerd-shim-protos", + "logging", + "runtimes", + "slog", + "slog-scope", + "tokio", + "ttrpc", +] + [[package]] name = "sha2" version = "0.9.3" @@ -1196,10 +1255,12 @@ dependencies = [ "libc", "log", "logging", - "nix 0.16.1", + "nix", "oci", "protobuf", + "rand", "serial_test", + "service", "sha2", "slog", "slog-async", @@ -1213,6 +1274,15 @@ dependencies = [ "vergen", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.5" @@ -1287,6 +1357,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "strum" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" +dependencies = [ + "heck 0.4.0", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subprocess" version = "0.2.8" @@ -1411,7 +1503,9 @@ dependencies = [ "memchr", "mio", "num_cpus", + "once_cell", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "winapi", @@ -1461,7 +1555,7 @@ dependencies = [ "futures 0.3.21", "libc", "log", - "nix 0.23.1", + "nix", "protobuf", "protobuf-codegen-pure", "thiserror", @@ -1580,10 +1674,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +name = "virt_container" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "common", + "kata-types", + "tokio", +] [[package]] name = "vsock" @@ -1592,7 +1691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" dependencies = [ "libc", - "nix 0.23.1", + "nix", ] [[package]] @@ -1607,6 +1706,16 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm_container" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "common", + "tokio", +] + [[package]] name = "which" version = "4.2.5" diff --git a/src/runtime-rs/crates/agent/src/sock/mod.rs b/src/runtime-rs/crates/agent/src/sock/mod.rs index 52ea993b503b..3ec4da6aa9b1 100644 --- a/src/runtime-rs/crates/agent/src/sock/mod.rs +++ b/src/runtime-rs/crates/agent/src/sock/mod.rs @@ -120,12 +120,12 @@ fn parse(address: &str, port: u32) -> Result { let url = Url::parse(address).context("parse url")?; match url.scheme() { VSOCK_SCHEME => { - let cid = url + let vsock_cid = url .host_str() .unwrap_or_default() .parse::() - .context("parse cid")?; - Ok(SockType::Vsock(Vsock::new(cid, port))) + .context("parse vsock cid")?; + Ok(SockType::Vsock(Vsock::new(vsock_cid, port))) } HYBRID_VSOCK_SCHEME => { let path: Vec<&str> = url.path().split(':').collect(); diff --git a/src/runtime-rs/crates/agent/src/sock/vsock.rs b/src/runtime-rs/crates/agent/src/sock/vsock.rs index 7f6a59d89a0d..9b62bb9766d6 100644 --- a/src/runtime-rs/crates/agent/src/sock/vsock.rs +++ b/src/runtime-rs/crates/agent/src/sock/vsock.rs @@ -14,13 +14,13 @@ unsafe impl Sync for Vsock {} #[derive(Debug, PartialEq)] pub struct Vsock { - cid: u32, + vsock_cid: u32, port: u32, } impl Vsock { - pub fn new(cid: u32, port: u32) -> Self { - Self { cid, port } + pub fn new(vsock_cid: u32, port: u32) -> Self { + Self { vsock_cid, port } } } diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml new file mode 100644 index 000000000000..304a7639bc54 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "runtimes" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dependencies] +anyhow = "^1.0" +lazy_static = "1.4.0" +slog = "2.5.2" +slog-scope = "4.4.0" +tokio = { version = "1.8.0", features = ["rt-multi-thread"] } + +common = { path = "./common" } +kata-types = { path = "../../../libs/kata-types" } +logging = { path = "../../../libs/logging"} +oci = { path = "../../../libs/oci" } + +# runtime handler +linux_container = { path = "./linux_container", optional = true } +virt_container = { path = "./virt_container", optional = true } +wasm_container = { path = "./wasm_container", optional = true } + +[features] +default = ["virt"] +linux = ["linux_container"] +virt = ["virt_container"] +wasm = ["wasm_container"] diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml new file mode 100644 index 000000000000..f2bff39c847d --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "common" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +containerd-shim-protos = { version = "0.2.0", features = ["async"]} +lazy_static = "1.4.0" +nix = "0.23.1" +protobuf = "2.23.0" +serde_json = "1.0.39" +slog = "2.5.2" +slog-scope = "4.4.0" +strum = { version = "0.24.0", features = ["derive"] } +thiserror = "^1.0" +tokio = { version = "1.8.0", features = ["rt-multi-thread", "process", "fs"] } +ttrpc = { version = "0.6.0" } + +kata-sys-util = { path = "../../../../libs/kata-sys-util" } +kata-types = { path = "../../../../libs/kata-types" } +oci = { path = "../../../../libs/oci" } diff --git a/src/runtime-rs/crates/runtimes/common/src/container_manager.rs b/src/runtime-rs/crates/runtimes/common/src/container_manager.rs new file mode 100644 index 000000000000..aeba770a660b --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/container_manager.rs @@ -0,0 +1,40 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; +use async_trait::async_trait; + +use crate::types::{ + ContainerConfig, ContainerID, ContainerProcess, ExecProcessRequest, KillRequest, + ProcessExitStatus, ProcessStateInfo, ResizePTYRequest, ShutdownRequest, StatsInfo, + UpdateRequest, PID, +}; + +#[async_trait] +pub trait ContainerManager: Send + Sync { + // container lifecycle + async fn create_container(&self, config: ContainerConfig) -> Result; + async fn pause_container(&self, container_id: &ContainerID) -> Result<()>; + async fn resume_container(&self, container_id: &ContainerID) -> Result<()>; + async fn stats_container(&self, container_id: &ContainerID) -> Result; + async fn update_container(&self, req: UpdateRequest) -> Result<()>; + async fn connect_container(&self, container_id: &ContainerID) -> Result; + + // process lifecycle + async fn close_process_io(&self, process_id: &ContainerProcess) -> Result<()>; + async fn delete_process(&self, process_id: &ContainerProcess) -> Result; + async fn exec_process(&self, req: ExecProcessRequest) -> Result<()>; + async fn kill_process(&self, req: &KillRequest) -> Result<()>; + async fn resize_process_pty(&self, req: &ResizePTYRequest) -> Result<()>; + async fn start_process(&self, process_id: &ContainerProcess) -> Result; + async fn state_process(&self, process_id: &ContainerProcess) -> Result; + async fn wait_process(&self, process_id: &ContainerProcess) -> Result; + + // utility + async fn pid(&self) -> Result; + async fn need_shutdown_sandbox(&self, req: &ShutdownRequest) -> bool; + async fn is_sandbox_container(&self, process_id: &ContainerProcess) -> bool; +} diff --git a/src/runtime-rs/crates/runtimes/common/src/error.rs b/src/runtime-rs/crates/runtimes/common/src/error.rs new file mode 100644 index 000000000000..2ec03c4c6cac --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/error.rs @@ -0,0 +1,17 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use crate::types::{ContainerProcess, Response}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("failed to find container {0}")] + ContainerNotFound(String), + #[error("failed to find process {0}")] + ProcessNotFound(ContainerProcess), + #[error("unexpected response {0} to shim {1}")] + UnexpectedResponse(Response, String), +} diff --git a/src/runtime-rs/crates/runtimes/common/src/lib.rs b/src/runtime-rs/crates/runtimes/common/src/lib.rs new file mode 100644 index 000000000000..36977964ad9b --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/lib.rs @@ -0,0 +1,15 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod container_manager; +pub use container_manager::ContainerManager; +pub mod error; +pub mod message; +mod runtime_handler; +pub use runtime_handler::{RuntimeHandler, RuntimeInstance}; +mod sandbox; +pub use sandbox::Sandbox; +pub mod types; diff --git a/src/runtime-rs/crates/runtimes/common/src/message.rs b/src/runtime-rs/crates/runtimes/common/src/message.rs new file mode 100644 index 000000000000..ff6ee960d71b --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/message.rs @@ -0,0 +1,44 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; +use tokio::sync::mpsc::{channel, Receiver, Sender}; + +/// message receiver buffer size +const MESSAGE_RECEIVER_BUFFER_SIZE: usize = 1; + +#[derive(Debug)] +pub enum Action { + Start, + Stop, + Shutdown, +} + +#[derive(Debug)] +pub struct Message { + pub action: Action, + pub resp_sender: Option>>, +} + +impl Message { + pub fn new(action: Action) -> Self { + Message { + action, + resp_sender: None, + } + } + + pub fn new_with_receiver(action: Action) -> (Receiver>, Self) { + let (resp_sender, receiver) = channel(MESSAGE_RECEIVER_BUFFER_SIZE); + ( + receiver, + Message { + action, + resp_sender: Some(resp_sender), + }, + ) + } +} diff --git a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs new file mode 100644 index 000000000000..d74b83b1d096 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs @@ -0,0 +1,38 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use tokio::sync::mpsc::Sender; + +use crate::{message::Message, ContainerManager, Sandbox}; + +#[derive(Clone)] +pub struct RuntimeInstance { + pub sandbox: Arc, + pub container_manager: Arc, +} + +#[async_trait] +pub trait RuntimeHandler: Send + Sync { + fn init() -> Result<()> + where + Self: Sized; + + fn name() -> String + where + Self: Sized; + + fn new_handler() -> Arc + where + Self: Sized; + + async fn new_instance(&self, sid: &str, msg_sender: Sender) + -> Result; + + fn cleanup(&self, id: &str) -> Result<()>; +} diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs new file mode 100644 index 000000000000..699fc1977a59 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; +use async_trait::async_trait; + +#[async_trait] +pub trait Sandbox: Send + Sync { + async fn start(&self) -> Result<()>; + async fn stop(&self) -> Result<()>; + async fn cleanup(&self, container_id: &str) -> Result<()>; + async fn shutdown(&self) -> Result<()>; +} diff --git a/src/runtime-rs/crates/runtimes/common/src/types/mod.rs b/src/runtime-rs/crates/runtimes/common/src/types/mod.rs new file mode 100644 index 000000000000..e398735ff706 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/types/mod.rs @@ -0,0 +1,219 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod trans_from_shim; +mod trans_into_shim; + +use std::fmt; + +use anyhow::{Context, Result}; +use kata_sys_util::validate; +use kata_types::mount::Mount; +use strum::Display; + +/// Request: request from shim +/// Request and Response messages need to be paired +#[derive(Debug, Clone, Display)] +pub enum Request { + CreateContainer(ContainerConfig), + CloseProcessIO(ContainerProcess), + DeleteProcess(ContainerProcess), + ExecProcess(ExecProcessRequest), + KillProcess(KillRequest), + WaitProcess(ContainerProcess), + StartProcess(ContainerProcess), + StateProcess(ContainerProcess), + ShutdownContainer(ShutdownRequest), + PauseContainer(ContainerID), + ResumeContainer(ContainerID), + ResizeProcessPTY(ResizePTYRequest), + StatsContainer(ContainerID), + UpdateContainer(UpdateRequest), + Pid, + ConnectContainer(ContainerID), +} + +/// Response: response to shim +/// Request and Response messages need to be paired +#[derive(Debug, Clone, Display)] +pub enum Response { + CreateContainer(PID), + CloseProcessIO, + DeleteProcess(ProcessStateInfo), + ExecProcess, + KillProcess, + WaitProcess(ProcessExitStatus), + StartProcess(PID), + StateProcess(ProcessStateInfo), + ShutdownContainer, + PauseContainer, + ResumeContainer, + ResizeProcessPTY, + StatsContainer(StatsInfo), + UpdateContainer, + Pid(PID), + ConnectContainer(PID), +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ProcessType { + Container, + Exec, +} + +#[derive(Clone, Debug)] +pub struct ContainerID { + pub container_id: String, +} + +impl ContainerID { + pub fn new(container_id: &str) -> Result { + validate::verify_id(container_id).context("verify container id")?; + Ok(Self { + container_id: container_id.to_string(), + }) + } +} + +#[derive(Clone, Debug)] +pub struct ContainerProcess { + pub container_id: ContainerID, + pub exec_id: String, + pub process_type: ProcessType, +} + +impl fmt::Display for ContainerProcess { + fn fmt(&self, f: &mut std::fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", &self) + } +} + +impl ContainerProcess { + pub fn new(container_id: &str, exec_id: &str) -> Result { + let (exec_id, process_type) = if exec_id.is_empty() || container_id == exec_id { + ("".to_string(), ProcessType::Container) + } else { + validate::verify_id(exec_id).context("verify exec id")?; + (exec_id.to_string(), ProcessType::Exec) + }; + Ok(Self { + container_id: ContainerID::new(container_id)?, + exec_id, + process_type, + }) + } +} +#[derive(Debug, Clone)] +pub struct ContainerConfig { + pub container_id: String, + pub bundle: String, + pub rootfs_mounts: Vec, + pub terminal: bool, + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, +} + +#[derive(Debug, Clone)] +pub struct PID { + pub pid: u32, +} + +impl PID { + pub fn new(pid: u32) -> Self { + Self { pid } + } +} + +#[derive(Debug, Clone)] +pub struct KillRequest { + pub process_id: ContainerProcess, + pub signal: u32, + pub all: bool, +} + +#[derive(Debug, Clone)] +pub struct ShutdownRequest { + pub container_id: String, + pub is_now: bool, +} + +#[derive(Debug, Clone)] +pub struct ResizePTYRequest { + pub process_id: ContainerProcess, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone)] +pub struct ExecProcessRequest { + pub process_id: ContainerProcess, + pub terminal: bool, + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, + pub spec_type_url: String, + pub spec_value: Vec, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum ProcessStatus { + Unknown = 0, + Created = 1, + Running = 2, + Stopped = 3, + Paused = 4, + Pausing = 5, +} + +#[derive(Debug, Clone)] +pub struct ProcessStateInfo { + pub container_id: String, + pub exec_id: String, + pub pid: PID, + pub bundle: String, + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, + pub terminal: bool, + pub status: ProcessStatus, + pub exit_status: i32, + pub exited_at: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct ProcessExitStatus { + pub exit_code: i32, + pub exit_time: Option, +} + +impl ProcessExitStatus { + pub fn new() -> Self { + Self::default() + } + + pub fn update_exit_code(&mut self, exit_code: i32) { + self.exit_code = exit_code; + self.exit_time = Some(std::time::SystemTime::now()); + } +} + +#[derive(Debug, Clone)] +pub struct StatsInfoValue { + pub type_url: String, + pub value: Vec, +} + +#[derive(Debug, Clone)] +pub struct StatsInfo { + pub value: Option, +} + +#[derive(Debug, Clone)] +pub struct UpdateRequest { + pub container_id: String, + pub value: Vec, +} diff --git a/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs b/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs new file mode 100644 index 000000000000..c26bb6828149 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs @@ -0,0 +1,198 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + convert::{From, TryFrom}, + path::PathBuf, +}; + +use anyhow::{Context, Result}; +use containerd_shim_protos::api; +use kata_types::mount::Mount; + +use super::{ + ContainerConfig, ContainerID, ContainerProcess, ExecProcessRequest, KillRequest, Request, + ResizePTYRequest, ShutdownRequest, UpdateRequest, +}; + +fn trans_from_shim_mount(from: api::Mount) -> Mount { + let options = from.options.to_vec(); + let mut read_only = false; + for o in &options { + if o == "ro" { + read_only = true; + break; + } + } + + Mount { + source: from.source.clone(), + destination: PathBuf::from(&from.target), + fs_type: from.field_type, + options, + device_id: None, + host_shared_fs_path: None, + read_only, + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::CreateTaskRequest) -> Result { + Ok(Request::CreateContainer(ContainerConfig { + container_id: from.id.clone(), + bundle: from.bundle.clone(), + rootfs_mounts: from + .rootfs + .to_vec() + .into_iter() + .map(trans_from_shim_mount) + .collect(), + terminal: from.terminal, + stdin: (!from.stdin.is_empty()).then(|| from.stdin.clone()), + stdout: (!from.stdout.is_empty()).then(|| from.stdout.clone()), + stderr: (!from.stderr.is_empty()).then(|| from.stderr.clone()), + })) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::CloseIORequest) -> Result { + Ok(Request::CloseProcessIO( + ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + )) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::DeleteRequest) -> Result { + Ok(Request::DeleteProcess( + ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + )) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::ExecProcessRequest) -> Result { + let spec = from.get_spec(); + Ok(Request::ExecProcess(ExecProcessRequest { + process_id: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + terminal: from.terminal, + stdin: (!from.stdin.is_empty()).then(|| from.stdin.clone()), + stdout: (!from.stdout.is_empty()).then(|| from.stdout.clone()), + stderr: (!from.stderr.is_empty()).then(|| from.stderr.clone()), + spec_type_url: spec.get_type_url().to_string(), + spec_value: spec.get_value().to_vec(), + })) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::KillRequest) -> Result { + Ok(Request::KillProcess(KillRequest { + process_id: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + signal: from.signal, + all: from.all, + })) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::WaitRequest) -> Result { + Ok(Request::WaitProcess( + ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + )) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::StartRequest) -> Result { + Ok(Request::StartProcess( + ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + )) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::StateRequest) -> Result { + Ok(Request::StateProcess( + ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + )) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::ShutdownRequest) -> Result { + Ok(Request::ShutdownContainer(ShutdownRequest { + container_id: from.id.to_string(), + is_now: from.now, + })) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::ResizePtyRequest) -> Result { + Ok(Request::ResizeProcessPTY(ResizePTYRequest { + process_id: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + width: from.width, + height: from.height, + })) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::PauseRequest) -> Result { + Ok(Request::PauseContainer(ContainerID::new(&from.id)?)) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::ResumeRequest) -> Result { + Ok(Request::ResumeContainer(ContainerID::new(&from.id)?)) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::StatsRequest) -> Result { + Ok(Request::StatsContainer(ContainerID::new(&from.id)?)) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::UpdateTaskRequest) -> Result { + Ok(Request::UpdateContainer(UpdateRequest { + container_id: from.id.to_string(), + value: from.get_resources().get_value().to_vec(), + })) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(_from: api::PidsRequest) -> Result { + Ok(Request::Pid) + } +} + +impl TryFrom for Request { + type Error = anyhow::Error; + fn try_from(from: api::ConnectRequest) -> Result { + Ok(Request::ConnectContainer(ContainerID::new(&from.id)?)) + } +} diff --git a/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs b/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs new file mode 100644 index 000000000000..3c3134e8fdce --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/types/trans_into_shim.rs @@ -0,0 +1,242 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + any::type_name, + convert::{Into, TryFrom, TryInto}, + time, +}; + +use anyhow::{anyhow, Result}; +use containerd_shim_protos::api; + +use super::{ProcessExitStatus, ProcessStateInfo, ProcessStatus, Response}; +use crate::error::Error; + +fn system_time_into(time: time::SystemTime) -> ::protobuf::well_known_types::Timestamp { + let mut proto_time = ::protobuf::well_known_types::Timestamp::new(); + proto_time.set_seconds( + time.duration_since(time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .try_into() + .unwrap_or_default(), + ); + proto_time +} + +fn option_system_time_into( + time: Option, +) -> ::protobuf::SingularPtrField<::protobuf::well_known_types::Timestamp> { + match time { + Some(v) => ::protobuf::SingularPtrField::some(system_time_into(v)), + None => ::protobuf::SingularPtrField::none(), + } +} + +impl From for api::WaitResponse { + fn from(from: ProcessExitStatus) -> Self { + Self { + exit_status: from.exit_code as u32, + exited_at: option_system_time_into(from.exit_time), + ..Default::default() + } + } +} + +impl From for api::Status { + fn from(from: ProcessStatus) -> Self { + match from { + ProcessStatus::Unknown => api::Status::UNKNOWN, + ProcessStatus::Created => api::Status::CREATED, + ProcessStatus::Running => api::Status::RUNNING, + ProcessStatus::Stopped => api::Status::STOPPED, + ProcessStatus::Paused => api::Status::PAUSED, + ProcessStatus::Pausing => api::Status::PAUSING, + } + } +} +impl From for api::StateResponse { + fn from(from: ProcessStateInfo) -> Self { + Self { + id: from.container_id.clone(), + bundle: from.bundle.clone(), + pid: from.pid.pid, + status: from.status.into(), + stdin: from.stdin.unwrap_or_default(), + stdout: from.stdout.unwrap_or_default(), + stderr: from.stderr.unwrap_or_default(), + terminal: from.terminal, + exit_status: from.exit_status as u32, + exited_at: option_system_time_into(from.exited_at), + exec_id: from.exec_id, + ..Default::default() + } + } +} + +impl From for api::DeleteResponse { + fn from(from: ProcessStateInfo) -> Self { + Self { + pid: from.pid.pid, + exit_status: from.exit_status as u32, + exited_at: option_system_time_into(from.exited_at), + ..Default::default() + } + } +} + +impl TryFrom for api::CreateTaskResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::CreateContainer(resp) => Ok(Self { + pid: resp.pid, + ..Default::default() + }), + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::DeleteResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::DeleteProcess(resp) => Ok(resp.into()), + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::WaitResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::WaitProcess(resp) => Ok(resp.into()), + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::StartResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::StartProcess(resp) => Ok(api::StartResponse { + pid: resp.pid, + ..Default::default() + }), + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::StateResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::StateProcess(resp) => Ok(resp.into()), + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::StatsResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + let mut any = ::protobuf::well_known_types::Any::new(); + let mut response = api::StatsResponse::new(); + match from { + Response::StatsContainer(resp) => { + if let Some(value) = resp.value { + any.set_type_url(value.type_url); + any.set_value(value.value); + response.set_stats(any); + } + Ok(response) + } + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::PidsResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::Pid(resp) => { + let mut processes: Vec = vec![]; + let mut p_info = api::ProcessInfo::new(); + let mut res = api::PidsResponse::new(); + p_info.set_pid(resp.pid); + processes.push(p_info); + let v = protobuf::RepeatedField::::from_vec(processes); + res.set_processes(v); + Ok(res) + } + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::ConnectResponse { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::ConnectContainer(resp) => { + let mut res = api::ConnectResponse::new(); + res.set_shim_pid(resp.pid); + Ok(res) + } + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} + +impl TryFrom for api::Empty { + type Error = anyhow::Error; + fn try_from(from: Response) -> Result { + match from { + Response::CloseProcessIO => Ok(api::Empty::new()), + Response::ExecProcess => Ok(api::Empty::new()), + Response::KillProcess => Ok(api::Empty::new()), + Response::ShutdownContainer => Ok(api::Empty::new()), + Response::PauseContainer => Ok(api::Empty::new()), + Response::ResumeContainer => Ok(api::Empty::new()), + Response::ResizeProcessPTY => Ok(api::Empty::new()), + Response::UpdateContainer => Ok(api::Empty::new()), + _ => Err(anyhow!(Error::UnexpectedResponse( + from, + type_name::().to_string() + ))), + } + } +} diff --git a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml new file mode 100644 index 000000000000..81d4e3e03a95 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "linux_container" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +tokio = { version = "1.8.0" } + +common = { path = "../common" } diff --git a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs new file mode 100644 index 000000000000..d50de90b17b0 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs @@ -0,0 +1,42 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use common::{message::Message, RuntimeHandler, RuntimeInstance}; +use tokio::sync::mpsc::Sender; + +unsafe impl Send for LinuxContainer {} +unsafe impl Sync for LinuxContainer {} +pub struct LinuxContainer {} + +#[async_trait] +impl RuntimeHandler for LinuxContainer { + fn init() -> Result<()> { + Ok(()) + } + + fn name() -> String { + "linux_container".to_string() + } + + fn new_handler() -> Arc { + Arc::new(LinuxContainer {}) + } + + async fn new_instance( + &self, + _sid: &str, + _msg_sender: Sender, + ) -> Result { + todo!() + } + + fn cleanup(&self, _id: &str) -> Result<()> { + todo!() + } +} diff --git a/src/runtime-rs/crates/runtimes/src/lib.rs b/src/runtime-rs/crates/runtimes/src/lib.rs new file mode 100644 index 000000000000..64c57feeaef8 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/src/lib.rs @@ -0,0 +1,13 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[macro_use] +extern crate slog; + +logging::logger_with_subsystem!(sl, "runtimes"); + +mod manager; +pub use manager::RuntimeHandlerManager; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs new file mode 100644 index 000000000000..f3006f8561b7 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -0,0 +1,251 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use anyhow::{anyhow, Context, Result}; +use common::{ + message::Message, + types::{Request, Response}, + RuntimeHandler, RuntimeInstance, +}; +use kata_types::{annotations::Annotation, config::TomlConfig}; +use tokio::sync::{mpsc::Sender, RwLock}; + +#[cfg(feature = "linux")] +use linux_container::LinuxContainer; +#[cfg(feature = "virt")] +use virt_container::VirtContainer; +#[cfg(feature = "wasm")] +use wasm_container::WasmContainer; + +struct RuntimeHandlerManagerInner { + id: String, + msg_sender: Sender, + runtime_instance: Option, +} + +impl RuntimeHandlerManagerInner { + fn new(id: &str, msg_sender: Sender) -> Result { + Ok(Self { + id: id.to_string(), + msg_sender, + runtime_instance: None, + }) + } + + async fn init_runtime_handler(&mut self, runtime_name: &str) -> Result<()> { + info!(sl!(), "new runtime handler {}", runtime_name); + + let runtime_handler = match runtime_name { + #[cfg(feature = "linux")] + name if name == LinuxContainer::name() => { + LinuxContainer::init().context("init linux container")?; + LinuxContainer::new_handler() + } + #[cfg(feature = "wasm")] + name if name == WasmContainer::name() => { + WasmContainer::init().context("init wasm container")?; + WasmContainer::new_handler() + } + #[cfg(feature = "virt")] + name if name == VirtContainer::name() => { + VirtContainer::init().context("init virt container")?; + VirtContainer::new_handler() + } + _ => return Err(anyhow!("Unsupported runtime: {}", runtime_name)), + }; + let runtime_instance = runtime_handler + .new_instance(&self.id, self.msg_sender.clone()) + .await + .context("new runtime instance")?; + + // start sandbox + runtime_instance + .sandbox + .start() + .await + .context("start sandbox")?; + self.runtime_instance = Some(runtime_instance); + Ok(()) + } + + async fn try_init(&mut self, spec: &oci::Spec) -> Result<()> { + // return if runtime instance has init + if self.runtime_instance.is_some() { + return Ok(()); + } + + let config = load_config(spec).context("load config")?; + self.init_runtime_handler(&config.runtime.name) + .await + .context("init runtime handler")?; + + Ok(()) + } + + fn get_runtime_instance(&self) -> Option { + self.runtime_instance.clone() + } +} + +unsafe impl Send for RuntimeHandlerManager {} +unsafe impl Sync for RuntimeHandlerManager {} +pub struct RuntimeHandlerManager { + inner: Arc>, +} + +impl RuntimeHandlerManager { + pub async fn new(id: &str, msg_sender: Sender) -> Result { + Ok(Self { + inner: Arc::new(RwLock::new(RuntimeHandlerManagerInner::new( + id, msg_sender, + )?)), + }) + } + + pub fn cleanup(_id: &str) -> Result<()> { + // TODO: load runtime from persist and cleanup + Ok(()) + } + + pub async fn handler_message(&self, req: Request) -> Result { + if let Request::CreateContainer(req) = req { + // get oci spec + let bundler_path = format!("{}/{}", req.bundle, oci::OCI_SPEC_CONFIG_FILE_NAME); + let spec = oci::Spec::load(&bundler_path).context("load spec")?; + + let mut inner = self.inner.write().await; + inner + .try_init(&spec) + .await + .context("try init runtime handler")?; + + let instance = inner + .get_runtime_instance() + .ok_or_else(|| anyhow!("runtime not ready"))?; + + let shim_pid = instance + .container_manager + .create_container(req) + .await + .context("create container")?; + Ok(Response::CreateContainer(shim_pid)) + } else { + self.handler_request(req).await.context("handler request") + } + } + + pub async fn handler_request(&self, req: Request) -> Result { + let inner = self.inner.read().await; + let instance = inner + .get_runtime_instance() + .ok_or_else(|| anyhow!("runtime not ready"))?; + let sandbox = instance.sandbox; + let cm = instance.container_manager; + + match req { + Request::CreateContainer(req) => Err(anyhow!("Unreachable request {:?}", req)), + Request::CloseProcessIO(process_id) => { + cm.close_process_io(&process_id).await.context("close io")?; + Ok(Response::CloseProcessIO) + } + Request::DeleteProcess(process_id) => { + let resp = cm.delete_process(&process_id).await.context("do delete")?; + Ok(Response::DeleteProcess(resp)) + } + Request::ExecProcess(req) => { + cm.exec_process(req).await.context("exec")?; + Ok(Response::ExecProcess) + } + Request::KillProcess(req) => { + cm.kill_process(&req).await.context("kill process")?; + Ok(Response::KillProcess) + } + Request::ShutdownContainer(req) => { + if cm.need_shutdown_sandbox(&req).await { + sandbox.shutdown().await.context("do shutdown")?; + } + Ok(Response::ShutdownContainer) + } + Request::WaitProcess(process_id) => { + let exit_status = cm.wait_process(&process_id).await.context("wait process")?; + if cm.is_sandbox_container(&process_id).await { + sandbox.stop().await.context("stop sandbox")?; + } + Ok(Response::WaitProcess(exit_status)) + } + Request::StartProcess(process_id) => { + let shim_pid = cm + .start_process(&process_id) + .await + .context("start process")?; + Ok(Response::StartProcess(shim_pid)) + } + + Request::StateProcess(process_id) => { + let state = cm + .state_process(&process_id) + .await + .context("state process")?; + Ok(Response::StateProcess(state)) + } + Request::PauseContainer(container_id) => { + cm.pause_container(&container_id) + .await + .context("pause container")?; + Ok(Response::PauseContainer) + } + Request::ResumeContainer(container_id) => { + cm.resume_container(&container_id) + .await + .context("resume container")?; + Ok(Response::ResumeContainer) + } + Request::ResizeProcessPTY(req) => { + cm.resize_process_pty(&req).await.context("resize pty")?; + Ok(Response::ResizeProcessPTY) + } + Request::StatsContainer(container_id) => { + let stats = cm + .stats_container(&container_id) + .await + .context("stats container")?; + Ok(Response::StatsContainer(stats)) + } + Request::UpdateContainer(req) => { + cm.update_container(req).await.context("update container")?; + Ok(Response::UpdateContainer) + } + Request::Pid => Ok(Response::Pid(cm.pid().await.context("pid")?)), + Request::ConnectContainer(container_id) => Ok(Response::ConnectContainer( + cm.connect_container(&container_id) + .await + .context("connect")?, + )), + } + } +} + +/// Config override ordering(high to low): +/// 1. podsandbox annotation +/// 2. shimv2 create task option +/// TODO: https://github.com/kata-containers/kata-containers/issues/3961 +/// 3. environment +fn load_config(spec: &oci::Spec) -> Result { + const KATA_CONF_FILE: &str = "KATA_CONF_FILE"; + let annotation = Annotation::new(spec.annotations.clone()); + let config_path = if let Some(path) = annotation.get_sandbox_config_path() { + path + } else if let Ok(path) = std::env::var(KATA_CONF_FILE) { + path + } else { + String::from("") + }; + info!(sl!(), "get config path {:?}", &config_path); + let (toml_config, _) = TomlConfig::load_from_file(&config_path).context("load toml config")?; + Ok(toml_config) +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml new file mode 100644 index 000000000000..8e63c01c1ad1 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "virt_container" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +tokio = { version = "1.8.0" } + +common = { path = "../common" } +kata-types = { path = "../../../../libs/kata-types" } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs new file mode 100644 index 000000000000..8419a9382838 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -0,0 +1,47 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use common::{message::Message, RuntimeHandler, RuntimeInstance}; +use kata_types::config::{hypervisor::register_hypervisor_plugin, DragonballConfig}; +use tokio::sync::mpsc::Sender; + +unsafe impl Send for VirtContainer {} +unsafe impl Sync for VirtContainer {} +pub struct VirtContainer {} + +#[async_trait] +impl RuntimeHandler for VirtContainer { + fn init() -> Result<()> { + // register + let dragonball_config = Arc::new(DragonballConfig::new()); + register_hypervisor_plugin("dragonball", dragonball_config); + Ok(()) + } + + fn name() -> String { + "virt_container".to_string() + } + + fn new_handler() -> Arc { + Arc::new(VirtContainer {}) + } + + async fn new_instance( + &self, + _sid: &str, + _msg_sender: Sender, + ) -> Result { + todo!() + } + + fn cleanup(&self, _id: &str) -> Result<()> { + // TODO + Ok(()) + } +} diff --git a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml new file mode 100644 index 000000000000..9dfce237e258 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "wasm_container" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +tokio = { version = "1.8.0" } + +common = { path = "../common" } diff --git a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs new file mode 100644 index 000000000000..c92cd965a0dc --- /dev/null +++ b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs @@ -0,0 +1,42 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use common::{message::Message, RuntimeHandler, RuntimeInstance}; +use tokio::sync::mpsc::Sender; + +unsafe impl Send for WasmContainer {} +unsafe impl Sync for WasmContainer {} +pub struct WasmContainer {} + +#[async_trait] +impl RuntimeHandler for WasmContainer { + fn init() -> Result<()> { + Ok(()) + } + + fn name() -> String { + "wasm_container".to_string() + } + + fn new_handler() -> Arc { + Arc::new(WasmContainer {}) + } + + async fn new_instance( + &self, + _sid: &str, + _msg_sender: Sender, + ) -> Result { + todo!() + } + + fn cleanup(&self, _id: &str) -> Result<()> { + todo!() + } +} diff --git a/src/runtime-rs/crates/service/Cargo.toml b/src/runtime-rs/crates/service/Cargo.toml new file mode 100644 index 000000000000..b3aa85a64fff --- /dev/null +++ b/src/runtime-rs/crates/service/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "service" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +slog = "2.5.2" +slog-scope = "4.4.0" +tokio = { version = "1.8.0", features = ["rt-multi-thread"] } +ttrpc = { version = "0.6.0" } + +common = { path = "../runtimes/common" } +containerd-shim-protos = { version = "0.2.0", features = ["async"]} +logging = { path = "../../../libs/logging"} +runtimes = { path = "../runtimes" } diff --git a/src/runtime-rs/crates/service/src/lib.rs b/src/runtime-rs/crates/service/src/lib.rs new file mode 100644 index 000000000000..1f28a8009c4b --- /dev/null +++ b/src/runtime-rs/crates/service/src/lib.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[macro_use] +extern crate slog; + +logging::logger_with_subsystem!(sl, "service"); + +mod manager; +pub use manager::ServiceManager; +mod task_service; diff --git a/src/runtime-rs/crates/service/src/manager.rs b/src/runtime-rs/crates/service/src/manager.rs new file mode 100644 index 000000000000..dac1b2275f0f --- /dev/null +++ b/src/runtime-rs/crates/service/src/manager.rs @@ -0,0 +1,107 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + os::unix::io::{FromRawFd, RawFd}, + sync::Arc, +}; + +use anyhow::{Context, Result}; +use common::message::{Action, Message}; +use containerd_shim_protos::shim_async; +use runtimes::RuntimeHandlerManager; +use tokio::sync::mpsc::{channel, Receiver}; +use ttrpc::asynchronous::Server; + +use crate::task_service::TaskService; + +/// message buffer size +const MESSAGE_BUFFER_SIZE: usize = 8; + +pub struct ServiceManager { + receiver: Option>, + handler: Arc, + task_server: Option, +} + +impl ServiceManager { + pub async fn new(id: &str, task_server_fd: RawFd) -> Result { + let (sender, receiver) = channel::(MESSAGE_BUFFER_SIZE); + let handler = Arc::new( + RuntimeHandlerManager::new(id, sender) + .await + .context("new runtime handler")?, + ); + let mut task_server = unsafe { Server::from_raw_fd(task_server_fd) }; + task_server = task_server.set_domain_unix(); + Ok(Self { + receiver: Some(receiver), + handler, + task_server: Some(task_server), + }) + } + + pub async fn run(&mut self) -> Result<()> { + info!(sl!(), "begin to run service"); + + self.start().await.context("start")?; + let mut rx = self.receiver.take(); + if let Some(rx) = rx.as_mut() { + while let Some(r) = rx.recv().await { + info!(sl!(), "receive action {:?}", &r.action); + let result = match r.action { + Action::Start => self.start().await.context("start listen"), + Action::Stop => self.stop_listen().await.context("stop listen"), + Action::Shutdown => { + self.stop_listen().await.context("stop listen")?; + break; + } + }; + + if let Some(ref sender) = r.resp_sender { + sender.send(result).await.context("send response")?; + } + } + } + + info!(sl!(), "end to run service"); + + Ok(()) + } + + pub fn cleanup(id: &str) -> Result<()> { + RuntimeHandlerManager::cleanup(id) + } + + async fn start(&mut self) -> Result<()> { + let task_service = Arc::new(Box::new(TaskService::new(self.handler.clone())) + as Box); + let task_server = self.task_server.take(); + let task_server = match task_server { + Some(t) => { + let mut t = t.register_service(shim_async::create_task(task_service)); + t.start().await.context("task server start")?; + Some(t) + } + None => None, + }; + self.task_server = task_server; + Ok(()) + } + + async fn stop_listen(&mut self) -> Result<()> { + let task_server = self.task_server.take(); + let task_server = match task_server { + Some(mut t) => { + t.stop_listen().await; + Some(t) + } + None => None, + }; + self.task_server = task_server; + Ok(()) + } +} diff --git a/src/runtime-rs/crates/service/src/task_service.rs b/src/runtime-rs/crates/service/src/task_service.rs new file mode 100644 index 000000000000..77c368d6c9bc --- /dev/null +++ b/src/runtime-rs/crates/service/src/task_service.rs @@ -0,0 +1,82 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + convert::{TryFrom, TryInto}, + sync::Arc, +}; + +use async_trait::async_trait; +use common::types::{Request, Response}; +use containerd_shim_protos::{api, shim_async}; +use ttrpc::{self, r#async::TtrpcContext}; + +use runtimes::RuntimeHandlerManager; + +pub(crate) struct TaskService { + handler: Arc, +} + +impl TaskService { + pub(crate) fn new(handler: Arc) -> Self { + Self { handler } + } +} + +async fn handler_message( + s: &RuntimeHandlerManager, + ctx: &TtrpcContext, + req: TtrpcReq, +) -> ttrpc::Result +where + Request: TryFrom, + >::Error: std::fmt::Debug, + TtrpcResp: TryFrom, + >::Error: std::fmt::Debug, +{ + let r = req + .try_into() + .map_err(|err| ttrpc::Error::Others(format!("failed to translate from shim {:?}", err)))?; + let logger = sl!().new(o!("steam id" => ctx.mh.stream_id)); + debug!(logger, "====> task service {:?}", &r); + let resp = s + .handler_message(r) + .await + .map_err(|err| ttrpc::Error::Others(format!("failed to handler message {:?}", err)))?; + debug!(logger, "<==== task service {:?}", &resp); + Ok(resp + .try_into() + .map_err(|err| ttrpc::Error::Others(format!("failed to translate to shim {:?}", err)))?) +} + +macro_rules! impl_service { + ($($name: tt | $req: ty | $resp: ty),*) => { + #[async_trait] + impl shim_async::Task for TaskService { + $(async fn $name(&self, ctx: &TtrpcContext, req: $req) -> ttrpc::Result<$resp> { + handler_message(&self.handler, ctx, req).await + })* + } + }; +} + +impl_service!( + state | api::StateRequest | api::StateResponse, + create | api::CreateTaskRequest | api::CreateTaskResponse, + start | api::StartRequest | api::StartResponse, + delete | api::DeleteRequest | api::DeleteResponse, + pids | api::PidsRequest | api::PidsResponse, + pause | api::PauseRequest | api::Empty, + resume | api::ResumeRequest | api::Empty, + kill | api::KillRequest | api::Empty, + exec | api::ExecProcessRequest | api::Empty, + resize_pty | api::ResizePtyRequest | api::Empty, + update | api::UpdateTaskRequest | api::Empty, + wait | api::WaitRequest | api::WaitResponse, + stats | api::StatsRequest | api::StatsResponse, + connect | api::ConnectRequest | api::ConnectResponse, + shutdown | api::ShutdownRequest | api::Empty +); diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml index e2c07e4e2878..67fac11c13fa 100644 --- a/src/runtime-rs/crates/shim/Cargo.toml +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -19,7 +19,7 @@ containerd-shim-protos = { version = "0.2.0", features = ["async"]} go-flag = "0.1.0" libc = "0.2.108" log = "0.4.14" -nix = "0.16.0" +nix = "0.23.1" protobuf = "2.23.0" sha2 = "=0.9.3" slog = {version = "2.7.0", features = ["std", "release_max_level_trace", "max_level_trace"]} @@ -34,11 +34,13 @@ kata-types = { path = "../../../libs/kata-types"} kata-sys-util = { path = "../../../libs/kata-sys-util"} logging = { path = "../../../libs/logging"} oci = { path = "../../../libs/oci" } +service = { path = "../service" } [build-dependencies] vergen = { version = "6", default-features = false, features = ["build", "git", "rustc"] } [dev-dependencies] tempfile = "3.2.0" +rand = "0.8.4" serial_test = "0.5.1" tests_utils = { path = "../../tests/utils"} diff --git a/src/runtime-rs/crates/shim/src/args.rs b/src/runtime-rs/crates/shim/src/args.rs index 62b9653cca32..0db33cbf81f9 100644 --- a/src/runtime-rs/crates/shim/src/args.rs +++ b/src/runtime-rs/crates/shim/src/args.rs @@ -51,8 +51,8 @@ impl Args { )))); } - validate::verify_cid(&self.id).context("verify cid")?; - validate::verify_cid(&self.namespace).context("verify namespace")?; + validate::verify_id(&self.id).context("verify container id")?; + validate::verify_id(&self.namespace).context("verify namespace")?; // Ensure `address` is a valid path. let path = PathBuf::from(self.address.clone()) diff --git a/src/runtime-rs/crates/shim/src/lib.rs b/src/runtime-rs/crates/shim/src/lib.rs index 1130cb7e4597..000c5620a2ed 100644 --- a/src/runtime-rs/crates/shim/src/lib.rs +++ b/src/runtime-rs/crates/shim/src/lib.rs @@ -7,11 +7,7 @@ #[macro_use] extern crate slog; -macro_rules! sl { - () => { - slog_scope::logger().new(slog::o!("subsystem" => "shim")) - }; -} +logging::logger_with_subsystem!(sl, "shim"); mod args; pub use args::Args; diff --git a/src/runtime-rs/crates/shim/src/shim.rs b/src/runtime-rs/crates/shim/src/shim.rs index 164e6c6f60d3..298f6a3e00b3 100644 --- a/src/runtime-rs/crates/shim/src/shim.rs +++ b/src/runtime-rs/crates/shim/src/shim.rs @@ -31,7 +31,7 @@ impl ShimExecutor { } pub(crate) fn load_oci_spec(&self, path: &Path) -> Result { - let spec_file = path.join("config.json"); + let spec_file = path.join(oci::OCI_SPEC_CONFIG_FILE_NAME); oci::Spec::load(spec_file.to_str().unwrap_or_default()).context("load spec") } diff --git a/src/runtime-rs/crates/shim/src/shim_delete.rs b/src/runtime-rs/crates/shim/src/shim_delete.rs index 4d9890d13822..57082d212925 100644 --- a/src/runtime-rs/crates/shim/src/shim_delete.rs +++ b/src/runtime-rs/crates/shim/src/shim_delete.rs @@ -5,7 +5,7 @@ // use anyhow::{Context, Result}; -use containerd_shim_protos::shim::shim::DeleteResponse; +use containerd_shim_protos::api; use protobuf::Message; use crate::{shim::ShimExecutor, Error}; @@ -19,8 +19,8 @@ impl ShimExecutor { Ok(()) } - fn do_cleanup(&self) -> Result { - let mut rsp = DeleteResponse::new(); + fn do_cleanup(&self) -> Result { + let mut rsp = api::DeleteResponse::new(); rsp.set_exit_status(128 + libc::SIGKILL as u32); let mut exited_time = protobuf::well_known_types::Timestamp::new(); let seconds = std::time::SystemTime::now() @@ -30,42 +30,7 @@ impl ShimExecutor { exited_time.set_seconds(seconds); rsp.set_exited_at(exited_time); - // TODO: implement cleanup + service::ServiceManager::cleanup(&self.args.id).context("cleanup")?; Ok(rsp) } } - -#[cfg(test)] -mod tests { - use serial_test::serial; - use tests_utils::gen_id; - - use super::*; - use crate::Args; - - #[test] - #[serial] - fn test_shim_delete() { - let dir = tempfile::tempdir().unwrap(); - let bundle_path = dir.path(); - std::env::set_current_dir(bundle_path).unwrap(); - - let id = gen_id(16); - let namespace = gen_id(16); - let args = Args { - id, - namespace, - address: "containerd_socket".into(), - publish_binary: "containerd".into(), - socket: "socket".into(), - bundle: bundle_path.to_str().unwrap().into(), - debug: false, - }; - - let executor = ShimExecutor::new(args); - - let resp = executor.do_cleanup().unwrap(); - assert_eq!(resp.exit_status, 128 + libc::SIGKILL as u32); - assert!(resp.exited_at.as_ref().unwrap().seconds > 0); - } -} diff --git a/src/runtime-rs/crates/shim/src/shim_run.rs b/src/runtime-rs/crates/shim/src/shim_run.rs index 76058964bd8d..112b83517966 100644 --- a/src/runtime-rs/crates/shim/src/shim_run.rs +++ b/src/runtime-rs/crates/shim/src/shim_run.rs @@ -38,8 +38,11 @@ impl ShimExecutor { info!(sl!(), "start to run"); self.args.validate(false).context("validata")?; - let _server_fd = get_server_fd().context("get server fd")?; - // TODO: implement run + let server_fd = get_server_fd().context("get server fd")?; + let mut service_manager = service::ServiceManager::new(&self.args.id, server_fd) + .await + .context("new runtime server")?; + service_manager.run().await.context("run")?; Ok(()) } From 75e282b4c19e49ad7d8ef246a1992e20816f8989 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Wed, 30 Mar 2022 10:13:17 +0800 Subject: [PATCH 0043/1953] runtime-rs: hypervisor base define Responsible for VM manager, such as Qemu, Dragonball Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/runtime-rs/Cargo.lock | 15 ++ src/runtime-rs/Cargo.toml | 2 + src/runtime-rs/crates/hypervisor/Cargo.toml | 21 +++ .../crates/hypervisor/src/device/block.rs | 24 +++ .../crates/hypervisor/src/device/mod.rs | 36 +++++ .../crates/hypervisor/src/device/network.rs | 32 ++++ .../hypervisor/src/device/share_fs_device.rs | 27 ++++ .../hypervisor/src/device/share_fs_mount.rs | 43 ++++++ .../crates/hypervisor/src/device/vfio.rs | 145 ++++++++++++++++++ .../crates/hypervisor/src/device/vsock.rs | 17 ++ src/runtime-rs/crates/hypervisor/src/lib.rs | 49 ++++++ 11 files changed, 411 insertions(+) create mode 100644 src/runtime-rs/crates/hypervisor/Cargo.toml create mode 100644 src/runtime-rs/crates/hypervisor/src/device/block.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/device/mod.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/device/network.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/device/vfio.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/device/vsock.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/lib.rs diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 4dda25cd7c33..f56b90609a87 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -513,6 +513,21 @@ dependencies = [ "libc", ] +[[package]] +name = "hypervisor" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "kata-types", + "libc", + "logging", + "serde_json", + "slog", + "slog-scope", + "thiserror", +] + [[package]] name = "idna" version = "0.2.3" diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index b359f9d1fb2f..176bb222adea 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -3,4 +3,6 @@ members = [ "crates/shim", # TODO: current only for check, delete after use the agent crate "crates/agent", + # TODO: current only for check, delete after use the hypervisor crate + "crates/hypervisor", ] diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml new file mode 100644 index 000000000000..4520409d2a2e --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "hypervisor" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +libc = ">=0.2.39" +serde_json = ">=1.0.9" +slog = "2.5.2" +slog-scope = "4.4.0" +thiserror = "1.0" + +kata-types = { path = "../../../libs/kata-types" } +logging = { path = "../../../libs/logging" } + +[features] diff --git a/src/runtime-rs/crates/hypervisor/src/device/block.rs b/src/runtime-rs/crates/hypervisor/src/device/block.rs new file mode 100644 index 000000000000..4f59cc0ea394 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/block.rs @@ -0,0 +1,24 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[derive(Debug)] +pub struct BlockConfig { + /// Unique identifier of the drive. + pub id: String, + + /// Path of the drive. + pub path_on_host: String, + + /// If set to true, the drive is opened in read-only mode. Otherwise, the + /// drive is opened as read-write. + pub is_readonly: bool, + + /// Don't close `path_on_host` file when dropping the device. + pub no_drop: bool, + + /// device index + pub index: u64, +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/mod.rs new file mode 100644 index 000000000000..49215e0d1a03 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/mod.rs @@ -0,0 +1,36 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod block; +pub use block::BlockConfig; +mod network; +pub use network::{Address, NetworkConfig}; +mod share_fs_device; +pub use share_fs_device::ShareFsDeviceConfig; +mod vfio; +pub use vfio::{bind_device_to_host, bind_device_to_vfio, VfioBusMode, VfioConfig}; +mod share_fs_mount; +pub use share_fs_mount::{ShareFsMountConfig, ShareFsMountType, ShareFsOperation}; +mod vsock; +pub use vsock::VsockConfig; + +use std::fmt; + +#[derive(Debug)] +pub enum Device { + Block(BlockConfig), + Network(NetworkConfig), + ShareFsDevice(ShareFsDeviceConfig), + Vfio(VfioConfig), + ShareFsMount(ShareFsMountConfig), + Vsock(VsockConfig), +} + +impl fmt::Display for Device { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/network.rs b/src/runtime-rs/crates/hypervisor/src/device/network.rs new file mode 100644 index 000000000000..6c13a9ca1e3f --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/network.rs @@ -0,0 +1,32 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fmt; + +pub struct Address(pub [u8; 6]); + +impl fmt::Debug for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let b = self.0; + write!( + f, + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + b[0], b[1], b[2], b[3], b[4], b[5] + ) + } +} + +#[derive(Debug)] +pub struct NetworkConfig { + /// Unique identifier of the device + pub id: String, + + /// Host level path for the guest network interface. + pub host_dev_name: String, + + /// Guest MAC address. + pub guest_mac: Option
, +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs b/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs new file mode 100644 index 000000000000..4bf73eab73fd --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/share_fs_device.rs @@ -0,0 +1,27 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +/// ShareFsDeviceConfig: share fs device config +#[derive(Debug)] +pub struct ShareFsDeviceConfig { + /// fs_type: virtiofs or inline-virtiofs + pub fs_type: String, + + /// socket_path: socket path for virtiofs + pub sock_path: String, + + /// mount_tag: a label used as a hint to the guest. + pub mount_tag: String, + + /// host_path: the host filesystem path for this volume. + pub host_path: String, + + /// queue_size: queue size + pub queue_size: u64, + + /// queue_num: queue number + pub queue_num: u64, +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs b/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs new file mode 100644 index 000000000000..85f516456243 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/share_fs_mount.rs @@ -0,0 +1,43 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[derive(Copy, Clone, Debug)] +pub enum ShareFsOperation { + Mount, + Umount, + Update, +} + +#[derive(Debug)] +pub enum ShareFsMountType { + PASSTHROUGH, + RAFS, +} + +/// ShareFsMountConfig: share fs mount config +#[derive(Debug)] +pub struct ShareFsMountConfig { + /// source: the passthrough fs exported dir or rafs meta file of rafs + pub source: String, + + /// fstype: specifies the type of this sub-fs, could be passthrough-fs or rafs + pub fstype: ShareFsMountType, + + /// mount_point: the mount point inside guest + pub mount_point: String, + + /// config: the rafs backend config file + pub config: Option, + + /// tag: is the tag used inside the kata guest. + pub tag: String, + + /// op: the operation to take, e.g. mount, umount or update + pub op: ShareFsOperation, + + /// prefetch_list_path: path to file that contains file lists that should be prefetched by rafs + pub prefetch_list_path: Option, +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/vfio.rs new file mode 100644 index 000000000000..db1a99fb3fad --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/vfio.rs @@ -0,0 +1,145 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{fs, path::Path, process::Command}; + +use anyhow::{anyhow, Context, Result}; + +fn override_driver(bdf: &str, driver: &str) -> Result<()> { + let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf); + fs::write(&driver_override, driver) + .context(format!("echo {} > {}", driver, &driver_override))?; + info!(sl!(), "echo {} > {}", driver, driver_override); + Ok(()) +} + +const SYS_PCI_DEVICES_PATH: &str = "/sys/bus/pci/devices"; +const PCI_DRIVER_PROBE: &str = "/sys/bus/pci/drivers_probe"; +const VFIO_NEW_ID_PATH: &str = "/sys/bus/pci/drivers/vfio-pci/new_id"; + +pub const VFIO_PCI: &str = "vfio-pci"; + +#[derive(Debug)] +pub enum VfioBusMode { + PCI, + MMIO, +} + +impl VfioBusMode { + pub fn new(mode: &str) -> Result { + Ok(match mode { + "mmio" => VfioBusMode::MMIO, + _ => VfioBusMode::PCI, + }) + } +} + +#[derive(Debug)] +pub struct VfioConfig { + /// Unique identifier of the device + pub id: String, + + /// Sysfs path for mdev bus type device + pub sysfs_path: String, + + /// PCI device information: "bus:slot:function" + pub bus_slot_func: String, + + /// Bus Mode, PCI or MMIO + pub mode: VfioBusMode, +} + +/// binds the device to vfio driver after unbinding from host. +/// Will be called by a network interface or a generic pcie device. +pub fn bind_device_to_vfio(bdf: &str, host_driver: &str, _vendor_device_id: &str) -> Result<()> { + // modprobe vfio-pci + if !Path::new(VFIO_NEW_ID_PATH).exists() { + Command::new("modprobe") + .arg(VFIO_PCI) + .output() + .expect("Failed to run modprobe vfio-pci"); + } + + // Arm does not need cmdline to open iommu, just set it through bios. + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + // check intel_iommu=on + let cmdline = fs::read_to_string("/proc/cmdline").unwrap(); + if cmdline.contains("iommu=off") || !cmdline.contains("iommu=") { + return Err(anyhow!("iommu isn't set on kernel cmdline")); + } + } + + // if it's already bound to vfio + if is_equal_driver(bdf, VFIO_PCI) { + info!(sl!(), "bdf : {} was already bound to vfio-pci", bdf); + return Ok(()); + } + + info!(sl!(), "host driver : {}", host_driver); + override_driver(bdf, VFIO_PCI).context("override driver")?; + + let unbind_path = format!("/sys/bus/pci/devices/{}/driver/unbind", bdf); + + // echo bdf > /sys/bus/pci/drivers/virtio-pci/unbind" + fs::write(&unbind_path, bdf) + .with_context(|| format!("Failed to echo {} > {}", bdf, &unbind_path))?; + + info!(sl!(), "{} is unbound from {}", bdf, host_driver); + + // echo bdf > /sys/bus/pci/drivers_probe + fs::write(PCI_DRIVER_PROBE, bdf) + .with_context(|| format!("Failed to echo {} > {}", bdf, PCI_DRIVER_PROBE))?; + + info!(sl!(), "echo {} > /sys/bus/pci/drivers_probe", bdf); + Ok(()) +} + +pub fn is_equal_driver(bdf: &str, host_driver: &str) -> bool { + let sys_pci_devices_path = Path::new(SYS_PCI_DEVICES_PATH); + let driver_file = sys_pci_devices_path.join(bdf).join("driver"); + + if driver_file.exists() { + let driver_path = fs::read_link(driver_file).unwrap_or_default(); + let driver_name = driver_path + .file_name() + .map_or(String::new(), |v| v.to_str().unwrap().to_owned()); + return driver_name.eq(host_driver); + } + + false +} + +/// bind_device_to_host binds the device to the host driver after unbinding from vfio-pci. +pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str) -> Result<()> { + // Unbind from vfio-pci driver to the original host driver + + info!(sl!(), "bind {} to {}", bdf, host_driver); + + // if it's already bound to host_driver + if is_equal_driver(bdf, host_driver) { + info!( + sl!(), + "bdf {} was already unbound to host driver {}", bdf, host_driver + ); + return Ok(()); + } + + override_driver(bdf, host_driver).context("override driver")?; + + let unbind_path = "/sys/bus/pci/drivers/vfio-pci/unbind"; + + // echo bdf > /sys/bus/pci/drivers/vfio-pci/unbind" + std::fs::write(unbind_path, bdf).with_context(|| format!("echo {}> {}", bdf, unbind_path))?; + info!(sl!(), "echo {} > {}", bdf, unbind_path); + + // echo bdf > /sys/bus/pci/drivers_probe + std::fs::write(PCI_DRIVER_PROBE, bdf) + .context(format!("echo {} > {}", bdf, PCI_DRIVER_PROBE))?; + info!(sl!(), "echo {} > {}", bdf, PCI_DRIVER_PROBE); + + Ok(()) +} diff --git a/src/runtime-rs/crates/hypervisor/src/device/vsock.rs b/src/runtime-rs/crates/hypervisor/src/device/vsock.rs new file mode 100644 index 000000000000..3a5b7c8b3cc7 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/device/vsock.rs @@ -0,0 +1,17 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[derive(Debug)] +pub struct VsockConfig { + /// Unique identifier of the device + pub id: String, + + /// A 32-bit Context Identifier (CID) used to identify the guest. + pub guest_cid: u32, + + /// unix domain socket path + pub uds_path: String, +} diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs new file mode 100644 index 000000000000..de8c172375b8 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -0,0 +1,49 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[macro_use] +extern crate slog; + +logging::logger_with_subsystem!(sl, "hypervisor"); + +pub mod device; + +use std::collections::HashMap; + +use anyhow::Result; +use async_trait::async_trait; +use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; + +#[derive(Debug)] +pub struct VcpuThreadIds { + /// List of tids of vcpu threads (vcpu index, tid) + pub vcpus: HashMap, +} + +#[async_trait] +pub trait Hypervisor: Send + Sync { + // vm manager + async fn prepare_vm(&self, id: &str, netns: Option) -> Result<()>; + async fn start_vm(&self, timeout: i32) -> Result<()>; + async fn stop_vm(&self) -> Result<()>; + async fn pause_vm(&self) -> Result<()>; + async fn save_vm(&self) -> Result<()>; + async fn resume_vm(&self) -> Result<()>; + + // device manager + async fn add_device(&self, device: device::Device) -> Result<()>; + async fn remove_device(&self, device: device::Device) -> Result<()>; + + // utils + async fn get_agent_socket(&self) -> Result; + async fn disconnect(&self); + async fn hypervisor_config(&self) -> HypervisorConfig; + async fn get_thread_ids(&self) -> Result; + async fn get_pids(&self) -> Result>; + async fn cleanup(&self) -> Result<()>; + async fn check(&self) -> Result<()>; + async fn get_jailer_root(&self) -> Result; +} From 234d7bca04ea4d02b9151f559789d2834ab28d31 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Wed, 30 Mar 2022 10:52:47 +0800 Subject: [PATCH 0044/1953] runtime-rs: support cgroup resource Fixes: #3785 Signed-off-by: Tim Zhang --- src/libs/kata-sys-util/src/cgroup.rs | 735 ------------------ src/libs/kata-sys-util/src/lib.rs | 2 +- src/libs/kata-sys-util/src/spec.rs | 94 +++ src/runtime-rs/Cargo.lock | 55 +- src/runtime-rs/Cargo.toml | 4 +- src/runtime-rs/crates/resource/Cargo.toml | 24 + .../crates/resource/src/cgroups/mod.rs | 217 ++++++ .../crates/resource/src/cgroups/utils.rs | 16 + src/runtime-rs/crates/resource/src/lib.rs | 7 + 9 files changed, 408 insertions(+), 746 deletions(-) delete mode 100644 src/libs/kata-sys-util/src/cgroup.rs create mode 100644 src/libs/kata-sys-util/src/spec.rs create mode 100644 src/runtime-rs/crates/resource/Cargo.toml create mode 100644 src/runtime-rs/crates/resource/src/cgroups/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/cgroups/utils.rs create mode 100644 src/runtime-rs/crates/resource/src/lib.rs diff --git a/src/libs/kata-sys-util/src/cgroup.rs b/src/libs/kata-sys-util/src/cgroup.rs deleted file mode 100644 index 3edcf98245ab..000000000000 --- a/src/libs/kata-sys-util/src/cgroup.rs +++ /dev/null @@ -1,735 +0,0 @@ -// Copyright (c) 2019-2021 Alibaba Cloud -// Copyright (c) 2019-2021 Ant Group -// -// SPDX-License-Identifier: Apache-2.0 -// - -use std::ops::Deref; -use std::path::{Component, Path, PathBuf}; -use std::sync::Mutex; - -use cgroups::{Cgroup, CgroupPid, Controllers, Hierarchy, Subsystem}; -use lazy_static::lazy_static; -use once_cell::sync::Lazy; - -use crate::sl; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Can not add tgid {0} to cgroup, {1:?}")] - AddTgid(u64, #[source] cgroups::error::Error), - #[error("failed to apply resources to cgroup: {0:?}")] - ApplyResource(#[source] cgroups::error::Error), - #[error("failed to delete cgroup after {0} retries")] - DeleteCgroup(u64), - #[error("Invalid cgroup path {0}")] - InvalidCgroupPath(String), -} - -pub type Result = std::result::Result; - -lazy_static! { - /// Disable cgroup v1 subsystems. - pub static ref DISABLED_HIERARCHIES: Mutex> = Mutex::new(Vec::new()); -} - -/// Update the disabled cgroup subsystems. -/// -/// Some cgroup controllers may be disabled by runtime configuration file. The sandbox may call -/// this method to disable those cgroup controllers once. -pub fn update_disabled_cgroup_list(hierarchies: &[String]) { - let mut disabled_hierarchies = DISABLED_HIERARCHIES.lock().unwrap(); - disabled_hierarchies.clear(); - for hierarchy in hierarchies { - //disabled_hierarchies.push(hie.clone()); - match hierarchy.as_str() { - "blkio" => disabled_hierarchies.push(cgroups::Controllers::BlkIo), - "cpu" => disabled_hierarchies.push(cgroups::Controllers::Cpu), - "cpuset" => disabled_hierarchies.push(cgroups::Controllers::CpuSet), - "cpuacct" => disabled_hierarchies.push(cgroups::Controllers::CpuAcct), - "devices" => disabled_hierarchies.push(cgroups::Controllers::Devices), - "freezer" => disabled_hierarchies.push(cgroups::Controllers::Freezer), - "hugetlb" => disabled_hierarchies.push(cgroups::Controllers::HugeTlb), - "memory" => disabled_hierarchies.push(cgroups::Controllers::Mem), - "net_cls" => disabled_hierarchies.push(cgroups::Controllers::NetCls), - "net_prio" => disabled_hierarchies.push(cgroups::Controllers::NetPrio), - "perf_event" => disabled_hierarchies.push(cgroups::Controllers::PerfEvent), - "pids" => disabled_hierarchies.push(cgroups::Controllers::Pids), - "systemd" => disabled_hierarchies.push(cgroups::Controllers::Systemd), - _ => warn!(sl!(), "unknown cgroup controller {}", hierarchy), - } - } - debug!( - sl!(), - "disable cgroup list {:?} from {:?}", disabled_hierarchies, hierarchies - ); -} - -/// Filter out disabled cgroup subsystems. -pub fn filter_disabled_cgroup(controllers: &mut Vec) { - let disabled_hierarchies = DISABLED_HIERARCHIES.lock().unwrap(); - controllers.retain(|x| !disabled_hierarchies.contains(x)); -} - -#[derive(Copy, Clone, Debug)] -pub enum PidType { - /// Add pid to `tasks` - Tasks, - /// Add pid to `cgroup.procs` - CgroupProcs, -} - -/// Get the singleton instance for cgroup v1 hierarchy object. -pub fn get_cgroup_hierarchies() -> &'static cgroups::hierarchies::V1 { - static GLOBAL: Lazy = Lazy::new(cgroups::hierarchies::V1::new); - GLOBAL.deref() -} - -// Prepend a kata specific string to oci cgroup path to form a different cgroup path, thus cAdvisor -// couldn't find kata containers cgroup path on host to prevent it from grabbing the stats data. -const CGROUP_KATA_PREFIX: &str = "kata"; - -/// Convert to a Kata specific cgroup path. -pub fn gen_kata_cgroup_path(path: &str) -> PathBuf { - // Be careful to trim off the possible '/' prefix. Joining an absolute path to a `Path` object - // will replace the old `Path` instead of concat. - Path::new(CGROUP_KATA_PREFIX).join(path.trim_start_matches('/')) -} - -/// Convert to a cgroup path for K8S sandbox. -pub fn gen_sandbox_cgroup_path(path: &str) -> PathBuf { - PathBuf::from(path) -} - -/// A customized cgroup v1 hierarchy object with configurable filters for supported subsystems. -#[derive(Debug)] -pub struct V1Customized { - mount_point: PathBuf, - controllers: Vec, -} - -impl V1Customized { - /// Create a new instance of [`V1Customized`]. - /// - /// The `controllers` configures the subsystems to enable. - /// - /// Note : - /// 1. When enabling both blkio and memory cgroups, blkio cgroup must be enabled before memory - /// cgroup due to a limitation in writeback control of blkio cgroup. - /// 2. cpu, cpuset, cpuacct should be adjacent to each other. - pub fn new(controllers: Vec) -> Self { - let mount_point = get_cgroup_hierarchies().root(); - - V1Customized { - mount_point, - controllers, - } - } -} - -impl Hierarchy for V1Customized { - fn subsystems(&self) -> Vec { - let subsystems = get_cgroup_hierarchies().subsystems(); - - subsystems - .into_iter() - .filter(|sub| { - self.controllers - .contains(&sub.to_controller().control_type()) - }) - .collect::>() - } - - fn root(&self) -> PathBuf { - self.mount_point.clone() - } - - fn root_control_group(&self) -> Cgroup { - Cgroup::load(Box::new(V1Customized::new(self.controllers.clone())), "") - } - - fn v2(&self) -> bool { - false - } -} - -/// An boxed cgroup hierarchy object. -pub type BoxedHierarchyObject = Box; - -/// Create a cgroup hierarchy object with all subsystems disabled. -pub fn get_empty_hierarchy() -> BoxedHierarchyObject { - Box::new(V1Customized::new(vec![])) -} - -/// Create a cgroup hierarchy object for pod sandbox. -pub fn get_sandbox_hierarchy(no_mem: bool) -> BoxedHierarchyObject { - let mut controllers = vec![ - cgroups::Controllers::BlkIo, - cgroups::Controllers::Cpu, - cgroups::Controllers::CpuSet, - cgroups::Controllers::CpuAcct, - cgroups::Controllers::PerfEvent, - ]; - - if !no_mem { - controllers.push(cgroups::Controllers::Mem); - } - filter_disabled_cgroup(&mut controllers); - Box::new(V1Customized::new(controllers)) -} - -/// Create a cgroup hierarchy object with mem subsystem. -/// -/// Note: the mem subsystem may have been disabled, so it will get filtered out. -pub fn get_mem_hierarchy() -> BoxedHierarchyObject { - let mut controllers = vec![cgroups::Controllers::Mem]; - filter_disabled_cgroup(&mut controllers); - Box::new(V1Customized::new(controllers)) -} - -/// Create a cgroup hierarchy object with CPU related subsystems. -/// -/// Note: the mem subsystem may have been disabled, so it will get filtered out. -pub fn get_cpu_hierarchy() -> BoxedHierarchyObject { - let mut controllers = vec![ - cgroups::Controllers::Cpu, - cgroups::Controllers::CpuSet, - cgroups::Controllers::CpuAcct, - ]; - filter_disabled_cgroup(&mut controllers); - Box::new(V1Customized::new(controllers)) -} - -/// Get cgroup hierarchy object from `path`. -pub fn get_hierarchy_by_path(path: &str) -> Result { - let v1 = get_cgroup_hierarchies().clone(); - let valid_path = valid_cgroup_path(path)?; - let cg = cgroups::Cgroup::load(Box::new(v1), valid_path.as_str()); - - let mut hierarchy = vec![]; - for subsys in cg.subsystems() { - let controller = subsys.to_controller(); - if controller.exists() { - hierarchy.push(controller.control_type()); - } - } - - Ok(Box::new(V1Customized::new(hierarchy))) -} - -/// Create or load a cgroup object from a path. -pub fn create_or_load_cgroup(path: &str) -> Result { - let hie = Box::new(get_cgroup_hierarchies().clone()); - - create_or_load_cgroup_with_hier(hie, path) -} - -/// Create or load a cgroup v1 object from a path, with a given hierarchy object. -pub fn create_or_load_cgroup_with_hier(hie: BoxedHierarchyObject, path: &str) -> Result { - let valid_path = valid_cgroup_path(path)?; - if is_cgroup_exist(valid_path.as_str()) { - Ok(cgroups::Cgroup::load(hie, valid_path.as_str())) - } else { - Ok(cgroups::Cgroup::new(hie, valid_path.as_str())) - } -} - -/// Check whether `path` hosts a cgroup hierarchy directory. -pub fn is_cgroup_exist(path: &str) -> bool { - let valid_path = match valid_cgroup_path(path) { - Ok(v) => v, - Err(e) => { - warn!(sl!(), "{}", e); - return false; - } - }; - - let v1 = get_cgroup_hierarchies().clone(); - let cg = cgroups::Cgroup::load(Box::new(v1), valid_path.as_str()); - for subsys in cg.subsystems() { - if subsys.to_controller().exists() { - debug!(sl!(), "cgroup {} exist", path); - return true; - } - } - - false -} - -// Validate the cgroup path is a relative path, do not include ".", "..". -fn valid_cgroup_path(path: &str) -> Result { - let path = path.trim_start_matches('/').to_string(); - - for comp in Path::new(&path).components() { - if !matches!(comp, Component::Normal(_)) { - return Err(Error::InvalidCgroupPath(path.to_string())); - } - } - - Ok(path) -} - -/// Remove all task from cgroup and delete the cgroup. -pub fn force_delete_cgroup(cg: cgroups::Cgroup) -> Result<()> { - delete_cgroup_with_retry(cg, |cg: &Cgroup| { - // if task exist need to delete first. - for cg_pid in cg.tasks() { - warn!(sl!(), "Delete cgroup task pid {}", cg_pid.pid); - cg.remove_task(cg_pid); - } - }) -} - -/// Try to delete a cgroup, call the `do_process` handler at each iteration. -pub fn delete_cgroup_with_retry(cg: Cgroup, mut do_process: F) -> Result<()> -where - F: FnMut(&Cgroup), -{ - // sleep DURATION - const SLEEP_MILLISECS: u64 = 10; - const RETRY_COUNT: u64 = 200; - - // In case of deletion failure caused by "Resource busy", sleep DURATION and retry RETRY times. - for index in 0..RETRY_COUNT { - do_process(&cg); - - if cg.delete().is_ok() { - if index > 0 { - info!( - sl!(), - "cgroup delete cgroup cost {} ms, retry {} times", - index * SLEEP_MILLISECS, - index, - ); - } - return Ok(()); - } - std::thread::sleep(std::time::Duration::from_millis(SLEEP_MILLISECS)) - } - - Err(Error::DeleteCgroup(RETRY_COUNT)) -} - -/// Move the process `pid` into the cgroup `to`. -pub fn move_tgid(pid: u64, to: &Cgroup) -> Result<()> { - info!(sl!(), "try to move tid {:?}", pid); - to.add_task_by_tgid(CgroupPid::from(pid)) - .map_err(|e| Error::AddTgid(pid, e)) -} - -/// Move all processes tasks from `from` to `to`. -pub fn move_cgroup_task(from: &Cgroup, to: &Cgroup) -> Result<()> { - info!(sl!(), "try to move tasks {:?}", from.tasks()); - for cg_pid in from.tasks() { - from.remove_task(CgroupPid::from(cg_pid.pid)); - // TODO: enhance cgroups to implement Copy for CgroupPid - // https://github.com/kata-containers/cgroups-rs/issues/70 - let pid = cg_pid.pid; - to.add_task(cg_pid).map_err(|e| Error::AddTgid(pid, e))?; - } - - Ok(()) -} - -/// Associate a group of tasks with a cgroup, and optionally configure resources for the cgroup. -pub fn update_cgroup_task_resources( - hierarchy: BoxedHierarchyObject, - path: &str, - pids: &[u64], - pid_type: PidType, - resources: Option<&cgroups::Resources>, -) -> Result<()> { - if hierarchy.subsystems().is_empty() { - return Ok(()); - } - fail::fail_point!("update_cgroup_task_resources", |_| { () }); - - let cg = create_or_load_cgroup_with_hier(hierarchy, path)?; - for pid in pids { - let result = match pid_type { - PidType::Tasks => cg.add_task(CgroupPid { pid: *pid }), - PidType::CgroupProcs => cg.add_task_by_tgid(CgroupPid { pid: *pid }), - }; - if let Err(err) = result { - return Err(Error::AddTgid(*pid, err)); - } - } - - if let Some(res) = resources { - cg.apply(res).map_err(Error::ApplyResource)?; - } - - debug!( - sl!(), - "update {:?} {:?} resources {:?} for cgroup {}", pid_type, pids, resources, path - ); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use cgroups::Controllers; - use serial_test::serial; - use std::sync::atomic::{AtomicUsize, Ordering}; - - static GLOBAL_COUNTER: AtomicUsize = AtomicUsize::new(0); - - fn gen_test_path() -> String { - let pid = nix::unistd::getpid().as_raw(); - let index = GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst); - let path = format!("kata-tests-{}-{}", pid, index); - println!("test path {}", path); - path - } - - fn get_hierarchy(controllers: Vec) -> Box { - Box::new(V1Customized::new(controllers)) - } - - #[test] - fn test_v1_customized_cgroup() { - update_disabled_cgroup_list(&[]); - - let c = V1Customized::new(vec![]); - assert_eq!(c.subsystems().len(), 0); - assert!(!c.v2()); - - let c = V1Customized::new(vec![Controllers::Cpu, Controllers::CpuSet]); - assert_eq!(c.subsystems().len(), 2); - assert!(!c.v2()); - } - - #[test] - #[serial] - fn test_filter_disabled_cgroup() { - update_disabled_cgroup_list(&[]); - assert_eq!(DISABLED_HIERARCHIES.lock().unwrap().len(), 0); - - let disabeld = ["perf_event".to_string()]; - update_disabled_cgroup_list(&disabeld); - assert_eq!(DISABLED_HIERARCHIES.lock().unwrap().len(), 1); - assert_eq!( - DISABLED_HIERARCHIES.lock().unwrap()[0], - Controllers::PerfEvent - ); - - let mut subsystems = vec![Controllers::BlkIo, Controllers::PerfEvent, Controllers::Cpu]; - filter_disabled_cgroup(&mut subsystems); - assert_eq!(subsystems.len(), 2); - assert_eq!(subsystems[0], Controllers::BlkIo); - assert_eq!(subsystems[1], Controllers::Cpu); - - let disabeld = ["cpu".to_string(), "cpuset".to_string()]; - update_disabled_cgroup_list(&disabeld); - assert_eq!(DISABLED_HIERARCHIES.lock().unwrap().len(), 2); - - let mut subsystems = vec![Controllers::BlkIo, Controllers::PerfEvent, Controllers::Cpu]; - filter_disabled_cgroup(&mut subsystems); - assert_eq!(subsystems.len(), 2); - assert_eq!(subsystems[0], Controllers::BlkIo); - assert_eq!(subsystems[1], Controllers::PerfEvent); - - update_disabled_cgroup_list(&[]); - } - - #[test] - fn test_create_empty_hierarchy() { - update_disabled_cgroup_list(&[]); - - let controller = get_empty_hierarchy(); - assert_eq!(controller.subsystems().len(), 0); - assert!(!controller.root_control_group().v2()); - } - - #[test] - #[serial] - fn test_create_sandbox_hierarchy() { - update_disabled_cgroup_list(&[]); - - let controller = get_sandbox_hierarchy(true); - assert_eq!(controller.subsystems().len(), 5); - assert!(!controller.root_control_group().v2()); - - let controller = get_sandbox_hierarchy(false); - assert_eq!(controller.subsystems().len(), 6); - assert!(!controller.root_control_group().v2()); - } - - #[test] - #[serial] - fn test_get_hierarchy() { - update_disabled_cgroup_list(&[]); - - let controller = get_mem_hierarchy(); - assert!(!controller.v2()); - assert_eq!(controller.subsystems().len(), 1); - - let controller = get_cpu_hierarchy(); - assert!(!controller.v2()); - assert_eq!(controller.subsystems().len(), 3); - } - - #[test] - #[serial] - fn test_create_cgroup_default() { - update_disabled_cgroup_list(&[]); - // test need root permission - if !nix::unistd::getuid().is_root() { - println!("test need root permission"); - return; - } - - let v1 = Box::new(cgroups::hierarchies::V1::new()); - let test_path = gen_test_path(); - let cg_path = test_path.as_str(); - assert!(!is_cgroup_exist(cg_path)); - - // new cgroup - let cg = cgroups::Cgroup::new(v1, cg_path); - assert!(is_cgroup_exist(cg_path)); - - // add task - let _ = cg.add_task(cgroups::CgroupPid { - pid: nix::unistd::getpid().as_raw() as u64, - }); - - // delete cgroup - force_delete_cgroup(cg).unwrap(); - assert!(!is_cgroup_exist(cg_path)); - } - - #[test] - #[serial] - fn test_create_cgroup_cpus() { - update_disabled_cgroup_list(&[]); - // test need root permission - if !nix::unistd::getuid().is_root() { - println!("test need root permission"); - return; - } - if num_cpus::get() <= 1 { - println!("The unit test is only supported on SMP systems."); - return; - } - - let test_path = gen_test_path(); - let cg_path = test_path.as_str(); - assert!(!is_cgroup_exist(cg_path)); - - // new cgroup - let cgroup = create_or_load_cgroup(cg_path).unwrap(); - let cpus: &cgroups::cpuset::CpuSetController = cgroup.controller_of().unwrap(); - cpus.set_cpus("0-1").unwrap(); - assert!(is_cgroup_exist(cg_path)); - - // current cgroup - let current_cgroup = create_or_load_cgroup(cg_path).unwrap(); - let current_cpus: &cgroups::cpuset::CpuSetController = - current_cgroup.controller_of().unwrap(); - // check value - assert_eq!(cpus.cpuset().cpus, current_cpus.cpuset().cpus); - - // delete cgroup - force_delete_cgroup(cgroup).unwrap(); - assert!(!is_cgroup_exist(cg_path)); - } - - #[test] - #[serial] - fn test_create_cgroup_with_parent() { - update_disabled_cgroup_list(&[]); - // test need root permission - if !nix::unistd::getuid().is_root() { - println!("test need root permission"); - return; - } - if num_cpus::get() <= 1 { - println!("The unit test is only supported on SMP systems."); - return; - } - - let test_path = gen_test_path(); - let cg_path = test_path.as_str(); - assert!(!is_cgroup_exist(cg_path)); - - // new cgroup - let cg = create_or_load_cgroup(cg_path).unwrap(); - let cpus: &cgroups::cpuset::CpuSetController = cg.controller_of().unwrap(); - cpus.set_cpus("0-1").unwrap(); - assert!(is_cgroup_exist(cg_path)); - - // new cgroup 1 - let cg_test_path_1 = format!("{}/vcpu0", test_path); - let cg_path_1 = cg_test_path_1.as_str(); - let cg1 = create_or_load_cgroup(cg_path_1).unwrap(); - let cpus1: &cgroups::cpuset::CpuSetController = cg1.controller_of().unwrap(); - cpus1.set_cpus("0").unwrap(); - assert!(is_cgroup_exist(cg_path_1)); - - // new cgroup 2 - let cg_test_path_2 = format!("{}/vcpu1", test_path); - let cg_path_2 = cg_test_path_2.as_str(); - // new cgroup - let cg2 = create_or_load_cgroup(cg_path_2).unwrap(); - let cpus2: &cgroups::cpuset::CpuSetController = cg2.controller_of().unwrap(); - cpus2.set_cpus("1").unwrap(); - assert!(is_cgroup_exist(cg_path_2)); - - // must delete sub dir first - force_delete_cgroup(cg1).unwrap(); - assert!(!is_cgroup_exist(cg_path_1)); - force_delete_cgroup(cg2).unwrap(); - assert!(!is_cgroup_exist(cg_path_2)); - force_delete_cgroup(cg).unwrap(); - assert!(!is_cgroup_exist(cg_path)); - } - - fn assert_customize_path_exist(path: &str, current_subsystems: &[Subsystem], expect: bool) { - println!("assert customize path {} exist expect {}", path, expect); - let v1 = Box::new(cgroups::hierarchies::V1::new()); - let v1_cg = Cgroup::load(v1, path); - let v1_subsystems = v1_cg.subsystems(); - - for v1_sub in v1_subsystems { - let check_expect = || -> bool { - for current_sub in current_subsystems { - if v1_sub.to_controller().control_type() - == current_sub.to_controller().control_type() - { - return expect; - } - } - false - }(); - assert_eq!( - check_expect, - v1_sub.to_controller().exists(), - "failed to check path {:?} subsystem {:?}", - path, - v1_sub - ) - } - } - - fn clean_cgroup_v1(path: &str) { - let v1 = Box::new(cgroups::hierarchies::V1::new()); - let cg = Cgroup::load(v1.clone(), path); - delete_cgroup_with_retry(cg, |_: &Cgroup| {}).unwrap(); - - let check_cg = Cgroup::load(v1, path); - assert_customize_path_exist(path, check_cg.subsystems(), false); - } - - #[test] - #[serial] - fn test_customize_hierarchies() { - update_disabled_cgroup_list(&[]); - // test need root permission - if !nix::unistd::getuid().is_root() { - println!("test need root permission"); - return; - } - - let cg_path_1 = "test_customize_hierarchies1"; - let cg_path_2 = "test_customize_hierarchies2"; - - // clean - clean_cgroup_v1(cg_path_1); - clean_cgroup_v1(cg_path_2); - - // check customized cgroup - // With some kernels, Cpu and CpuAcct are combined into one directory, so enable both - // to ease test code. - let controllers_1 = vec![Controllers::Cpu, Controllers::CpuAcct]; - let controllers_2 = vec![Controllers::Cpu, Controllers::CpuSet, Controllers::CpuAcct]; - let cg_1 = Cgroup::new(get_hierarchy(controllers_1.clone()), cg_path_1); - let cg_2 = Cgroup::new(get_hierarchy(controllers_2.clone()), cg_path_2); - - assert_customize_path_exist(cg_path_1, cg_1.subsystems(), true); - assert_customize_path_exist(cg_path_2, cg_2.subsystems(), true); - - // delete - let _ = cg_1.delete(); - let _ = cg_2.delete(); - - // check after delete - let check_cg_1 = Cgroup::load(get_hierarchy(controllers_1), cg_path_1); - let check_cg_2 = Cgroup::load(get_hierarchy(controllers_2), cg_path_2); - assert_customize_path_exist(cg_path_1, check_cg_1.subsystems(), false); - assert_customize_path_exist(cg_path_2, check_cg_2.subsystems(), false); - } - - #[test] - #[serial] - fn test_task_move() { - update_disabled_cgroup_list(&[]); - // test need root permission - if !nix::unistd::getuid().is_root() { - println!("test need root permission"); - return; - } - - let cg_path_1 = "test_task_move_before"; - let cg_path_2 = "test_task_move_after"; - - // clean - clean_cgroup_v1(cg_path_1); - clean_cgroup_v1(cg_path_2); - - // With some kernels, Cpu and CpuAcct are combined into one directory, so enable both - // to ease test code. - let controllers = vec![Controllers::Cpu, Controllers::CpuAcct]; - let cg_1 = Cgroup::new(get_hierarchy(controllers.clone()), cg_path_1); - let cg_2 = Cgroup::new(get_hierarchy(controllers.clone()), cg_path_2); - - assert_customize_path_exist(cg_path_1, cg_1.subsystems(), true); - assert_customize_path_exist(cg_path_2, cg_2.subsystems(), true); - - // add task - let pid = libc::pid_t::from(nix::unistd::getpid()) as u64; - let _ = cg_1.add_task(CgroupPid::from(pid)).unwrap(); - let mut cg_task_1 = cg_1.tasks(); - let mut cg_task_2 = cg_2.tasks(); - assert_eq!(1, cg_task_1.len()); - assert_eq!(0, cg_task_2.len()); - - // move task - let _ = cg_2.add_task(CgroupPid::from(pid)).unwrap(); - cg_task_1 = cg_1.tasks(); - cg_task_2 = cg_2.tasks(); - assert_eq!(0, cg_task_1.len()); - assert_eq!(1, cg_task_2.len()); - - cg_2.remove_task(CgroupPid::from(pid)); - - // delete - cg_1.delete().unwrap(); - // delete cg_2 with retry because of possible unknown failed - // caused by "Resource busy", we do the same in the production - // code, so it makes sense in the test. - delete_cgroup_with_retry(cg_2, |_| {}).unwrap(); - - // check after delete - let check_cg_1 = Cgroup::load(get_hierarchy(controllers.clone()), cg_path_1); - let check_cg_2 = Cgroup::load(get_hierarchy(controllers), cg_path_2); - assert_customize_path_exist(cg_path_1, check_cg_1.subsystems(), false); - assert_customize_path_exist(cg_path_2, check_cg_2.subsystems(), false); - } - - #[test] - fn test_gen_kata_cgroup_path() { - assert_eq!( - &gen_kata_cgroup_path("sandbox1/container2"), - Path::new("kata/sandbox1/container2") - ); - assert_eq!( - &gen_kata_cgroup_path("/sandbox1/container2"), - Path::new("kata/sandbox1/container2") - ); - assert_eq!( - &gen_kata_cgroup_path("/sandbox1:container2"), - Path::new("kata/sandbox1:container2") - ); - } -} diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 606ebf6bde29..656a7c666cbd 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -6,13 +6,13 @@ #[macro_use] extern crate slog; -pub mod cgroup; pub mod device; pub mod fs; pub mod hooks; pub mod k8s; pub mod mount; pub mod numa; +pub mod spec; pub mod validate; // Convenience macro to obtain the scoped logger diff --git a/src/libs/kata-sys-util/src/spec.rs b/src/libs/kata-sys-util/src/spec.rs new file mode 100644 index 000000000000..3aa32434b13a --- /dev/null +++ b/src/libs/kata-sys-util/src/spec.rs @@ -0,0 +1,94 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::PathBuf; + +use kata_types::container::ContainerType; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// unknow container type + #[error("unknow container type {0}")] + UnknowContainerType(String), + /// missing sandboxID + #[error("missing sandboxID")] + MissingSandboxID, + /// oci error + #[error("oci error")] + Oci(#[from] oci::Error), +} + +const CRI_CONTAINER_TYPE_KEY_LIST: &[&str] = &[ + // cri containerd + "io.kubernetes.cri.container-type", + // cri-o + "io.kubernetes.cri-o.ContainerType", + // docker shim + "io.kubernetes.docker.type", +]; + +const CRI_SANDBOX_ID_KEY_LIST: &[&str] = &[ + // cri containerd + "io.kubernetes.cri.sandbox-id", + // cri-o + "io.kubernetes.cri-o.SandboxID", + // docker shim + "io.kubernetes.sandbox.id", +]; + +/// container sandbox info +#[derive(Debug, Clone)] +pub enum ShimIdInfo { + /// Sandbox + Sandbox, + /// Container + Container(String), +} + +/// get container type +pub fn get_contaier_type(spec: &oci::Spec) -> Result { + for k in CRI_CONTAINER_TYPE_KEY_LIST.iter() { + if let Some(type_value) = spec.annotations.get(*k) { + match type_value.as_str() { + "sandbox" => return Ok(ContainerType::PodSandbox), + "podsandbox" => return Ok(ContainerType::PodSandbox), + "container" => return Ok(ContainerType::PodContainer), + _ => return Err(Error::UnknowContainerType(type_value.clone())), + } + } + } + + Ok(ContainerType::PodSandbox) +} + +/// get shim id info +pub fn get_shim_id_info() -> Result { + let spec = load_oci_spec()?; + match get_contaier_type(&spec)? { + ContainerType::PodSandbox => Ok(ShimIdInfo::Sandbox), + ContainerType::PodContainer => { + for k in CRI_SANDBOX_ID_KEY_LIST { + if let Some(sandbox_id) = spec.annotations.get(*k) { + return Ok(ShimIdInfo::Container(sandbox_id.into())); + } + } + Err(Error::MissingSandboxID) + } + } +} + +/// get bundle path +pub fn get_bundle_path() -> std::io::Result { + std::env::current_dir() +} + +/// load oci spec +pub fn load_oci_spec() -> oci::Result { + let bundle_path = get_bundle_path()?; + let spec_file = bundle_path.join("config.json"); + + oci::Spec::load(spec_file.to_str().unwrap_or_default()) +} diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index f56b90609a87..7cb064508e62 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -93,9 +93,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "block-buffer" @@ -157,7 +157,7 @@ checksum = "cdae996d9638ba03253ffa1c93345a585974a97abbdeab9176c77922f3efc1e8" dependencies = [ "libc", "log", - "nix", + "nix 0.23.1", "regex", ] @@ -184,7 +184,7 @@ dependencies = [ "kata-sys-util", "kata-types", "lazy_static", - "nix", + "nix 0.23.1", "oci", "protobuf", "serde_json", @@ -602,7 +602,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix", + "nix 0.23.1", "oci", "once_cell", "serde_json", @@ -764,6 +764,19 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + [[package]] name = "nix" version = "0.23.1" @@ -1118,6 +1131,26 @@ dependencies = [ "winapi", ] +[[package]] +name = "resource" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "cgroups-rs", + "hypervisor", + "kata-sys-util", + "kata-types", + "lazy_static", + "libc", + "log", + "logging", + "nix 0.16.1", + "oci", + "slog", + "tokio", +] + [[package]] name = "runtimes" version = "0.1.0" @@ -1270,7 +1303,7 @@ dependencies = [ "libc", "log", "logging", - "nix", + "nix 0.23.1", "oci", "protobuf", "rand", @@ -1570,7 +1603,7 @@ dependencies = [ "futures 0.3.21", "libc", "log", - "nix", + "nix 0.23.1", "protobuf", "protobuf-codegen-pure", "thiserror", @@ -1699,6 +1732,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "vsock" version = "0.2.6" @@ -1706,7 +1745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" dependencies = [ "libc", - "nix", + "nix 0.23.1", ] [[package]] diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index 176bb222adea..dcd34e64bd7b 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -3,6 +3,6 @@ members = [ "crates/shim", # TODO: current only for check, delete after use the agent crate "crates/agent", - # TODO: current only for check, delete after use the hypervisor crate - "crates/hypervisor", + # TODO: current only for check, delete after use the resource crate + "crates/resource", ] diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml new file mode 100644 index 000000000000..743575329be3 --- /dev/null +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "resource" +version = "0.1.0" +authors = ["The Kata Containers community "] +edition = "2018" + +[dependencies] +anyhow = "^1.0" +async-trait = "0.1.48" +cgroups-rs = "0.2.9" +lazy_static = "1.4.0" +libc = ">=0.2.39" +log = "^0.4.0" +nix = "0.16.0" +slog = "2.5.2" +tokio = { version = "1.8.0", features = ["sync"] } + +hypervisor = { path = "../hypervisor" } +kata-types = { path = "../../../libs/kata-types" } +kata-sys-util = { path = "../../../libs/kata-sys-util" } +logging = { path = "../../../libs/logging" } +oci = { path = "../../../libs/oci" } + +[features] diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs new file mode 100644 index 000000000000..367e9fba8339 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs @@ -0,0 +1,217 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod utils; + +use std::{ + collections::{HashMap, HashSet}, + iter::FromIterator, + sync::Arc, +}; + +use anyhow::{anyhow, Context, Result}; +use cgroups_rs::{cgroup_builder::CgroupBuilder, Cgroup, CgroupPid, CpuResources, Resources}; +use hypervisor::Hypervisor; +use kata_sys_util::spec::load_oci_spec; +use kata_types::config::TomlConfig; +use oci::LinuxResources; +use tokio::sync::RwLock; + +pub struct CgroupConfig { + pub path: String, + pub overhead_path: String, + pub sandbox_cgroup_only: bool, +} + +impl CgroupConfig { + fn new(sid: &str, toml_config: &TomlConfig) -> Result { + let overhead_path = utils::gen_overhead_path(sid); + let spec = load_oci_spec()?; + let path = spec + .linux + // The trim of '/' is important, because cgroup_path is a relative path. + .map(|linux| linux.cgroups_path.trim_start_matches('/').to_string()) + .unwrap_or_default(); + + Ok(Self { + path, + overhead_path, + sandbox_cgroup_only: toml_config.runtime.sandbox_cgroup_only, + }) + } +} + +pub struct CgroupsResource { + resources: Arc>>, + cgroup_manager: Cgroup, + overhead_cgroup_manager: Option, +} + +impl CgroupsResource { + pub fn new(sid: &str, toml_config: &TomlConfig) -> Result { + let config = CgroupConfig::new(sid, toml_config)?; + + // Create the sandbox cgroups manager (cgroups on Linux). + // Depending on the sandbox_cgroup_only value, this cgroup + // will either hold all the pod threads (sandbox_cgroup_only is true) + // or only the virtual CPU ones (sandbox_cgroup_only is false). + let hier = cgroups_rs::hierarchies::auto(); + let cgroup_manager = CgroupBuilder::new(&config.path).build(hier); + + // The shim configuration is requesting that we do not put all threads + // into the sandbox resource controller. + // We're creating an overhead controller, with no constraints. Everything but + // the vCPU threads will eventually make it there. + let overhead_cgroup_manager = if !config.sandbox_cgroup_only { + let hier = cgroups_rs::hierarchies::auto(); + Some(CgroupBuilder::new(&config.overhead_path).build(hier)) + } else { + None + }; + + // Add the runtime to the VMM sandbox resource controller + + // By adding the runtime process to either the sandbox or overhead controller, we are making + // sure that any child process of the runtime (i.e. *all* processes serving a Kata pod) + // will initially live in this controller. Depending on the sandbox_cgroup_only settings, we will + // then move the vCPU threads between resource controllers. + let pid = CgroupPid { pid: 0 }; + if let Some(manager) = overhead_cgroup_manager.as_ref() { + manager.add_task_by_tgid(pid).context("add task by tgid")?; + } else { + cgroup_manager + .add_task_by_tgid(pid) + .context("add task by tgid with sandbox only")?; + } + + Ok(Self { + cgroup_manager, + resources: Arc::new(RwLock::new(HashMap::new())), + overhead_cgroup_manager, + }) + } + + /// delete will move the running processes in the cgroup_manager and + /// overhead_cgroup_manager to the parent and then delete the cgroups. + pub async fn delete(&self) -> Result<()> { + for cg_pid in self.cgroup_manager.tasks() { + self.cgroup_manager.remove_task(cg_pid); + } + self.cgroup_manager.delete()?; + + if let Some(overhead) = self.overhead_cgroup_manager.as_ref() { + for cg_pid in overhead.tasks() { + overhead.remove_task(cg_pid); + } + overhead.delete()?; + } + + Ok(()) + } + + pub async fn update_cgroups( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + h: &dyn Hypervisor, + ) -> Result<()> { + let resource = self.calc_resource(linux_resources); + let changed = self.update_resources(cid, resource).await; + + if !changed { + return Ok(()); + } + + self.do_update_cgroups(h).await + } + + async fn update_resources(&self, cid: &str, new_resource: Resources) -> bool { + let mut resources = self.resources.write().await; + let old_resource = resources.insert(cid.to_owned(), new_resource.clone()); + + if let Some(old_resource) = old_resource { + if old_resource == new_resource { + return false; + } + } + + true + } + + async fn do_update_cgroups(&self, h: &dyn Hypervisor) -> Result<()> { + let merged_resources = self.merge_resources().await; + self.cgroup_manager + .apply(&merged_resources) + .map_err(|e| anyhow!(e))?; + + if self.overhead_cgroup_manager.is_some() { + // If we have an overhead controller, new vCPU threads would start there, + // as being children of the VMM PID. + // We need to constrain them by moving them into the sandbox controller. + self.constrain_hypervisor(h).await? + } + + Ok(()) + } + + /// constrain_hypervisor will place the VMM and vCPU threads into resource controllers (cgroups on Linux). + async fn constrain_hypervisor(&self, h: &dyn Hypervisor) -> Result<()> { + let tids = h.get_thread_ids().await?; + let tids = tids.vcpus.values(); + + // All vCPU threads move to the sandbox controller. + for tid in tids { + self.cgroup_manager + .add_task_by_tgid(CgroupPid { pid: *tid as u64 })? + } + + Ok(()) + } + + async fn merge_resources(&self) -> Resources { + let resources = self.resources.read().await; + + let mut cpu_list: HashSet = HashSet::new(); + let mut mem_list: HashSet = HashSet::new(); + + resources.values().for_each(|r| { + if let Some(cpus) = &r.cpu.cpus { + cpu_list.insert(cpus.clone()); + } + if let Some(mems) = &r.cpu.mems { + mem_list.insert(mems.clone()); + } + }); + + let cpu_resource = CpuResources { + cpus: Some(Vec::from_iter(cpu_list.into_iter()).join(",")), + mems: Some(Vec::from_iter(mem_list.into_iter()).join(",")), + ..Default::default() + }; + + Resources { + cpu: cpu_resource, + ..Default::default() + } + } + + fn calc_cpu_resources(&self, linux_resources: Option<&LinuxResources>) -> CpuResources { + let cpu = || -> Option { linux_resources.as_ref()?.cpu.clone() }(); + + CpuResources { + cpus: cpu.clone().map(|cpu| cpu.cpus), + mems: cpu.map(|cpu| cpu.mems), + ..Default::default() + } + } + + fn calc_resource(&self, linux_resources: Option<&LinuxResources>) -> Resources { + Resources { + cpu: self.calc_cpu_resources(linux_resources), + ..Default::default() + } + } +} diff --git a/src/runtime-rs/crates/resource/src/cgroups/utils.rs b/src/runtime-rs/crates/resource/src/cgroups/utils.rs new file mode 100644 index 000000000000..a81316358cb3 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/cgroups/utils.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +// When the Kata overhead threads (I/O, VMM, etc) are not +// placed in the sandbox resource controller (A cgroup on Linux), +// they are moved to a specific, unconstrained resource controller. +// On Linux, assuming the cgroup mount point is at /sys/fs/cgroup/, +// on a cgroup v1 system, the Kata overhead memory cgroup will be at +// /sys/fs/cgroup/memory/kata_overhead/$CGPATH where $CGPATH is +// defined by the orchestrator. +pub(crate) fn gen_overhead_path(path: &str) -> String { + format!("/kata_overhead/{}", path.trim_start_matches('/')) +} diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs new file mode 100644 index 000000000000..8609c82214b1 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub mod cgroups; From 3ff0db05a7fb17517a0bb0bbb66b225564b4b525 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Wed, 30 Mar 2022 10:12:55 +0800 Subject: [PATCH 0045/1953] runtime-rs: support rootfs volume for resource Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/runtime-rs/Cargo.lock | 75 +++++++- src/runtime-rs/crates/resource/Cargo.toml | 5 +- src/runtime-rs/crates/resource/src/lib.rs | 21 +++ src/runtime-rs/crates/resource/src/manager.rs | 92 +++++++++ .../crates/resource/src/manager_inner.rs | 134 +++++++++++++ .../crates/resource/src/rootfs/mod.rs | 123 ++++++++++++ .../resource/src/rootfs/share_fs_rootfs.rs | 59 ++++++ .../crates/resource/src/share_fs/mod.rs | 78 ++++++++ .../resource/src/share_fs/share_virtio_fs.rs | 53 ++++++ .../src/share_fs/share_virtio_fs_inline.rs | 114 +++++++++++ .../share_fs/share_virtio_fs_standalone.rs | 178 ++++++++++++++++++ .../crates/resource/src/share_fs/utils.rs | 94 +++++++++ .../src/share_fs/virtio_fs_share_mount.rs | 50 +++++ .../resource/src/volume/block_volume.rs | 37 ++++ .../resource/src/volume/default_volume.rs | 36 ++++ .../crates/resource/src/volume/mod.rs | 99 ++++++++++ .../resource/src/volume/share_fs_volume.rs | 153 +++++++++++++++ .../crates/resource/src/volume/shm_volume.rs | 105 +++++++++++ 18 files changed, 1500 insertions(+), 6 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/manager.rs create mode 100644 src/runtime-rs/crates/resource/src/manager_inner.rs create mode 100644 src/runtime-rs/crates/resource/src/rootfs/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs create mode 100644 src/runtime-rs/crates/resource/src/share_fs/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs create mode 100644 src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs create mode 100644 src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs create mode 100644 src/runtime-rs/crates/resource/src/share_fs/utils.rs create mode 100644 src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs create mode 100644 src/runtime-rs/crates/resource/src/volume/block_volume.rs create mode 100644 src/runtime-rs/crates/resource/src/volume/default_volume.rs create mode 100644 src/runtime-rs/crates/resource/src/volume/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs create mode 100644 src/runtime-rs/crates/resource/src/volume/shm_volume.rs diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 7cb064508e62..f9e199efc87b 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -293,7 +293,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" dependencies = [ "lazy_static", "log", - "rand", + "rand 0.8.5", ] [[package]] @@ -321,6 +321,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.1.31" @@ -1066,6 +1072,29 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -1074,7 +1103,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.3", ] [[package]] @@ -1084,9 +1113,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.3" @@ -1096,6 +1140,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.11" @@ -1135,6 +1188,7 @@ dependencies = [ name = "resource" version = "0.1.0" dependencies = [ + "agent", "anyhow", "async-trait", "cgroups-rs", @@ -1148,7 +1202,9 @@ dependencies = [ "nix 0.16.1", "oci", "slog", + "slog-scope", "tokio", + "uuid", ] [[package]] @@ -1306,7 +1362,7 @@ dependencies = [ "nix 0.23.1", "oci", "protobuf", - "rand", + "rand 0.8.5", "serial_test", "service", "sha2", @@ -1472,7 +1528,7 @@ dependencies = [ name = "tests_utils" version = "0.1.0" dependencies = [ - "rand", + "rand 0.8.5", ] [[package]] @@ -1692,6 +1748,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cfec50b0842181ba6e713151b72f4ec84a6a7e2c9c8a8a3ffc37bb1cd16b231" +dependencies = [ + "rand 0.3.23", +] + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 743575329be3..115d4fc56bb7 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -13,8 +13,11 @@ libc = ">=0.2.39" log = "^0.4.0" nix = "0.16.0" slog = "2.5.2" -tokio = { version = "1.8.0", features = ["sync"] } +slog-scope = "4.4.0" +tokio = { version = "1.8.0", features = ["process"] } +uuid = { version = "0.4", features = ["v4"] } +agent = { path = "../agent" } hypervisor = { path = "../hypervisor" } kata-types = { path = "../../../libs/kata-types" } kata-sys-util = { path = "../../../libs/kata-sys-util" } diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs index 8609c82214b1..86dc71fe787e 100644 --- a/src/runtime-rs/crates/resource/src/lib.rs +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -4,4 +4,25 @@ // SPDX-License-Identifier: Apache-2.0 // +#[macro_use] +extern crate lazy_static; + +#[macro_use] +extern crate slog; + +logging::logger_with_subsystem!(sl, "resource"); + pub mod cgroups; +pub mod manager; +mod manager_inner; +pub mod rootfs; +pub mod share_fs; +pub mod volume; +pub use manager::ResourceManager; + +use kata_types::config::hypervisor::SharedFsInfo; + +#[derive(Debug)] +pub enum ResourceConfig { + ShareFs(SharedFsInfo), +} diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs new file mode 100644 index 000000000000..f5db93a175cd --- /dev/null +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -0,0 +1,92 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use agent::{Agent, Storage}; +use anyhow::Result; +use hypervisor::Hypervisor; +use kata_types::config::TomlConfig; +use kata_types::mount::Mount; +use oci::LinuxResources; +use tokio::sync::RwLock; + +use crate::{manager_inner::ResourceManagerInner, rootfs::Rootfs, volume::Volume, ResourceConfig}; + +pub struct ResourceManager { + inner: Arc>, +} + +impl ResourceManager { + pub fn new( + sid: &str, + agent: Arc, + hypervisor: Arc, + toml_config: &TomlConfig, + ) -> Result { + Ok(Self { + inner: Arc::new(RwLock::new(ResourceManagerInner::new( + sid, + agent, + hypervisor, + toml_config, + )?)), + }) + } + + pub async fn prepare_before_start_vm(&self, device_configs: Vec) -> Result<()> { + let mut inner = self.inner.write().await; + inner.prepare_before_start_vm(device_configs).await + } + + pub async fn setup_after_start_vm(&self) -> Result<()> { + let mut inner = self.inner.write().await; + inner.setup_after_start_vm().await + } + + pub async fn get_storage_for_sandbox(&self) -> Result> { + let inner = self.inner.read().await; + inner.get_storage_for_sandbox().await + } + + pub async fn handler_rootfs( + &self, + cid: &str, + bundle_path: &str, + rootfs_mounts: &[Mount], + ) -> Result> { + let inner = self.inner.read().await; + inner.handler_rootfs(cid, bundle_path, rootfs_mounts).await + } + + pub async fn handler_volumes( + &self, + cid: &str, + oci_mounts: &[oci::Mount], + ) -> Result>> { + let inner = self.inner.read().await; + inner.handler_volumes(cid, oci_mounts).await + } + + pub async fn dump(&self) { + let inner = self.inner.read().await; + inner.dump().await + } + + pub async fn update_cgroups( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + ) -> Result<()> { + let inner = self.inner.read().await; + inner.update_cgroups(cid, linux_resources).await + } + + pub async fn delete_cgroups(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.delete_cgroups().await + } +} diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs new file mode 100644 index 000000000000..7b7cfea4fdf4 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -0,0 +1,134 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use agent::{Agent, Storage}; +use anyhow::{Context, Result}; +use hypervisor::Hypervisor; +use kata_types::config::TomlConfig; +use kata_types::mount::Mount; +use oci::LinuxResources; + +use crate::{ + cgroups::CgroupsResource, + rootfs::{RootFsResource, Rootfs}, + share_fs::{self, ShareFs}, + volume::{Volume, VolumeResource}, + ResourceConfig, +}; + +pub(crate) struct ResourceManagerInner { + sid: String, + // TODO: remove + #[allow(dead_code)] + agent: Arc, + hypervisor: Arc, + share_fs: Option>, + + pub rootfs_resource: RootFsResource, + pub volume_resource: VolumeResource, + pub cgroups_resource: CgroupsResource, +} + +impl ResourceManagerInner { + pub(crate) fn new( + sid: &str, + agent: Arc, + hypervisor: Arc, + toml_config: &TomlConfig, + ) -> Result { + Ok(Self { + sid: sid.to_string(), + agent, + hypervisor, + share_fs: None, + rootfs_resource: RootFsResource::new(), + volume_resource: VolumeResource::new(), + cgroups_resource: CgroupsResource::new(sid, toml_config)?, + }) + } + + pub async fn prepare_before_start_vm( + &mut self, + device_configs: Vec, + ) -> Result<()> { + for dc in device_configs { + match dc { + ResourceConfig::ShareFs(c) => { + let share_fs = share_fs::new(&self.sid, &c).context("new share fs")?; + share_fs + .setup_device_before_start_vm(self.hypervisor.as_ref()) + .await + .context("setup share fs device before start vm")?; + self.share_fs = Some(share_fs); + } + }; + } + + Ok(()) + } + + pub async fn setup_after_start_vm(&mut self) -> Result<()> { + if let Some(share_fs) = self.share_fs.as_ref() { + share_fs + .setup_device_after_start_vm(self.hypervisor.as_ref()) + .await + .context("setup share fs device after start vm")?; + } + + Ok(()) + } + + pub async fn get_storage_for_sandbox(&self) -> Result> { + let mut storages = vec![]; + if let Some(d) = self.share_fs.as_ref() { + let mut s = d.get_storages().await.context("get storage")?; + storages.append(&mut s); + } + Ok(storages) + } + + pub async fn handler_rootfs( + &self, + cid: &str, + bundle_path: &str, + rootfs_mounts: &[Mount], + ) -> Result> { + self.rootfs_resource + .handler_rootfs(&self.share_fs, cid, bundle_path, rootfs_mounts) + .await + } + + pub async fn handler_volumes( + &self, + cid: &str, + oci_mounts: &[oci::Mount], + ) -> Result>> { + self.volume_resource + .handler_volumes(&self.share_fs, cid, oci_mounts) + .await + } + + pub async fn update_cgroups( + &self, + cid: &str, + linux_resources: Option<&LinuxResources>, + ) -> Result<()> { + self.cgroups_resource + .update_cgroups(cid, linux_resources, self.hypervisor.as_ref()) + .await + } + + pub async fn delete_cgroups(&self) -> Result<()> { + self.cgroups_resource.delete().await + } + + pub async fn dump(&self) { + self.rootfs_resource.dump().await; + self.volume_resource.dump().await; + } +} diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs new file mode 100644 index 000000000000..4063c8bf8205 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod share_fs_rootfs; + +use std::{sync::Arc, vec::Vec}; + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use kata_types::mount::Mount; +use log::{error, info}; +use nix::sys::stat::{self, SFlag}; +use tokio::sync::RwLock; + +use crate::share_fs::ShareFs; + +const ROOTFS: &str = "rootfs"; + +#[async_trait] +pub trait Rootfs: Send + Sync { + async fn get_guest_rootfs_path(&self) -> Result; + async fn get_rootfs_mount(&self) -> Result>; +} + +#[derive(Default)] +struct RootFsResourceInner { + rootfs: Vec>, +} + +pub struct RootFsResource { + inner: Arc>, +} + +impl Default for RootFsResource { + fn default() -> Self { + Self::new() + } +} + +impl RootFsResource { + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(RootFsResourceInner::default())), + } + } + + pub async fn handler_rootfs( + &self, + share_fs: &Option>, + cid: &str, + bundle_path: &str, + rootfs_mounts: &[Mount], + ) -> Result> { + match rootfs_mounts { + mounts_vec if is_single_layer_rootfs(mounts_vec) => { + // Safe as single_layer_rootfs must have one layer + let layer = &mounts_vec[0]; + + let rootfs = if let Some(_dev_id) = get_block_device(&layer.source) { + // block rootfs + unimplemented!() + } else if let Some(share_fs) = share_fs { + // share fs rootfs + let share_fs_mount = share_fs.get_share_fs_mount(); + share_fs_rootfs::ShareFsRootfs::new(&share_fs_mount, cid, bundle_path, layer) + .await + .context("new share fs rootfs")? + } else { + return Err(anyhow!("unsupported rootfs {:?}", &layer)); + }; + + let mut inner = self.inner.write().await; + let r = Arc::new(rootfs); + inner.rootfs.push(r.clone()); + Ok(r) + } + _ => { + return Err(anyhow!( + "unsupported rootfs mounts count {}", + rootfs_mounts.len() + )) + } + } + } + + pub async fn dump(&self) { + let inner = self.inner.read().await; + for r in &inner.rootfs { + info!( + "rootfs {:?}: count {}", + r.get_guest_rootfs_path().await, + Arc::strong_count(r) + ); + } + } +} + +fn is_single_layer_rootfs(rootfs_mounts: &[Mount]) -> bool { + rootfs_mounts.len() == 1 +} + +fn get_block_device(file_path: &str) -> Option { + if file_path.is_empty() { + return None; + } + + match stat::stat(file_path) { + Ok(fstat) => { + if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK { + return Some(fstat.st_rdev); + } + } + Err(err) => { + error!("failed to stat for {} {:?}", file_path, err); + return None; + } + }; + + None +} diff --git a/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs new file mode 100644 index 000000000000..643af13fedb7 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/rootfs/share_fs_rootfs.rs @@ -0,0 +1,59 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use kata_sys_util::mount::Mounter; +use kata_types::mount::Mount; + +use super::{Rootfs, ROOTFS}; +use crate::share_fs::{ShareFsMount, ShareFsRootfsConfig}; + +pub(crate) struct ShareFsRootfs { + guest_path: String, +} + +impl ShareFsRootfs { + pub async fn new( + share_fs_mount: &Arc, + cid: &str, + bundle_path: &str, + rootfs: &Mount, + ) -> Result { + let bundle_rootfs = format!("{}/{}", bundle_path, ROOTFS); + rootfs.mount(&bundle_rootfs).context(format!( + "mount rootfs from {:?} to {}", + &rootfs, &bundle_rootfs + ))?; + + let mount_result = share_fs_mount + .share_rootfs(ShareFsRootfsConfig { + cid: cid.to_string(), + source: bundle_rootfs.to_string(), + target: ROOTFS.to_string(), + readonly: false, + }) + .await + .context("share rootfs")?; + + Ok(ShareFsRootfs { + guest_path: mount_result.guest_path, + }) + } +} + +#[async_trait] +impl Rootfs for ShareFsRootfs { + async fn get_guest_rootfs_path(&self) -> Result { + Ok(self.guest_path.clone()) + } + + async fn get_rootfs_mount(&self) -> Result> { + todo!() + } +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/mod.rs b/src/runtime-rs/crates/resource/src/share_fs/mod.rs new file mode 100644 index 000000000000..36f4f1ec2610 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/mod.rs @@ -0,0 +1,78 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod share_virtio_fs; +mod share_virtio_fs_inline; +use share_virtio_fs_inline::ShareVirtioFsInline; +mod share_virtio_fs_standalone; +use share_virtio_fs_standalone::ShareVirtioFsStandalone; +mod utils; +mod virtio_fs_share_mount; +use virtio_fs_share_mount::VirtiofsShareMount; + +use std::sync::Arc; + +use agent::Storage; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use hypervisor::Hypervisor; +use kata_types::config::hypervisor::SharedFsInfo; + +const VIRTIO_FS: &str = "virtio-fs"; +const INLINE_VIRTIO_FS: &str = "inline-virtio-fs"; + +const KATA_HOST_SHARED_DIR: &str = "/run/kata-containers/shared/sandboxes/"; +const KATA_GUEST_SHARE_DIR: &str = "/run/kata-containers/shared/containers/"; +pub(crate) const DEFAULT_KATA_GUEST_SANDBOX_DIR: &str = "/run/kata-containers/sandbox/"; + +const PASSTHROUGH_FS_DIR: &str = "passthrough"; + +#[async_trait] +pub trait ShareFs: Send + Sync { + fn get_share_fs_mount(&self) -> Arc; + async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()>; + async fn setup_device_after_start_vm(&self, h: &dyn Hypervisor) -> Result<()>; + async fn get_storages(&self) -> Result>; +} + +pub struct ShareFsRootfsConfig { + // TODO: for nydus v5/v6 need to update ShareFsMount + pub cid: String, + pub source: String, + pub target: String, + pub readonly: bool, +} + +pub struct ShareFsVolumeConfig { + pub cid: String, + pub source: String, + pub target: String, + pub readonly: bool, +} + +pub struct ShareFsMountResult { + pub guest_path: String, +} + +#[async_trait] +pub trait ShareFsMount: Send + Sync { + async fn share_rootfs(&self, config: ShareFsRootfsConfig) -> Result; + async fn share_volume(&self, config: ShareFsVolumeConfig) -> Result; +} + +pub fn new(id: &str, config: &SharedFsInfo) -> Result> { + let shared_fs = config.shared_fs.clone(); + let shared_fs = shared_fs.unwrap_or_default(); + match shared_fs.as_str() { + INLINE_VIRTIO_FS => Ok(Arc::new( + ShareVirtioFsInline::new(id, config).context("new inline virtio fs")?, + )), + VIRTIO_FS => Ok(Arc::new( + ShareVirtioFsStandalone::new(id, config).context("new standalone virtio fs")?, + )), + _ => Err(anyhow!("unsupported shred fs {:?}", &shared_fs)), + } +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs new file mode 100644 index 000000000000..364614dfd84e --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -0,0 +1,53 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::Path; + +use anyhow::{Context, Result}; +use hypervisor::{device, Hypervisor}; +use kata_sys_util::mount; + +use super::utils; + +pub(crate) const MOUNT_GUEST_TAG: &str = "kataShared"; +pub(crate) const PASSTHROUGH_FS_DIR: &str = "passthrough"; + +pub(crate) const FS_TYPE_VIRTIO_FS: &str = "virtio_fs"; +pub(crate) const KATA_VIRTIO_FS_DEV_TYPE: &str = "virtio-fs"; + +const VIRTIO_FS_SOCKET: &str = "virtiofsd.sock"; + +pub(crate) fn generate_sock_path(root: &str) -> String { + let socket_path = Path::new(root).join(VIRTIO_FS_SOCKET); + socket_path.to_str().unwrap().to_string() +} + +pub(crate) async fn prepare_virtiofs( + h: &dyn Hypervisor, + fs_type: &str, + id: &str, + root: &str, +) -> Result<()> { + let host_ro_dest = utils::get_host_ro_shared_path(id); + utils::ensure_dir_exist(&host_ro_dest)?; + + let host_rw_dest = utils::get_host_rw_shared_path(id); + utils::ensure_dir_exist(&host_rw_dest)?; + + mount::bind_mount_unchecked(&host_rw_dest, &host_ro_dest, true) + .context("bind mount shared_fs directory")?; + + let share_fs_device = device::Device::ShareFsDevice(device::ShareFsDeviceConfig { + sock_path: generate_sock_path(root), + mount_tag: String::from(MOUNT_GUEST_TAG), + host_path: String::from(host_ro_dest.to_str().unwrap()), + fs_type: fs_type.to_string(), + queue_size: 0, + queue_num: 0, + }); + h.add_device(share_fs_device).await.context("add device")?; + Ok(()) +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs new file mode 100644 index 000000000000..146efc609400 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs @@ -0,0 +1,114 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use agent::Storage; +use anyhow::{Context, Result}; +use async_trait::async_trait; +use hypervisor::{ + device::{Device as HypervisorDevice, ShareFsMountConfig, ShareFsMountType, ShareFsOperation}, + Hypervisor, +}; +use kata_types::config::hypervisor::SharedFsInfo; + +use super::{ + share_virtio_fs::{ + prepare_virtiofs, FS_TYPE_VIRTIO_FS, KATA_VIRTIO_FS_DEV_TYPE, MOUNT_GUEST_TAG, + PASSTHROUGH_FS_DIR, + }, + utils, ShareFs, *, +}; + +lazy_static! { + pub(crate) static ref SHARED_DIR_VIRTIO_FS_OPTIONS: Vec:: = vec![ + String::from("default_permissions,allow_other,rootmode=040000,user_id=0,group_id=0"), + String::from("nodev"), + ]; +} + +#[derive(Debug, Clone)] +pub struct ShareVirtioFsInlineConfig { + pub id: String, +} + +pub struct ShareVirtioFsInline { + config: ShareVirtioFsInlineConfig, + share_fs_mount: Arc, +} + +impl ShareVirtioFsInline { + pub(crate) fn new(id: &str, _config: &SharedFsInfo) -> Result { + Ok(Self { + config: ShareVirtioFsInlineConfig { id: id.to_string() }, + share_fs_mount: Arc::new(VirtiofsShareMount::new(id)), + }) + } +} + +#[async_trait] +impl ShareFs for ShareVirtioFsInline { + fn get_share_fs_mount(&self) -> Arc { + self.share_fs_mount.clone() + } + + async fn setup_device_before_start_vm(&self, h: &dyn Hypervisor) -> Result<()> { + prepare_virtiofs(h, INLINE_VIRTIO_FS, &self.config.id, "") + .await + .context("prepare virtiofs")?; + Ok(()) + } + + async fn setup_device_after_start_vm(&self, h: &dyn Hypervisor) -> Result<()> { + setup_inline_virtiofs(&self.config.id, h) + .await + .context("setup inline virtiofs")?; + Ok(()) + } + async fn get_storages(&self) -> Result> { + // setup storage + let mut storages: Vec = Vec::new(); + + let mut shared_options = SHARED_DIR_VIRTIO_FS_OPTIONS.clone(); + shared_options.push(format!("tag={}", MOUNT_GUEST_TAG)); + + let shared_volume: Storage = Storage { + driver: String::from(KATA_VIRTIO_FS_DEV_TYPE), + driver_options: Vec::new(), + source: String::from(MOUNT_GUEST_TAG), + fs_type: String::from(FS_TYPE_VIRTIO_FS), + options: shared_options, + mount_point: String::from(KATA_GUEST_SHARE_DIR), + }; + + storages.push(shared_volume); + Ok(storages) + } +} + +async fn setup_inline_virtiofs(id: &str, h: &dyn Hypervisor) -> Result<()> { + // - source is the absolute path of PASSTHROUGH_FS_DIR on host, e.g. + // /run/kata-containers/shared/sandboxes//passthrough + // - mount point is the path relative to KATA_GUEST_SHARE_DIR in guest + let mnt = format!("/{}", PASSTHROUGH_FS_DIR); + + let rw_source = utils::get_host_rw_shared_path(id).join(PASSTHROUGH_FS_DIR); + utils::ensure_dir_exist(&rw_source)?; + + let ro_source = utils::get_host_ro_shared_path(id).join(PASSTHROUGH_FS_DIR); + let source = String::from(ro_source.to_str().unwrap()); + + let virtio_fs = HypervisorDevice::ShareFsMount(ShareFsMountConfig { + source: source.clone(), + fstype: ShareFsMountType::PASSTHROUGH, + mount_point: mnt, + config: None, + tag: String::from(MOUNT_GUEST_TAG), + op: ShareFsOperation::Mount, + prefetch_list_path: None, + }); + h.add_device(virtio_fs) + .await + .context(format!("fail to attach passthrough fs {:?}", source)) +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs new file mode 100644 index 000000000000..b6f143dcd319 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs @@ -0,0 +1,178 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{process::Stdio, sync::Arc}; + +use agent::Storage; +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use hypervisor::Hypervisor; +use kata_types::config::hypervisor::SharedFsInfo; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::{Child, Command}, + sync::{ + mpsc::{channel, Receiver, Sender}, + RwLock, + }, +}; + +use super::{ + share_virtio_fs::generate_sock_path, utils::get_host_ro_shared_path, + virtio_fs_share_mount::VirtiofsShareMount, ShareFs, ShareFsMount, +}; + +#[derive(Debug, Clone)] +pub struct ShareVirtioFsStandaloneConfig { + id: String, + jail_root: String, + + // virtio_fs_daemon is the virtio-fs vhost-user daemon path + pub virtio_fs_daemon: String, + // virtio_fs_cache cache mode for fs version cache or "none" + pub virtio_fs_cache: String, + // virtio_fs_extra_args passes options to virtiofsd daemon + pub virtio_fs_extra_args: Vec, +} + +#[derive(Default)] +struct ShareVirtioFsStandaloneInner { + pid: Option, +} +pub(crate) struct ShareVirtioFsStandalone { + inner: Arc>, + config: ShareVirtioFsStandaloneConfig, + share_fs_mount: Arc, +} + +impl ShareVirtioFsStandalone { + pub(crate) fn new(id: &str, _config: &SharedFsInfo) -> Result { + Ok(Self { + inner: Arc::new(RwLock::new(ShareVirtioFsStandaloneInner::default())), + // TODO: update with config + config: ShareVirtioFsStandaloneConfig { + id: id.to_string(), + jail_root: "".to_string(), + virtio_fs_daemon: "".to_string(), + virtio_fs_cache: "".to_string(), + virtio_fs_extra_args: vec![], + }, + share_fs_mount: Arc::new(VirtiofsShareMount::new(id)), + }) + } + + fn virtiofsd_args(&self, sock_path: &str) -> Result> { + let source_path = get_host_ro_shared_path(&self.config.id); + if !source_path.exists() { + return Err(anyhow!("The virtiofs shared path didn't exist")); + } + + let mut args: Vec = vec![ + String::from("-f"), + String::from("-o"), + format!("vhost_user_socket={}", sock_path), + String::from("-o"), + format!("source={}", source_path.to_str().unwrap()), + String::from("-o"), + format!("cache={}", self.config.virtio_fs_cache), + ]; + + if !self.config.virtio_fs_extra_args.is_empty() { + let mut extra_args: Vec = self.config.virtio_fs_extra_args.clone(); + args.append(&mut extra_args); + } + + Ok(args) + } + + async fn setup_virtiofsd(&self) -> Result<()> { + let sock_path = generate_sock_path(&self.config.jail_root); + let args = self.virtiofsd_args(&sock_path).context("virtiofsd args")?; + + let mut cmd = Command::new(&self.config.virtio_fs_daemon); + let child_cmd = cmd.args(&args).stderr(Stdio::piped()); + let child = child_cmd.spawn().context("spawn virtiofsd")?; + + // update virtiofsd pid{ + { + let mut inner = self.inner.write().await; + inner.pid = child.id(); + } + + let (tx, mut rx): (Sender>, Receiver>) = channel(100); + tokio::spawn(run_virtiofsd(child, tx)); + + // TODO: support timeout + match rx.recv().await.unwrap() { + Ok(_) => { + info!(sl!(), "start virtiofsd successfully"); + Ok(()) + } + Err(e) => { + error!(sl!(), "failed to start virtiofsd {}", e); + self.shutdown_virtiofsd() + .await + .context("shutdown_virtiofsd")?; + Err(anyhow!("failed to start virtiofsd")) + } + } + } + + async fn shutdown_virtiofsd(&self) -> Result<()> { + let mut inner = self.inner.write().await; + + if let Some(pid) = inner.pid.take() { + info!(sl!(), "shutdown virtiofsd pid {}", pid); + let pid = ::nix::unistd::Pid::from_raw(pid as i32); + if let Err(err) = ::nix::sys::signal::kill(pid, nix::sys::signal::SIGKILL) { + if err != ::nix::Error::Sys(nix::errno::Errno::ESRCH) { + return Err(anyhow!("failed to kill virtiofsd pid {} {}", pid, err)); + } + } + } + + Ok(()) + } +} + +async fn run_virtiofsd(mut child: Child, tx: Sender>) -> Result<()> { + let stderr = child.stderr.as_mut().unwrap(); + let stderr_reader = BufReader::new(stderr); + let mut lines = stderr_reader.lines(); + + while let Some(buffer) = lines.next_line().await.context("read next line")? { + let trim_buffer = buffer.trim_end(); + if !trim_buffer.is_empty() { + info!(sl!(), "source: virtiofsd {}", trim_buffer); + } + if buffer.contains("Waiting for vhost-user socket connection") { + tx.send(Ok(())).await.unwrap(); + } + } + + info!(sl!(), "wait virtiofsd {:?}", child.wait().await); + Ok(()) +} + +#[async_trait] +impl ShareFs for ShareVirtioFsStandalone { + fn get_share_fs_mount(&self) -> Arc { + self.share_fs_mount.clone() + } + + async fn setup_device_before_start_vm(&self, _h: &dyn Hypervisor) -> Result<()> { + self.setup_virtiofsd().await.context("setup virtiofsd")?; + Ok(()) + } + + async fn setup_device_after_start_vm(&self, _h: &dyn Hypervisor) -> Result<()> { + Ok(()) + } + + async fn get_storages(&self) -> Result> { + Ok(vec![]) + } +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs new file mode 100644 index 000000000000..3a4b0c74394d --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -0,0 +1,94 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use kata_sys_util::mount; + +use super::*; + +pub(crate) fn ensure_dir_exist(path: &Path) -> Result<()> { + if !path.exists() { + std::fs::create_dir_all(path).context(format!("failed to create directory {:?}", path))?; + } + Ok(()) +} + +pub(crate) fn share_to_guest( + // absolute path for source + source: &str, + // relative path for target + target: &str, + sid: &str, + cid: &str, + readonly: bool, + is_volume: bool, +) -> Result { + let host_dest = do_get_host_path(target, sid, cid, is_volume, false); + mount::bind_mount_unchecked(source, &host_dest, readonly) + .context(format!("failed to bind mount {} to {}", source, &host_dest))?; + + // bind mount remount event is not propagated to mount subtrees, so we have + // to remount the read only dir mount point directly. + if readonly { + let dst = do_get_host_path(target, sid, cid, is_volume, true); + mount::bind_remount_read_only(&dst).context("bind remount readonly")?; + } + + Ok(do_get_guest_path(target, cid, is_volume)) +} + +pub(crate) fn get_host_ro_shared_path(id: &str) -> PathBuf { + Path::new(KATA_HOST_SHARED_DIR).join(id).join("ro") +} + +pub(crate) fn get_host_rw_shared_path(id: &str) -> PathBuf { + Path::new(KATA_HOST_SHARED_DIR).join(id).join("rw") +} + +fn do_get_guest_any_path(target: &str, cid: &str, is_volume: bool, is_virtiofs: bool) -> String { + let dir = PASSTHROUGH_FS_DIR; + let guest_share_dir = if is_virtiofs { + Path::new("/") + } else { + Path::new(KATA_GUEST_SHARE_DIR) + }; + + let path = if is_volume && !is_virtiofs { + guest_share_dir.join(dir).join(target) + } else { + guest_share_dir.join(dir).join(cid).join(target) + }; + path.to_str().unwrap().to_string() +} + +fn do_get_guest_path(target: &str, cid: &str, is_volume: bool) -> String { + do_get_guest_any_path(target, cid, is_volume, false) +} + +fn do_get_host_path( + target: &str, + sid: &str, + cid: &str, + is_volume: bool, + read_only: bool, +) -> String { + let dir = PASSTHROUGH_FS_DIR; + + let get_host_path = if read_only { + get_host_ro_shared_path + } else { + get_host_rw_shared_path + }; + + let path = if is_volume { + get_host_path(sid).join(dir).join(target) + } else { + get_host_path(sid).join(dir).join(cid).join(target) + }; + path.to_str().unwrap().to_string() +} diff --git a/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs new file mode 100644 index 000000000000..1f1abdb1cb17 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs @@ -0,0 +1,50 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use super::{utils, ShareFsMount, ShareFsMountResult, ShareFsRootfsConfig, ShareFsVolumeConfig}; + +pub struct VirtiofsShareMount { + id: String, +} + +impl VirtiofsShareMount { + pub fn new(id: &str) -> Self { + Self { id: id.to_string() } + } +} + +#[async_trait] +impl ShareFsMount for VirtiofsShareMount { + async fn share_rootfs(&self, config: ShareFsRootfsConfig) -> Result { + // TODO: select virtiofs or support nydus + let guest_path = utils::share_to_guest( + &config.source, + &config.target, + &self.id, + &config.cid, + config.readonly, + false, + ) + .context("share to guest")?; + Ok(ShareFsMountResult { guest_path }) + } + + async fn share_volume(&self, config: ShareFsVolumeConfig) -> Result { + let guest_path = utils::share_to_guest( + &config.source, + &config.target, + &self.id, + &config.cid, + config.readonly, + true, + ) + .context("share to guest")?; + Ok(ShareFsMountResult { guest_path }) + } +} diff --git a/src/runtime-rs/crates/resource/src/volume/block_volume.rs b/src/runtime-rs/crates/resource/src/volume/block_volume.rs new file mode 100644 index 000000000000..f015c927857a --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/block_volume.rs @@ -0,0 +1,37 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; + +use super::Volume; + +pub(crate) struct BlockVolume {} + +/// BlockVolume: block device volume +impl BlockVolume { + pub(crate) fn new(_m: &oci::Mount) -> Result { + Ok(Self {}) + } +} + +impl Volume for BlockVolume { + fn get_volume_mount(&self) -> anyhow::Result> { + todo!() + } + + fn get_storage(&self) -> Result> { + todo!() + } + + fn cleanup(&self) -> Result<()> { + todo!() + } +} + +pub(crate) fn is_block_volume(_m: &oci::Mount) -> bool { + // attach block device + false +} diff --git a/src/runtime-rs/crates/resource/src/volume/default_volume.rs b/src/runtime-rs/crates/resource/src/volume/default_volume.rs new file mode 100644 index 000000000000..3b7752a4e75c --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/default_volume.rs @@ -0,0 +1,36 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; + +use super::Volume; + +pub(crate) struct DefaultVolume { + mount: oci::Mount, +} + +/// DefaultVolume: passthrough the mount to guest +impl DefaultVolume { + pub fn new(mount: &oci::Mount) -> Result { + Ok(Self { + mount: mount.clone(), + }) + } +} + +impl Volume for DefaultVolume { + fn get_volume_mount(&self) -> anyhow::Result> { + Ok(vec![self.mount.clone()]) + } + + fn get_storage(&self) -> Result> { + Ok(vec![]) + } + + fn cleanup(&self) -> Result<()> { + todo!() + } +} diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs new file mode 100644 index 000000000000..2ad12f119608 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -0,0 +1,99 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod block_volume; +mod default_volume; +mod share_fs_volume; +mod shm_volume; + +use std::{sync::Arc, vec::Vec}; + +use anyhow::{Context, Result}; +use tokio::sync::RwLock; + +use crate::share_fs::ShareFs; + +pub trait Volume: Send + Sync { + fn get_volume_mount(&self) -> Result>; + fn get_storage(&self) -> Result>; + fn cleanup(&self) -> Result<()>; +} + +#[derive(Default)] +pub struct VolumeResourceInner { + volumes: Vec>, +} + +#[derive(Default)] +pub struct VolumeResource { + inner: Arc>, +} + +impl VolumeResource { + pub fn new() -> Self { + Self::default() + } + + pub async fn handler_volumes( + &self, + share_fs: &Option>, + cid: &str, + oci_mounts: &[oci::Mount], + ) -> Result>> { + let mut volumes: Vec> = vec![]; + for m in oci_mounts { + let volume: Arc = if shm_volume::is_shim_volume(m) { + let shm_size = shm_volume::DEFAULT_SHM_SIZE; + Arc::new( + shm_volume::ShmVolume::new(m, shm_size) + .context(format!("new shm volume {:?}", m))?, + ) + } else if share_fs_volume::is_share_fs_volume(m) { + Arc::new( + share_fs_volume::ShareFsVolume::new(share_fs, m, cid) + .await + .context(format!("new share fs volume {:?}", m))?, + ) + } else if block_volume::is_block_volume(m) { + Arc::new( + block_volume::BlockVolume::new(m) + .context(format!("new block volume {:?}", m))?, + ) + } else if is_skip_volume(m) { + info!(sl!(), "skip volume {:?}", m); + continue; + } else { + Arc::new( + default_volume::DefaultVolume::new(m) + .context(format!("new default volume {:?}", m))?, + ) + }; + + volumes.push(volume.clone()); + let mut inner = self.inner.write().await; + inner.volumes.push(volume); + } + + Ok(volumes) + } + + pub async fn dump(&self) { + let inner = self.inner.read().await; + for v in &inner.volumes { + info!( + sl!(), + "volume mount {:?}: count {}", + v.get_volume_mount(), + Arc::strong_count(v) + ); + } + } +} + +fn is_skip_volume(_m: &oci::Mount) -> bool { + // TODO: support volume check + false +} diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs new file mode 100644 index 000000000000..4fa6b341f183 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs @@ -0,0 +1,153 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{path::Path, sync::Arc}; + +use anyhow::{anyhow, Context, Result}; +use log::debug; +use nix::sys::stat::{stat, SFlag}; + +use super::Volume; +use crate::share_fs::{ShareFs, ShareFsVolumeConfig}; + +// copy file to container's rootfs if filesystem sharing is not supported, otherwise +// bind mount it in the shared directory. +// Ignore /dev, directories and all other device files. We handle +// only regular files in /dev. It does not make sense to pass the host +// device nodes to the guest. +// skip the volumes whose source had already set to guest share dir. +pub(crate) struct ShareFsVolume { + mounts: Vec, +} + +impl ShareFsVolume { + pub(crate) async fn new( + share_fs: &Option>, + m: &oci::Mount, + cid: &str, + ) -> Result { + let file_name = Path::new(&m.source).file_name().unwrap().to_str().unwrap(); + let file_name = generate_mount_path(cid, file_name); + + let mut volume = Self { mounts: vec![] }; + match share_fs { + None => { + let mut need_copy = false; + match stat(Path::new(&m.source)) { + Ok(stat) => { + // Ignore the mount if this is not a regular file (excludes + // directory, socket, device, ...) as it cannot be handled by + // a simple copy. But this should not be treated as an error, + // only as a limitation. + // golang implement: + // ModeType = ModeDir | ModeSymlink | ModeNamedPipe | ModeSocket | + // ModeDevice | ModeCharDevice | ModeIrregular + let file_type = SFlag::S_IFDIR + | SFlag::S_IFLNK + | SFlag::S_IFIFO + | SFlag::S_IFSOCK + | SFlag::S_IFCHR + | SFlag::S_IFREG; + if !file_type.contains(SFlag::from_bits_truncate(stat.st_mode)) { + debug!( + "Ignoring non-regular file as FS sharing not supported. mount: {:?}", + m + ); + return Ok(volume); + } + if SFlag::from_bits_truncate(stat.st_mode) != SFlag::S_IFDIR { + need_copy = true; + } + } + Err(err) => { + return Err(anyhow!(format!( + "failed to stat file {} {:?}", + &m.source, err + ))); + } + }; + + if need_copy { + // TODO: copy file + } + } + Some(share_fs) => { + let share_fs_mount = share_fs.get_share_fs_mount(); + let mount_result = share_fs_mount + .share_volume(ShareFsVolumeConfig { + cid: cid.to_string(), + source: m.source.clone(), + target: file_name, + readonly: false, + }) + .await + .context("share fs volume")?; + + volume.mounts.push(oci::Mount { + destination: m.destination.clone(), + r#type: "bind".to_string(), + source: mount_result.guest_path, + options: m.options.clone(), + }); + } + } + Ok(volume) + } +} + +impl Volume for ShareFsVolume { + fn get_volume_mount(&self) -> anyhow::Result> { + Ok(self.mounts.clone()) + } + + fn get_storage(&self) -> Result> { + Ok(vec![]) + } + + fn cleanup(&self) -> Result<()> { + todo!() + } +} + +pub(crate) fn is_share_fs_volume(m: &oci::Mount) -> bool { + m.r#type == "bind" && !is_host_device(&m.destination) +} + +fn is_host_device(dest: &str) -> bool { + if dest == "/dev" { + return true; + } + + if dest.starts_with("/dev") { + let src = match std::fs::canonicalize(dest) { + Err(_) => return false, + Ok(src) => src, + }; + + if src.is_file() { + return false; + } + + return true; + } + + false +} + +// Note, don't generate random name, attaching rafs depends on the predictable name. +// If template_mnt is passed, just use existed name in it +pub fn generate_mount_path(id: &str, file_name: &str) -> String { + let mut nid = String::from(id); + if nid.len() > 10 { + nid = nid.chars().take(10).collect(); + } + + let mut uid = uuid::Uuid::new_v4().to_string(); + let uid_vec: Vec<&str> = uid.splitn(2, '-').collect(); + uid = String::from(uid_vec[0]); + + format!("{}-{}-{}", nid, uid, file_name) +} diff --git a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs new file mode 100644 index 000000000000..e26fe19046bd --- /dev/null +++ b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs @@ -0,0 +1,105 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::Path; + +use anyhow::Result; + +use super::Volume; +use crate::share_fs::DEFAULT_KATA_GUEST_SANDBOX_DIR; + +pub const SHM_DIR: &str = "shm"; +// DEFAULT_SHM_SIZE is the default shm size to be used in case host +// IPC is used. +pub const DEFAULT_SHM_SIZE: u64 = 65536 * 1024; + +// KATA_EPHEMERAL_DEV_TYPE creates a tmpfs backed volume for sharing files between containers. +pub const KATA_EPHEMERAL_DEV_TYPE: &str = "ephemeral"; + +pub(crate) struct ShmVolume { + mount: oci::Mount, + storage: Option, +} + +impl ShmVolume { + pub(crate) fn new(m: &oci::Mount, shm_size: u64) -> Result { + let (storage, mount) = if shm_size > 0 { + // storage + let mount_path = Path::new(DEFAULT_KATA_GUEST_SANDBOX_DIR).join(SHM_DIR); + let mount_path = mount_path.to_str().unwrap(); + let option = format!("size={}", shm_size); + + let options = vec![ + String::from("noexec"), + String::from("nosuid"), + String::from("nodev"), + String::from("mode=1777"), + option, + ]; + + let storage = agent::Storage { + driver: String::from(KATA_EPHEMERAL_DEV_TYPE), + driver_options: Vec::new(), + source: String::from("shm"), + fs_type: String::from("tmpfs"), + options, + mount_point: mount_path.to_string(), + }; + + // mount + let mount = oci::Mount { + r#type: "bind".to_string(), + destination: m.destination.clone(), + source: mount_path.to_string(), + options: vec!["rbind".to_string()], + }; + + (Some(storage), mount) + } else { + let mount = oci::Mount { + r#type: "tmpfs".to_string(), + destination: m.destination.clone(), + source: "shm".to_string(), + options: vec![ + "noexec", + "nosuid", + "nodev", + "mode=1777", + &format!("size={}", DEFAULT_SHM_SIZE), + ] + .iter() + .map(|s| s.to_string()) + .collect(), + }; + (None, mount) + }; + + Ok(Self { storage, mount }) + } +} + +impl Volume for ShmVolume { + fn get_volume_mount(&self) -> anyhow::Result> { + Ok(vec![self.mount.clone()]) + } + + fn get_storage(&self) -> Result> { + let s = if let Some(s) = self.storage.as_ref() { + vec![s.clone()] + } else { + vec![] + }; + Ok(s) + } + + fn cleanup(&self) -> Result<()> { + todo!() + } +} + +pub(crate) fn is_shim_volume(m: &oci::Mount) -> bool { + m.destination == "/dev/shm" && m.r#type != KATA_EPHEMERAL_DEV_TYPE +} From 9887272db93035eebdbc3c26e67adb1198be5eb1 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Sat, 26 Mar 2022 17:33:41 +0800 Subject: [PATCH 0046/1953] libs: enhance kata-sys-util and kata-types Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/agent/Cargo.lock | 80 +++++++++++++++--- src/libs/Cargo.lock | 82 ++++++++++++++++--- src/libs/kata-sys-util/Cargo.toml | 2 + src/libs/kata-sys-util/src/lib.rs | 1 + src/libs/kata-sys-util/src/mount.rs | 1 + src/libs/kata-sys-util/src/rand/mod.rs | 10 +++ .../kata-sys-util/src/rand/random_bytes.rs | 51 ++++++++++++ src/libs/kata-sys-util/src/rand/uuid.rs | 69 ++++++++++++++++ .../kata-types/src/config/hypervisor/mod.rs | 24 +++--- .../src/config/hypervisor/vendor.rs | 2 +- src/libs/kata-types/src/config/runtime.rs | 4 + src/libs/protocols/Cargo.toml | 2 +- 12 files changed, 293 insertions(+), 35 deletions(-) create mode 100644 src/libs/kata-sys-util/src/rand/mod.rs create mode 100644 src/libs/kata-sys-util/src/rand/random_bytes.rs create mode 100644 src/libs/kata-sys-util/src/rand/uuid.rs diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 3f1b4df85dc2..b2eaf5a7c0c1 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -325,7 +325,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" dependencies = [ "lazy_static", "log", - "rand", + "rand 0.8.4", ] [[package]] @@ -446,6 +446,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.3" @@ -454,7 +465,7 @@ checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -636,6 +647,7 @@ dependencies = [ name = "kata-sys-util" version = "0.1.0" dependencies = [ + "byteorder", "cgroups-rs", "chrono", "common-path", @@ -646,6 +658,7 @@ dependencies = [ "nix 0.23.1", "oci", "once_cell", + "rand 0.7.3", "serde_json", "slog", "slog-scope", @@ -962,7 +975,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project", - "rand", + "rand 0.8.4", "serde", "thiserror", "tokio", @@ -1279,6 +1292,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + [[package]] name = "rand" version = "0.8.4" @@ -1286,9 +1312,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -1298,7 +1334,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -1307,7 +1352,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom", + "getrandom 0.2.3", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -1316,7 +1370,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core", + "rand_core 0.6.3", ] [[package]] @@ -1664,7 +1718,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand", + "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -1986,6 +2040,12 @@ dependencies = [ "tokio-vsock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 0917f9552daa..4b83b3b4ffe8 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -156,7 +156,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011" dependencies = [ "lazy_static", "log", - "rand", + "rand 0.8.5", ] [[package]] @@ -263,6 +263,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.6" @@ -351,6 +362,7 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" name = "kata-sys-util" version = "0.1.0" dependencies = [ + "byteorder", "cgroups-rs", "chrono", "common-path", @@ -362,6 +374,7 @@ dependencies = [ "num_cpus", "oci", "once_cell", + "rand 0.7.3", "serde_json", "serial_test", "slog", @@ -657,9 +670,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.27.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" +checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" dependencies = [ "serde", "serde_derive", @@ -667,18 +680,18 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "2.27.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" +checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c" dependencies = [ "protobuf", ] [[package]] name = "protobuf-codegen-pure" -version = "2.27.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" +checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440" dependencies = [ "protobuf", "protobuf-codegen", @@ -706,6 +719,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -713,8 +739,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -724,7 +760,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -733,7 +778,16 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom", + "getrandom 0.2.6", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -1104,6 +1158,12 @@ dependencies = [ "nix", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 524a9b3bb16c..7eba88a420f9 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -11,6 +11,7 @@ license = "Apache-2.0" edition = "2018" [dependencies] +byteorder = "~1" cgroups = { package = "cgroups-rs", version = "0.2.7" } chrono = "0.4.0" common-path = "=1.0.0" @@ -23,6 +24,7 @@ serde_json = "1.0.73" slog = "2.5.2" slog-scope = "4.4.0" subprocess = "0.2.8" +rand = "^0.7.2" thiserror = "1.0.30" kata-types = { path = "../kata-types" } diff --git a/src/libs/kata-sys-util/src/lib.rs b/src/libs/kata-sys-util/src/lib.rs index 656a7c666cbd..2c90adb7c434 100644 --- a/src/libs/kata-sys-util/src/lib.rs +++ b/src/libs/kata-sys-util/src/lib.rs @@ -12,6 +12,7 @@ pub mod hooks; pub mod k8s; pub mod mount; pub mod numa; +pub mod rand; pub mod spec; pub mod validate; diff --git a/src/libs/kata-sys-util/src/mount.rs b/src/libs/kata-sys-util/src/mount.rs index 2831ff96dd66..c659fb62806f 100644 --- a/src/libs/kata-sys-util/src/mount.rs +++ b/src/libs/kata-sys-util/src/mount.rs @@ -262,6 +262,7 @@ pub fn bind_mount_unchecked, D: AsRef>( .canonicalize() .map_err(|_e| Error::InvalidPath(src.to_path_buf()))?; + create_mount_destination(src, dst, "/", "bind")?; // Bind mount `src` to `dst`. mount( Some(&abs_src), diff --git a/src/libs/kata-sys-util/src/rand/mod.rs b/src/libs/kata-sys-util/src/rand/mod.rs new file mode 100644 index 000000000000..adc098ff6865 --- /dev/null +++ b/src/libs/kata-sys-util/src/rand/mod.rs @@ -0,0 +1,10 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod random_bytes; +pub use random_bytes::RandomBytes; +mod uuid; +pub use uuid::UUID; diff --git a/src/libs/kata-sys-util/src/rand/random_bytes.rs b/src/libs/kata-sys-util/src/rand/random_bytes.rs new file mode 100644 index 000000000000..44f792962053 --- /dev/null +++ b/src/libs/kata-sys-util/src/rand/random_bytes.rs @@ -0,0 +1,51 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fmt; + +use rand::RngCore; + +pub struct RandomBytes { + pub bytes: Vec, +} + +impl RandomBytes { + pub fn new(n: usize) -> Self { + let mut bytes = vec![0u8; n]; + rand::thread_rng().fill_bytes(&mut bytes); + Self { bytes } + } +} + +impl fmt::LowerHex for RandomBytes { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for byte in &self.bytes { + write!(f, "{:x}", byte)?; + } + Ok(()) + } +} + +impl fmt::UpperHex for RandomBytes { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for byte in &self.bytes { + write!(f, "{:X}", byte)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn random_bytes() { + let b = RandomBytes::new(16); + assert_eq!(b.bytes.len(), 16); + println!("{:?}", b.bytes); + } +} diff --git a/src/libs/kata-sys-util/src/rand/uuid.rs b/src/libs/kata-sys-util/src/rand/uuid.rs new file mode 100644 index 000000000000..a257c9480390 --- /dev/null +++ b/src/libs/kata-sys-util/src/rand/uuid.rs @@ -0,0 +1,69 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{convert::From, fmt}; + +use byteorder::{BigEndian, ByteOrder}; +use rand::RngCore; + +pub struct UUID([u8; 16]); + +impl Default for UUID { + fn default() -> Self { + Self::new() + } +} + +impl UUID { + pub fn new() -> Self { + let mut b = [0u8; 16]; + rand::thread_rng().fill_bytes(&mut b); + b[6] = (b[6] & 0x0f) | 0x40; + b[8] = (b[8] & 0x3f) | 0x80; + Self(b) + } +} + +impl From<&UUID> for String { + fn from(from: &UUID) -> Self { + let time_low = BigEndian::read_u32(&from.0[..4]); + let time_mid = BigEndian::read_u16(&from.0[4..6]); + let time_hi = BigEndian::read_u16(&from.0[6..8]); + let clk_seq_hi = from.0[8]; + let clk_seq_low = from.0[9]; + let mut buf = [0u8; 8]; + buf[2..].copy_from_slice(&from.0[10..]); + let node = BigEndian::read_u64(&buf); + + format!( + "{:08x}-{:04x}-{:04x}-{:02x}{:02x}-{:012x}", + time_low, time_mid, time_hi, clk_seq_hi, clk_seq_low, node + ) + } +} + +impl fmt::Display for UUID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", String::from(self)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_uuid() { + let uuid = UUID::new(); + let sss: String = String::from(&uuid); + println!("{}", sss); + + let uuid2 = UUID([0u8, 1u8, 2u8, 3u8, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let sss2 = String::from(&uuid2); + println!("Display: {}", uuid2); + assert_eq!(&sss2, "00010203-0405-0607-0809-0a0b0c0d0e0f"); + } +} diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 9e0c83445b6f..34a8e4c19fcd 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -68,7 +68,7 @@ pub fn get_hypervisor_plugin(name: &str) -> Option> { } /// Configuration information for block device. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct BlockDeviceInfo { /// Disable block device from being used for a container's rootfs. /// @@ -194,7 +194,7 @@ impl BlockDeviceInfo { } /// Guest kernel boot information. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct BootInfo { /// Path to guest kernel file on host #[serde(default)] @@ -245,7 +245,7 @@ impl BootInfo { } /// Virtual CPU configuration information. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct CpuInfo { /// CPU features, comma-separated list of cpu features to pass to the cpu. /// For example, `cpu_features = "pmu=off,vmx=off" @@ -321,7 +321,7 @@ impl CpuInfo { } /// Configuration information for shared filesystem, such virtio-9p and virtio-fs. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct DebugInfo { /// This option changes the default hypervisor and kernel parameters to enable debug output /// where available. @@ -358,7 +358,7 @@ impl DebugInfo { } /// Virtual machine device configuration information. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct DeviceInfo { /// Bridges can be used to hot plug devices. /// @@ -427,7 +427,7 @@ impl DeviceInfo { } /// Configuration information for virtual machine. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct MachineInfo { /// Virtual machine model/type. #[serde(default)] @@ -495,7 +495,7 @@ impl MachineInfo { } /// Virtual machine memory configuration information. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct MemoryInfo { /// Default memory size in MiB for SB/VM. #[serde(default)] @@ -597,7 +597,7 @@ impl MemoryInfo { } /// Configuration information for virtual machine. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct NetworkInfo { /// If vhost-net backend for virtio-net is not desired, set to true. /// @@ -635,7 +635,7 @@ impl NetworkInfo { } /// Configuration information for virtual machine. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct SecurityInfo { /// Enable running QEMU VMM as a non-root user. /// @@ -721,7 +721,7 @@ impl SecurityInfo { } /// Configuration information for shared filesystem, such virtio-9p and virtio-fs. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct SharedFsInfo { /// Shared file system type: /// - virtio-fs (default) @@ -862,7 +862,7 @@ impl SharedFsInfo { } /// Common configuration information for hypervisors. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct Hypervisor { /// Path to the hypervisor executable. #[serde(default)] @@ -1035,7 +1035,7 @@ mod vendor { use super::*; /// Vendor customization runtime configuration. - #[derive(Debug, Default, Deserialize, Serialize)] + #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct HypervisorVendor {} impl ConfigOps for HypervisorVendor {} diff --git a/src/libs/kata-types/src/config/hypervisor/vendor.rs b/src/libs/kata-types/src/config/hypervisor/vendor.rs index 9b51d1016564..39f5779a451a 100644 --- a/src/libs/kata-types/src/config/hypervisor/vendor.rs +++ b/src/libs/kata-types/src/config/hypervisor/vendor.rs @@ -8,7 +8,7 @@ use super::*; /// Vendor customization runtime configuration. -#[derive(Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct HypervisorVendor {} impl ConfigOps for HypervisorVendor {} diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 75b25f166d4c..a39b71220591 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -17,6 +17,10 @@ pub struct Runtime { #[serde(default)] pub name: String, + /// Hypervisor name: Plan to support dragonball, qemu + #[serde(default)] + pub hypervisor_name: String, + /// If enabled, the runtime will log additional debug messages to the system log. #[serde(default, rename = "enable_debug")] pub debug: bool, diff --git a/src/libs/protocols/Cargo.toml b/src/libs/protocols/Cargo.toml index eda2eeffc7da..cf2559b2e0e4 100644 --- a/src/libs/protocols/Cargo.toml +++ b/src/libs/protocols/Cargo.toml @@ -12,7 +12,7 @@ async = ["ttrpc/async", "async-trait"] [dependencies] ttrpc = { version = "0.6.0" } async-trait = { version = "0.1.42", optional = true } -protobuf = { version = "2.23.0", features = ["with-serde"] } +protobuf = { version = "=2.14.0", features = ["with-serde"] } serde = { version = "1.0.130", features = ["derive"], optional = true } serde_json = { version = "1.0.68", optional = true } oci = { path = "../oci" } From 10343b1f3d1afdc376cb4e6eddf44bc98d7c0989 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Sat, 26 Mar 2022 17:33:41 +0800 Subject: [PATCH 0047/1953] runtime-rs: enhance runtimes 1. support oom event 2. use ContainerProcess to store container_id and exec_id 3. support stats Fixes: #3785 Signed-off-by: Zhongtao Hu --- src/runtime-rs/Cargo.lock | 158 +++++++++---- src/runtime-rs/crates/agent/Cargo.toml | 1 + src/runtime-rs/crates/agent/src/kata/agent.rs | 13 +- src/runtime-rs/crates/agent/src/kata/mod.rs | 6 + src/runtime-rs/crates/agent/src/kata/trans.rs | 32 +-- src/runtime-rs/crates/agent/src/lib.rs | 27 +-- .../crates/agent/src/sock/hybrid_vsock.rs | 2 +- src/runtime-rs/crates/agent/src/sock/mod.rs | 1 + src/runtime-rs/crates/agent/src/types.rs | 47 ++-- .../crates/runtimes/common/Cargo.toml | 1 + .../runtimes/common/src/container_manager.rs | 2 +- .../crates/runtimes/common/src/message.rs | 30 ++- .../crates/runtimes/common/src/sandbox.rs | 4 +- .../crates/runtimes/common/src/types/mod.rs | 22 +- .../common/src/types/trans_from_agent.rs | 214 ++++++++++++++++++ .../common/src/types/trans_from_shim.rs | 6 +- .../common/src/types/trans_into_agent.rs | 28 +++ src/runtime-rs/crates/runtimes/src/manager.rs | 96 +++++--- src/runtime-rs/crates/service/src/manager.rs | 104 ++++++++- src/runtime-rs/crates/shim/src/shim.rs | 10 +- src/runtime-rs/crates/shim/src/shim_delete.rs | 11 + src/runtime-rs/crates/shim/src/shim_run.rs | 17 +- src/runtime-rs/crates/shim/src/shim_start.rs | 17 +- 23 files changed, 674 insertions(+), 175 deletions(-) create mode 100644 src/runtime-rs/crates/runtimes/common/src/types/trans_from_agent.rs create mode 100644 src/runtime-rs/crates/runtimes/common/src/types/trans_into_agent.rs diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index f9e199efc87b..2533ab1ebb4b 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -26,6 +26,7 @@ dependencies = [ "futures 0.1.31", "kata-types", "log", + "logging", "oci", "protobuf", "protocols", @@ -61,9 +62,9 @@ checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ "proc-macro2", "quote", @@ -93,9 +94,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" @@ -170,7 +171,7 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time 0.1.43", + "time 0.1.44", "winapi", ] @@ -178,6 +179,7 @@ dependencies = [ name = "common" version = "0.1.0" dependencies = [ + "agent", "anyhow", "async-trait", "containerd-shim-protos", @@ -434,13 +436,24 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] @@ -463,9 +476,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" [[package]] name = "git2" -version = "0.14.2" +version = "0.13.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3826a6e0e2215d7a41c2bfc7c9244123969273f3476b939a226aac0ab56e9e3c" +checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" dependencies = [ "bitflags", "libc", @@ -547,9 +560,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown", @@ -601,6 +614,7 @@ dependencies = [ name = "kata-sys-util" version = "0.1.0" dependencies = [ + "byteorder", "cgroups-rs", "chrono", "common-path", @@ -611,6 +625,7 @@ dependencies = [ "nix 0.23.1", "oci", "once_cell", + "rand 0.7.3", "serde_json", "slog", "slog-scope", @@ -643,15 +658,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" [[package]] name = "libgit2-sys" -version = "0.13.2+1.4.2" +version = "0.12.26+1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a42de9a51a5c12e00fc0e4ca6bc2ea43582fc6418488e8f615e905d886f258b" +checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" dependencies = [ "cc", "libc", @@ -683,18 +698,19 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ "cfg-if 1.0.0", ] @@ -929,9 +945,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "ppv-lite86" @@ -965,9 +981,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] @@ -1065,9 +1081,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1095,6 +1111,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -1102,10 +1131,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.3", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -1131,13 +1170,31 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom", + "getrandom 0.2.6", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -1151,9 +1208,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] @@ -1260,9 +1317,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "semver" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" [[package]] name = "serde" @@ -1389,9 +1446,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "slog" @@ -1413,14 +1470,14 @@ dependencies = [ [[package]] name = "slog-json" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f825ce7346f40aa318111df5d3a94945a7fdca9081584cb9b05692fb3dfcb4" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" dependencies = [ "serde", "serde_json", "slog", - "time 0.3.7", + "time 0.3.9", ] [[package]] @@ -1436,9 +1493,9 @@ dependencies = [ [[package]] name = "slog-stdlog" -version = "4.1.0" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8228ab7302adbf4fcb37e66f3cda78003feb521e7fd9e3847ec117a7784d0f5a" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" dependencies = [ "log", "slog", @@ -1495,9 +1552,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" dependencies = [ "proc-macro2", "quote", @@ -1562,19 +1619,20 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] [[package]] name = "time" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ "itoa", "libc", @@ -1815,9 +1873,15 @@ dependencies = [ [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml index bd17f82d0053..7fde8bf85ce1 100644 --- a/src/runtime-rs/crates/agent/Cargo.toml +++ b/src/runtime-rs/crates/agent/Cargo.toml @@ -21,6 +21,7 @@ tokio = { version = "1.8.0", features = ["fs", "rt"] } url = "2.2.2" kata-types = { path = "../../../libs/kata-types"} +logging = { path = "../../../libs/logging"} oci = { path = "../../../libs/oci" } protocols = { path = "../../../libs/protocols", features=["async"] } diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index 9f8b4304d3e2..90a812d44167 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -20,14 +20,11 @@ fn new_ttrpc_ctx(timeout: i64) -> ttrpc_ctx::Context { #[async_trait] impl AgentManager for KataAgent { - async fn set_socket_address(&self, address: &str) -> Result<()> { - let mut inner = self.inner.lock().await; - inner.socket_address = address.to_string(); - Ok(()) - } - - async fn start(&self) -> Result<()> { - info!(sl!(), "begin to connect agent"); + async fn start(&self, address: &str) -> Result<()> { + info!(sl!(), "begin to connect agent {:?}", address); + self.set_socket_address(address) + .await + .context("set socket")?; self.connect_agent_server() .await .context("connect agent server")?; diff --git a/src/runtime-rs/crates/agent/src/kata/mod.rs b/src/runtime-rs/crates/agent/src/kata/mod.rs index 043b9aa14e5b..dd7831d35b94 100644 --- a/src/runtime-rs/crates/agent/src/kata/mod.rs +++ b/src/runtime-rs/crates/agent/src/kata/mod.rs @@ -82,6 +82,12 @@ impl KataAgent { }) } + pub(crate) async fn set_socket_address(&self, address: &str) -> Result<()> { + let mut inner = self.inner.lock().await; + inner.socket_address = address.to_string(); + Ok(()) + } + pub(crate) async fn connect_agent_server(&self) -> Result<()> { let mut inner = self.inner.lock().await; diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index c8e4dbca7c5a..033f0bd9796b 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -254,8 +254,8 @@ impl From for Routes { impl From for agent::CreateContainerRequest { fn from(from: CreateContainerRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), string_user: from_option(from.string_user), devices: from_vec(from.devices), storages: from_vec(from.storages), @@ -321,8 +321,8 @@ impl From for agent::ResumeContainerRequest { impl From for agent::SignalProcessRequest { fn from(from: SignalProcessRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), signal: from.signal, unknown_fields: Default::default(), cached_size: Default::default(), @@ -333,8 +333,8 @@ impl From for agent::SignalProcessRequest { impl From for agent::WaitProcessRequest { fn from(from: WaitProcessRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), unknown_fields: Default::default(), cached_size: Default::default(), } @@ -355,8 +355,8 @@ impl From for agent::UpdateContainerRequest { impl From for agent::WriteStreamRequest { fn from(from: WriteStreamRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), data: from.data, unknown_fields: Default::default(), cached_size: Default::default(), @@ -373,8 +373,8 @@ impl From for WriteStreamResponse { impl From for agent::ExecProcessRequest { fn from(from: ExecProcessRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), string_user: from_option(from.string_user), process: from_option(from.process), unknown_fields: Default::default(), @@ -523,8 +523,8 @@ impl From for StatsContainerResponse { impl From for agent::ReadStreamRequest { fn from(from: ReadStreamRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), len: from.len, unknown_fields: Default::default(), cached_size: Default::default(), @@ -541,8 +541,8 @@ impl From for ReadStreamResponse { impl From for agent::CloseStdinRequest { fn from(from: CloseStdinRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), unknown_fields: Default::default(), cached_size: Default::default(), } @@ -552,8 +552,8 @@ impl From for agent::CloseStdinRequest { impl From for agent::TtyWinResizeRequest { fn from(from: TtyWinResizeRequest) -> Self { Self { - container_id: from.container_id, - exec_id: from.exec_id, + container_id: from.process_id.container_id(), + exec_id: from.process_id.exec_id(), row: from.row, column: from.column, unknown_fields: Default::default(), diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index a9d8a15f923c..9c72a76847e4 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -7,11 +7,7 @@ #[macro_use] extern crate slog; -macro_rules! sl { - () => { - slog_scope::logger().new(slog::o!("subsystem" => "agent")) - }; -} +logging::logger_with_subsystem!(sl, "agent"); pub mod kata; mod log_forwarder; @@ -19,14 +15,15 @@ mod sock; mod types; pub use types::{ ARPNeighbor, ARPNeighbors, AddArpNeighborRequest, BlkioStatsEntry, CheckRequest, - CloseStdinRequest, ContainerID, CopyFileRequest, CreateContainerRequest, CreateSandboxRequest, - Empty, ExecProcessRequest, GetGuestDetailsRequest, GuestDetailsResponse, HealthCheckResponse, - IPAddress, IPFamily, Interface, Interfaces, ListProcessesRequest, MemHotplugByProbeRequest, - OnlineCPUMemRequest, OomEventResponse, ReadStreamRequest, ReadStreamResponse, - RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, SetGuestDateTimeRequest, - SignalProcessRequest, StatsContainerResponse, Storage, TtyWinResizeRequest, - UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, - WaitProcessRequest, WaitProcessResponse, WriteStreamRequest, WriteStreamResponse, + CloseStdinRequest, ContainerID, ContainerProcessID, CopyFileRequest, CreateContainerRequest, + CreateSandboxRequest, Empty, ExecProcessRequest, GetGuestDetailsRequest, GuestDetailsResponse, + HealthCheckResponse, IPAddress, IPFamily, Interface, Interfaces, ListProcessesRequest, + MemHotplugByProbeRequest, OnlineCPUMemRequest, OomEventResponse, ReadStreamRequest, + ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, + SetGuestDateTimeRequest, SignalProcessRequest, StatsContainerResponse, Storage, + TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, + VersionCheckResponse, WaitProcessRequest, WaitProcessResponse, WriteStreamRequest, + WriteStreamResponse, }; use anyhow::Result; @@ -34,8 +31,7 @@ use async_trait::async_trait; #[async_trait] pub trait AgentManager: Send + Sync { - async fn set_socket_address(&self, address: &str) -> Result<()>; - async fn start(&self) -> Result<()>; + async fn start(&self, address: &str) -> Result<()>; async fn stop(&self); } @@ -57,6 +53,7 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { async fn list_routes(&self, req: Empty) -> Result; async fn update_interface(&self, req: UpdateInterfaceRequest) -> Result; async fn update_routes(&self, req: UpdateRoutesRequest) -> Result; + // container async fn create_container(&self, req: CreateContainerRequest) -> Result; async fn pause_container(&self, req: ContainerID) -> Result; diff --git a/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs b/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs index 7079ea392e4d..59e93a64d247 100644 --- a/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs +++ b/src/runtime-rs/crates/agent/src/sock/hybrid_vsock.rs @@ -56,7 +56,7 @@ impl Sock for HybridVsock { } } } - Err(anyhow!("cannot connect to agent ttrpc server")) + Err(anyhow!("cannot connect to agent ttrpc server {:?}", config)) } } diff --git a/src/runtime-rs/crates/agent/src/sock/mod.rs b/src/runtime-rs/crates/agent/src/sock/mod.rs index 3ec4da6aa9b1..371f62cd445f 100644 --- a/src/runtime-rs/crates/agent/src/sock/mod.rs +++ b/src/runtime-rs/crates/agent/src/sock/mod.rs @@ -79,6 +79,7 @@ impl AsyncRead for Stream { } /// Connect config +#[derive(Debug)] pub struct ConnectConfig { dial_timeout_ms: u64, reconnect_timeout_ms: u64, diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index 41e0e3ee36ff..caf507c9ae90 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -97,8 +97,7 @@ pub struct Routes { #[derive(PartialEq, Clone, Default)] pub struct CreateContainerRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, pub string_user: Option, pub devices: Vec, pub storages: Vec, @@ -121,6 +120,29 @@ impl ContainerID { } } +#[derive(PartialEq, Clone, Default)] +pub struct ContainerProcessID { + pub container_id: ContainerID, + pub exec_id: String, +} + +impl ContainerProcessID { + pub fn new(container_id: &str, exec_id: &str) -> Self { + Self { + container_id: ContainerID::new(container_id), + exec_id: exec_id.to_string(), + } + } + + pub fn container_id(&self) -> String { + self.container_id.container_id.clone() + } + + pub fn exec_id(&self) -> String { + self.exec_id.clone() + } +} + #[derive(PartialEq, Clone, Debug, Default)] pub struct RemoveContainerRequest { pub container_id: String, @@ -138,15 +160,13 @@ impl RemoveContainerRequest { #[derive(PartialEq, Clone, Default)] pub struct SignalProcessRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, pub signal: u32, } #[derive(PartialEq, Clone, Default)] pub struct WaitProcessRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, } #[derive(PartialEq, Clone, Default)] @@ -165,8 +185,7 @@ pub struct UpdateContainerRequest { #[derive(PartialEq, Clone, Default)] pub struct WriteStreamRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, pub data: Vec, } @@ -177,8 +196,7 @@ pub struct WriteStreamResponse { #[derive(PartialEq, Clone, Default)] pub struct ExecProcessRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, pub string_user: Option, pub process: Option, } @@ -297,8 +315,7 @@ pub struct WaitProcessResponse { #[derive(PartialEq, Clone, Default)] pub struct ReadStreamRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, pub len: u32, } @@ -309,14 +326,12 @@ pub struct ReadStreamResponse { #[derive(PartialEq, Clone, Default)] pub struct CloseStdinRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, } #[derive(PartialEq, Clone, Default)] pub struct TtyWinResizeRequest { - pub container_id: String, - pub exec_id: String, + pub process_id: ContainerProcessID, pub row: u32, pub column: u32, } diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index f2bff39c847d..bfa08452b7cf 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -21,6 +21,7 @@ thiserror = "^1.0" tokio = { version = "1.8.0", features = ["rt-multi-thread", "process", "fs"] } ttrpc = { version = "0.6.0" } +agent = { path = "../../agent" } kata-sys-util = { path = "../../../../libs/kata-sys-util" } kata-types = { path = "../../../../libs/kata-types" } oci = { path = "../../../../libs/oci" } diff --git a/src/runtime-rs/crates/runtimes/common/src/container_manager.rs b/src/runtime-rs/crates/runtimes/common/src/container_manager.rs index aeba770a660b..040b557ee641 100644 --- a/src/runtime-rs/crates/runtimes/common/src/container_manager.rs +++ b/src/runtime-rs/crates/runtimes/common/src/container_manager.rs @@ -16,7 +16,7 @@ use crate::types::{ #[async_trait] pub trait ContainerManager: Send + Sync { // container lifecycle - async fn create_container(&self, config: ContainerConfig) -> Result; + async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result; async fn pause_container(&self, container_id: &ContainerID) -> Result<()>; async fn resume_container(&self, container_id: &ContainerID) -> Result<()>; async fn stats_container(&self, container_id: &ContainerID) -> Result; diff --git a/src/runtime-rs/crates/runtimes/common/src/message.rs b/src/runtime-rs/crates/runtimes/common/src/message.rs index ff6ee960d71b..856a6e59900a 100644 --- a/src/runtime-rs/crates/runtimes/common/src/message.rs +++ b/src/runtime-rs/crates/runtimes/common/src/message.rs @@ -3,8 +3,10 @@ // // SPDX-License-Identifier: Apache-2.0 // +use std::sync::Arc; -use anyhow::Result; +use anyhow::{Context, Result}; +use containerd_shim_protos::{events::task::TaskOOM, protobuf::Message as ProtobufMessage}; use tokio::sync::mpsc::{channel, Receiver, Sender}; /// message receiver buffer size @@ -15,8 +17,12 @@ pub enum Action { Start, Stop, Shutdown, + Event(Arc), } +unsafe impl Send for Message {} +unsafe impl Sync for Message {} + #[derive(Debug)] pub struct Message { pub action: Action, @@ -42,3 +48,25 @@ impl Message { ) } } + +const TASK_OOM_EVENT_TOPIC: &str = "/tasks/oom"; + +pub trait Event: std::fmt::Debug + Send { + fn r#type(&self) -> String; + fn type_url(&self) -> String; + fn value(&self) -> Result>; +} + +impl Event for TaskOOM { + fn r#type(&self) -> String { + TASK_OOM_EVENT_TOPIC.to_string() + } + + fn type_url(&self) -> String { + "containerd.events.TaskOOM".to_string() + } + + fn value(&self) -> Result> { + self.write_to_bytes().context("get oom value") + } +} diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index 699fc1977a59..1b175204c5bb 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -7,9 +7,11 @@ use anyhow::Result; use async_trait::async_trait; +use kata_types::config::TomlConfig; + #[async_trait] pub trait Sandbox: Send + Sync { - async fn start(&self) -> Result<()>; + async fn start(&self, netns: Option, config: &TomlConfig) -> Result<()>; async fn stop(&self) -> Result<()>; async fn cleanup(&self, container_id: &str) -> Result<()>; async fn shutdown(&self) -> Result<()>; diff --git a/src/runtime-rs/crates/runtimes/common/src/types/mod.rs b/src/runtime-rs/crates/runtimes/common/src/types/mod.rs index e398735ff706..14f188d7d3d4 100644 --- a/src/runtime-rs/crates/runtimes/common/src/types/mod.rs +++ b/src/runtime-rs/crates/runtimes/common/src/types/mod.rs @@ -4,7 +4,9 @@ // SPDX-License-Identifier: Apache-2.0 // +mod trans_from_agent; mod trans_from_shim; +mod trans_into_agent; mod trans_into_shim; use std::fmt; @@ -69,6 +71,12 @@ pub struct ContainerID { pub container_id: String, } +impl ToString for ContainerID { + fn to_string(&self) -> String { + self.container_id.clone() + } +} + impl ContainerID { pub fn new(container_id: &str) -> Result { validate::verify_id(container_id).context("verify container id")?; @@ -105,6 +113,14 @@ impl ContainerProcess { process_type, }) } + + pub fn container_id(&self) -> &str { + &self.container_id.container_id + } + + pub fn exec_id(&self) -> &str { + &self.exec_id + } } #[derive(Debug, Clone)] pub struct ContainerConfig { @@ -130,7 +146,7 @@ impl PID { #[derive(Debug, Clone)] pub struct KillRequest { - pub process_id: ContainerProcess, + pub process: ContainerProcess, pub signal: u32, pub all: bool, } @@ -143,14 +159,14 @@ pub struct ShutdownRequest { #[derive(Debug, Clone)] pub struct ResizePTYRequest { - pub process_id: ContainerProcess, + pub process: ContainerProcess, pub width: u32, pub height: u32, } #[derive(Debug, Clone)] pub struct ExecProcessRequest { - pub process_id: ContainerProcess, + pub process: ContainerProcess, pub terminal: bool, pub stdin: Option, pub stdout: Option, diff --git a/src/runtime-rs/crates/runtimes/common/src/types/trans_from_agent.rs b/src/runtime-rs/crates/runtimes/common/src/types/trans_from_agent.rs new file mode 100644 index 000000000000..887777122348 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/types/trans_from_agent.rs @@ -0,0 +1,214 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::convert::From; + +use containerd_shim_protos::cgroups::metrics; +use protobuf::Message; + +use super::{StatsInfo, StatsInfoValue}; + +// TODO: trans from agent proto? +impl From> for StatsInfo { + fn from(c_stats: Option) -> Self { + let mut metric = metrics::Metrics::new(); + let stats = match c_stats { + None => { + return StatsInfo { value: None }; + } + Some(stats) => stats, + }; + + if let Some(cg_stats) = stats.cgroup_stats { + if let Some(cpu) = cg_stats.cpu_stats { + // set protobuf cpu stat + let mut p_cpu = metrics::CPUStat::new(); + if let Some(usage) = cpu.cpu_usage { + let mut p_usage = metrics::CPUUsage::new(); + p_usage.set_total(usage.total_usage); + p_usage.set_per_cpu(usage.percpu_usage); + p_usage.set_kernel(usage.usage_in_kernelmode); + p_usage.set_user(usage.usage_in_usermode); + + // set protobuf cpu usage + p_cpu.set_usage(p_usage); + } + + if let Some(throttle) = cpu.throttling_data { + let mut p_throttle = metrics::Throttle::new(); + p_throttle.set_periods(throttle.periods); + p_throttle.set_throttled_time(throttle.throttled_time); + p_throttle.set_throttled_periods(throttle.throttled_periods); + + // set protobuf cpu usage + p_cpu.set_throttling(p_throttle); + } + + metric.set_cpu(p_cpu); + } + + if let Some(m_stats) = cg_stats.memory_stats { + let mut p_m = metrics::MemoryStat::new(); + p_m.set_cache(m_stats.cache); + // memory usage + if let Some(m_data) = m_stats.usage { + let mut p_m_entry = metrics::MemoryEntry::new(); + p_m_entry.set_usage(m_data.usage); + p_m_entry.set_limit(m_data.limit); + p_m_entry.set_failcnt(m_data.failcnt); + p_m_entry.set_max(m_data.max_usage); + + p_m.set_usage(p_m_entry); + } + // memory swap_usage + if let Some(m_data) = m_stats.swap_usage { + let mut p_m_entry = metrics::MemoryEntry::new(); + p_m_entry.set_usage(m_data.usage); + p_m_entry.set_limit(m_data.limit); + p_m_entry.set_failcnt(m_data.failcnt); + p_m_entry.set_max(m_data.max_usage); + + p_m.set_swap(p_m_entry); + } + // memory kernel_usage + if let Some(m_data) = m_stats.kernel_usage { + let mut p_m_entry = metrics::MemoryEntry::new(); + p_m_entry.set_usage(m_data.usage); + p_m_entry.set_limit(m_data.limit); + p_m_entry.set_failcnt(m_data.failcnt); + p_m_entry.set_max(m_data.max_usage); + + p_m.set_kernel(p_m_entry); + } + + for (k, v) in m_stats.stats { + match k.as_str() { + "dirty" => p_m.set_dirty(v), + "rss" => p_m.set_rss(v), + "rss_huge" => p_m.set_rss_huge(v), + "mapped_file" => p_m.set_mapped_file(v), + "writeback" => p_m.set_writeback(v), + "pg_pg_in" => p_m.set_pg_pg_in(v), + "pg_pg_out" => p_m.set_pg_pg_out(v), + "pg_fault" => p_m.set_pg_fault(v), + "pg_maj_fault" => p_m.set_pg_maj_fault(v), + "inactive_file" => p_m.set_inactive_file(v), + "inactive_anon" => p_m.set_inactive_anon(v), + "active_file" => p_m.set_active_file(v), + "unevictable" => p_m.set_unevictable(v), + "hierarchical_memory_limit" => p_m.set_hierarchical_memory_limit(v), + "hierarchical_swap_limit" => p_m.set_hierarchical_swap_limit(v), + "total_cache" => p_m.set_total_cache(v), + "total_rss" => p_m.set_total_rss(v), + "total_mapped_file" => p_m.set_total_mapped_file(v), + "total_dirty" => p_m.set_total_dirty(v), + + "total_pg_pg_in" => p_m.set_total_pg_pg_in(v), + "total_pg_pg_out" => p_m.set_total_pg_pg_out(v), + "total_pg_fault" => p_m.set_total_pg_fault(v), + "total_pg_maj_fault" => p_m.set_total_pg_maj_fault(v), + "total_inactive_file" => p_m.set_total_inactive_file(v), + "total_inactive_anon" => p_m.set_total_inactive_anon(v), + "total_active_file" => p_m.set_total_active_file(v), + "total_unevictable" => p_m.set_total_unevictable(v), + _ => (), + } + } + metric.set_memory(p_m); + } + + if let Some(pid_stats) = cg_stats.pids_stats { + let mut p_pid = metrics::PidsStat::new(); + p_pid.set_limit(pid_stats.limit); + p_pid.set_current(pid_stats.current); + metric.set_pids(p_pid); + } + + if let Some(blk_stats) = cg_stats.blkio_stats { + let mut p_blk_stats = metrics::BlkIOStat::new(); + p_blk_stats + .set_io_serviced_recursive(copy_blkio_entry(&blk_stats.io_serviced_recursive)); + p_blk_stats.set_io_service_bytes_recursive(copy_blkio_entry( + &blk_stats.io_service_bytes_recursive, + )); + p_blk_stats + .set_io_queued_recursive(copy_blkio_entry(&blk_stats.io_queued_recursive)); + p_blk_stats.set_io_service_time_recursive(copy_blkio_entry( + &blk_stats.io_service_time_recursive, + )); + p_blk_stats.set_io_wait_time_recursive(copy_blkio_entry( + &blk_stats.io_wait_time_recursive, + )); + p_blk_stats + .set_io_merged_recursive(copy_blkio_entry(&blk_stats.io_merged_recursive)); + p_blk_stats.set_io_time_recursive(copy_blkio_entry(&blk_stats.io_time_recursive)); + p_blk_stats.set_sectors_recursive(copy_blkio_entry(&blk_stats.sectors_recursive)); + + metric.set_blkio(p_blk_stats); + } + + if !cg_stats.hugetlb_stats.is_empty() { + let mut p_huge = ::protobuf::RepeatedField::new(); + for (k, v) in cg_stats.hugetlb_stats { + let mut h = metrics::HugetlbStat::new(); + h.set_pagesize(k); + h.set_max(v.max_usage); + h.set_usage(v.usage); + h.set_failcnt(v.failcnt); + p_huge.push(h); + } + metric.set_hugetlb(p_huge); + } + } + + let net_stats = stats.network_stats; + if !net_stats.is_empty() { + let mut p_net = ::protobuf::RepeatedField::new(); + for v in net_stats.iter() { + let mut h = metrics::NetworkStat::new(); + h.set_name(v.name.clone()); + + h.set_tx_bytes(v.tx_bytes); + h.set_tx_packets(v.tx_packets); + h.set_tx_errors(v.tx_errors); + h.set_tx_dropped(v.tx_dropped); + + h.set_rx_bytes(v.rx_bytes); + h.set_rx_packets(v.rx_packets); + h.set_rx_errors(v.rx_errors); + h.set_rx_dropped(v.rx_dropped); + + p_net.push(h); + } + metric.set_network(p_net); + } + + StatsInfo { + value: Some(StatsInfoValue { + type_url: "io.containerd.cgroups.v1.Metrics".to_string(), + value: metric.write_to_bytes().unwrap(), + }), + } + } +} + +fn copy_blkio_entry( + entry: &[agent::BlkioStatsEntry], +) -> ::protobuf::RepeatedField { + let mut p_entry = ::protobuf::RepeatedField::new(); + + for e in entry.iter() { + let mut blk = metrics::BlkIOEntry::new(); + blk.set_op(e.op.clone()); + blk.set_value(e.value); + blk.set_major(e.major); + blk.set_minor(e.minor); + + p_entry.push(blk); + } + + p_entry +} diff --git a/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs b/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs index c26bb6828149..07f1f8d79e44 100644 --- a/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs +++ b/src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs @@ -82,7 +82,7 @@ impl TryFrom for Request { fn try_from(from: api::ExecProcessRequest) -> Result { let spec = from.get_spec(); Ok(Request::ExecProcess(ExecProcessRequest { - process_id: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + process: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, terminal: from.terminal, stdin: (!from.stdin.is_empty()).then(|| from.stdin.clone()), stdout: (!from.stdout.is_empty()).then(|| from.stdout.clone()), @@ -97,7 +97,7 @@ impl TryFrom for Request { type Error = anyhow::Error; fn try_from(from: api::KillRequest) -> Result { Ok(Request::KillProcess(KillRequest { - process_id: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + process: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, signal: from.signal, all: from.all, })) @@ -145,7 +145,7 @@ impl TryFrom for Request { type Error = anyhow::Error; fn try_from(from: api::ResizePtyRequest) -> Result { Ok(Request::ResizeProcessPTY(ResizePTYRequest { - process_id: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, + process: ContainerProcess::new(&from.id, &from.exec_id).context("new process id")?, width: from.width, height: from.height, })) diff --git a/src/runtime-rs/crates/runtimes/common/src/types/trans_into_agent.rs b/src/runtime-rs/crates/runtimes/common/src/types/trans_into_agent.rs new file mode 100644 index 000000000000..f032fd70bcf7 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/common/src/types/trans_into_agent.rs @@ -0,0 +1,28 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::convert::From; + +use agent; + +use super::{ContainerID, ContainerProcess}; + +impl From for agent::ContainerID { + fn from(from: ContainerID) -> Self { + Self { + container_id: from.container_id, + } + } +} + +impl From for agent::ContainerProcessID { + fn from(from: ContainerProcess) -> Self { + Self { + container_id: from.container_id.into(), + exec_id: from.exec_id, + } + } +} diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index f3006f8561b7..cfeab919be8e 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -25,7 +25,7 @@ use wasm_container::WasmContainer; struct RuntimeHandlerManagerInner { id: String, msg_sender: Sender, - runtime_instance: Option, + runtime_instance: Option>, } impl RuntimeHandlerManagerInner { @@ -37,26 +37,22 @@ impl RuntimeHandlerManagerInner { }) } - async fn init_runtime_handler(&mut self, runtime_name: &str) -> Result<()> { - info!(sl!(), "new runtime handler {}", runtime_name); - - let runtime_handler = match runtime_name { + async fn init_runtime_handler( + &mut self, + netns: Option, + config: &TomlConfig, + ) -> Result<()> { + info!(sl!(), "new runtime handler {}", &config.runtime.name); + let runtime_handler = match config.runtime.name.as_str() { #[cfg(feature = "linux")] - name if name == LinuxContainer::name() => { - LinuxContainer::init().context("init linux container")?; - LinuxContainer::new_handler() - } + name if name == LinuxContainer::name() => LinuxContainer::new_handler(), #[cfg(feature = "wasm")] - name if name == WasmContainer::name() => { - WasmContainer::init().context("init wasm container")?; - WasmContainer::new_handler() - } + name if name == WasmContainer::name() => WasmContainer::new_handler(), #[cfg(feature = "virt")] - name if name == VirtContainer::name() => { - VirtContainer::init().context("init virt container")?; + name if name == VirtContainer::name() || name.is_empty() => { VirtContainer::new_handler() } - _ => return Err(anyhow!("Unsupported runtime: {}", runtime_name)), + _ => return Err(anyhow!("Unsupported runtime: {}", &config.runtime.name)), }; let runtime_instance = runtime_handler .new_instance(&self.id, self.msg_sender.clone()) @@ -66,10 +62,10 @@ impl RuntimeHandlerManagerInner { // start sandbox runtime_instance .sandbox - .start() + .start(netns, config) .await .context("start sandbox")?; - self.runtime_instance = Some(runtime_instance); + self.runtime_instance = Some(Arc::new(runtime_instance)); Ok(()) } @@ -79,15 +75,39 @@ impl RuntimeHandlerManagerInner { return Ok(()); } + #[cfg(feature = "linux")] + LinuxContainer::init().context("init linux container")?; + #[cfg(feature = "wasm")] + WasmContainer::init().context("init wasm container")?; + #[cfg(feature = "virt")] + VirtContainer::init().context("init virt container")?; + + let netns = if let Some(linux) = &spec.linux { + let mut netns = None; + for ns in &linux.namespaces { + if ns.r#type.as_str() != oci::NETWORKNAMESPACE { + continue; + } + + if !ns.path.is_empty() { + netns = Some(ns.path.clone()); + break; + } + } + netns + } else { + None + }; + let config = load_config(spec).context("load config")?; - self.init_runtime_handler(&config.runtime.name) + self.init_runtime_handler(netns, &config) .await .context("init runtime handler")?; Ok(()) } - fn get_runtime_instance(&self) -> Option { + fn get_runtime_instance(&self) -> Option> { self.runtime_instance.clone() } } @@ -112,25 +132,35 @@ impl RuntimeHandlerManager { Ok(()) } + async fn get_runtime_instance(&self) -> Result> { + let inner = self.inner.read().await; + inner + .get_runtime_instance() + .ok_or_else(|| anyhow!("runtime not ready")) + } + + async fn try_init_runtime_instance(&self, spec: &oci::Spec) -> Result<()> { + let mut inner = self.inner.write().await; + inner.try_init(spec).await + } + pub async fn handler_message(&self, req: Request) -> Result { if let Request::CreateContainer(req) = req { // get oci spec let bundler_path = format!("{}/{}", req.bundle, oci::OCI_SPEC_CONFIG_FILE_NAME); let spec = oci::Spec::load(&bundler_path).context("load spec")?; - let mut inner = self.inner.write().await; - inner - .try_init(&spec) + self.try_init_runtime_instance(&spec) .await - .context("try init runtime handler")?; - - let instance = inner + .context("try init runtime instance")?; + let instance = self .get_runtime_instance() - .ok_or_else(|| anyhow!("runtime not ready"))?; + .await + .context("get runtime instance")?; let shim_pid = instance .container_manager - .create_container(req) + .create_container(req, spec) .await .context("create container")?; Ok(Response::CreateContainer(shim_pid)) @@ -140,12 +170,12 @@ impl RuntimeHandlerManager { } pub async fn handler_request(&self, req: Request) -> Result { - let inner = self.inner.read().await; - let instance = inner + let instance = self .get_runtime_instance() - .ok_or_else(|| anyhow!("runtime not ready"))?; - let sandbox = instance.sandbox; - let cm = instance.container_manager; + .await + .context("get runtime instance")?; + let sandbox = instance.sandbox.clone(); + let cm = instance.container_manager.clone(); match req { Request::CreateContainer(req) => Err(anyhow!("Unreachable request {:?}", req)), diff --git a/src/runtime-rs/crates/service/src/manager.rs b/src/runtime-rs/crates/service/src/manager.rs index dac1b2275f0f..d22cdf86f100 100644 --- a/src/runtime-rs/crates/service/src/manager.rs +++ b/src/runtime-rs/crates/service/src/manager.rs @@ -5,15 +5,24 @@ // use std::{ + fs, os::unix::io::{FromRawFd, RawFd}, + process::Stdio, sync::Arc, }; use anyhow::{Context, Result}; -use common::message::{Action, Message}; -use containerd_shim_protos::shim_async; +use common::message::{Action, Event, Message}; +use containerd_shim_protos::{ + protobuf::{well_known_types::Any, Message as ProtobufMessage}, + shim_async, +}; use runtimes::RuntimeHandlerManager; -use tokio::sync::mpsc::{channel, Receiver}; +use tokio::{ + io::AsyncWriteExt, + process::Command, + sync::mpsc::{channel, Receiver}, +}; use ttrpc::asynchronous::Server; use crate::task_service::TaskService; @@ -21,14 +30,66 @@ use crate::task_service::TaskService; /// message buffer size const MESSAGE_BUFFER_SIZE: usize = 8; +pub const KATA_PATH: &str = "/run/kata"; + pub struct ServiceManager { receiver: Option>, handler: Arc, task_server: Option, + binary: String, + address: String, + namespace: String, +} + +async fn send_event( + containerd_binary: String, + address: String, + namespace: String, + event: Arc, +) -> Result<()> { + let any = Any { + type_url: event.type_url(), + value: event.value().context("get event value")?, + ..Default::default() + }; + let data = any.write_to_bytes().context("write to any")?; + let mut child = Command::new(containerd_binary) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .args(&[ + "--address", + &address, + "publish", + "--topic", + &event.r#type(), + "--namespace", + &namespace, + ]) + .spawn() + .context("sawn cmd")?; + + let stdin = child.stdin.as_mut().context("failed to open stdin")?; + stdin + .write_all(&data) + .await + .context("failed to write to stdin")?; + let output = child + .wait_with_output() + .await + .context("failed to read stdout")?; + info!(sl!(), "get output: {:?}", output); + Ok(()) } impl ServiceManager { - pub async fn new(id: &str, task_server_fd: RawFd) -> Result { + pub async fn new( + id: &str, + containerd_binary: &str, + address: &str, + namespace: &str, + task_server_fd: RawFd, + ) -> Result { let (sender, receiver) = channel::(MESSAGE_BUFFER_SIZE); let handler = Arc::new( RuntimeHandlerManager::new(id, sender) @@ -41,13 +102,17 @@ impl ServiceManager { receiver: Some(receiver), handler, task_server: Some(task_server), + binary: containerd_binary.to_string(), + address: address.to_string(), + namespace: namespace.to_string(), }) } pub async fn run(&mut self) -> Result<()> { info!(sl!(), "begin to run service"); - self.start().await.context("start")?; + + info!(sl!(), "wait server message"); let mut rx = self.receiver.take(); if let Some(rx) = rx.as_mut() { while let Some(r) = rx.recv().await { @@ -59,9 +124,24 @@ impl ServiceManager { self.stop_listen().await.context("stop listen")?; break; } + Action::Event(event) => { + info!(sl!(), "get event {:?}", &event); + send_event( + self.binary.clone(), + self.address.clone(), + self.namespace.clone(), + event, + ) + .await + .context("send event")?; + Ok(()) + } }; if let Some(ref sender) = r.resp_sender { + if let Err(err) = result.as_ref() { + error!(sl!(), "failed to process action {:?}", err); + } sender.send(result).await.context("send response")?; } } @@ -72,8 +152,18 @@ impl ServiceManager { Ok(()) } - pub fn cleanup(id: &str) -> Result<()> { - RuntimeHandlerManager::cleanup(id) + pub fn cleanup(sid: &str) -> Result<()> { + let temp_dir = [KATA_PATH, sid].join("/"); + if std::fs::metadata(temp_dir.as_str()).is_ok() { + // try to remove dir and skip the result + fs::remove_dir_all(temp_dir) + .map_err(|err| { + warn!(sl!(), "failed to clean up sandbox tmp dir"); + err + }) + .ok(); + } + Ok(()) } async fn start(&mut self) -> Result<()> { diff --git a/src/runtime-rs/crates/shim/src/shim.rs b/src/runtime-rs/crates/shim/src/shim.rs index 298f6a3e00b3..83060f2b2292 100644 --- a/src/runtime-rs/crates/shim/src/shim.rs +++ b/src/runtime-rs/crates/shim/src/shim.rs @@ -60,10 +60,6 @@ impl ShimExecutor { data.parse::().context(Error::ParsePid) } - pub(crate) fn get_bundle_path(&self) -> Result { - std::env::current_dir().context(Error::GetBundlePath) - } - pub(crate) fn socket_address(&self, id: &str) -> Result { if id.is_empty() { return Err(anyhow!(Error::EmptySandboxId)); @@ -72,8 +68,6 @@ impl ShimExecutor { let data = [&self.args.address, &self.args.namespace, id].join("/"); let mut hasher = sha2::Sha256::new(); hasher.update(data); - - // Follow // https://github.com/containerd/containerd/blob/main/runtime/v2/shim/util_unix.go#L68 to // generate a shim socket path. Ok(PathBuf::from(format!( @@ -89,6 +83,8 @@ mod tests { use super::*; use serial_test::serial; + use kata_sys_util::spec::get_bundle_path; + #[test] #[serial] fn test_shim_executor() { @@ -111,7 +107,7 @@ mod tests { executor .write_address(bundle_path, Path::new("12345")) .unwrap(); - let dir = executor.get_bundle_path().unwrap(); + let dir = get_bundle_path().unwrap(); let file_path = &dir.join("address"); let buf = std::fs::read_to_string(file_path).unwrap(); assert_eq!(&buf, "12345"); diff --git a/src/runtime-rs/crates/shim/src/shim_delete.rs b/src/runtime-rs/crates/shim/src/shim_delete.rs index 57082d212925..fd907756620a 100644 --- a/src/runtime-rs/crates/shim/src/shim_delete.rs +++ b/src/runtime-rs/crates/shim/src/shim_delete.rs @@ -7,6 +7,7 @@ use anyhow::{Context, Result}; use containerd_shim_protos::api; use protobuf::Message; +use std::{fs, path::Path}; use crate::{shim::ShimExecutor, Error}; @@ -30,6 +31,16 @@ impl ShimExecutor { exited_time.set_seconds(seconds); rsp.set_exited_at(exited_time); + let address = self + .socket_address(&self.args.id) + .context("socket address")?; + let trim_path = address.strip_prefix("unix://").context("trim path")?; + let file_path = Path::new("/").join(trim_path); + let file_path = file_path.as_path(); + if std::fs::metadata(&file_path).is_ok() { + info!(sl!(), "remote socket path: {:?}", &file_path); + fs::remove_file(file_path).ok(); + } service::ServiceManager::cleanup(&self.args.id).context("cleanup")?; Ok(rsp) } diff --git a/src/runtime-rs/crates/shim/src/shim_run.rs b/src/runtime-rs/crates/shim/src/shim_run.rs index 112b83517966..cde365780ec3 100644 --- a/src/runtime-rs/crates/shim/src/shim_run.rs +++ b/src/runtime-rs/crates/shim/src/shim_run.rs @@ -7,6 +7,7 @@ use std::os::unix::io::RawFd; use anyhow::{Context, Result}; +use kata_sys_util::spec::get_bundle_path; use crate::{ logger, @@ -18,7 +19,7 @@ impl ShimExecutor { pub async fn run(&mut self) -> Result<()> { crate::panic_hook::set_panic_hook(); let sid = self.args.id.clone(); - let bundle_path = self.get_bundle_path().context("get bundle")?; + let bundle_path = get_bundle_path().context("get bundle")?; let path = bundle_path.join("log"); let _logger_guard = logger::set_logger(path.to_str().unwrap(), &sid, self.args.debug).context("set logger"); @@ -36,12 +37,18 @@ impl ShimExecutor { async fn do_run(&mut self) -> Result<()> { info!(sl!(), "start to run"); - self.args.validate(false).context("validata")?; + self.args.validate(false).context("validate")?; let server_fd = get_server_fd().context("get server fd")?; - let mut service_manager = service::ServiceManager::new(&self.args.id, server_fd) - .await - .context("new runtime server")?; + let mut service_manager = service::ServiceManager::new( + &self.args.id, + &self.args.publish_binary, + &self.args.address, + &self.args.namespace, + server_fd, + ) + .await + .context("new shim server")?; service_manager.run().await.context("run")?; Ok(()) diff --git a/src/runtime-rs/crates/shim/src/shim_start.rs b/src/runtime-rs/crates/shim/src/shim_start.rs index a84053209151..414e0ccf3f8d 100644 --- a/src/runtime-rs/crates/shim/src/shim_start.rs +++ b/src/runtime-rs/crates/shim/src/shim_start.rs @@ -12,6 +12,7 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; +use kata_sys_util::spec::get_bundle_path; use kata_types::{container::ContainerType, k8s}; use unix_socket::UnixListener; @@ -32,7 +33,7 @@ impl ShimExecutor { } fn do_start(&mut self) -> Result { - let bundle_path = self.get_bundle_path().context("get bundle path")?; + let bundle_path = get_bundle_path().context("get bundle path")?; let spec = self.load_oci_spec(&bundle_path)?; let (container_type, id) = k8s::container_type_with_id(&spec); @@ -66,7 +67,7 @@ impl ShimExecutor { return Err(anyhow!("invalid param")); } - let bundle_path = self.get_bundle_path().context("get bundle path")?; + let bundle_path = get_bundle_path().context("get bundle path")?; let self_exec = std::env::current_exe().map_err(Error::SelfExec)?; let mut command = std::process::Command::new(self_exec); @@ -109,7 +110,7 @@ impl ShimExecutor { fn get_shim_info_from_sandbox(&self, sandbox_id: &str) -> Result<(PathBuf, u32)> { // All containers of a pod share the same pod socket address. let address = self.socket_address(sandbox_id).context("socket address")?; - let bundle_path = self.get_bundle_path().context("get bundle path")?; + let bundle_path = get_bundle_path().context("get bundle path")?; let parent_bundle_path = Path::new(&bundle_path) .parent() .unwrap_or_else(|| Path::new("")); @@ -165,19 +166,13 @@ mod tests { let cmd = executor.new_command().unwrap(); assert_eq!(cmd.get_args().len(), 8); assert_eq!(cmd.get_envs().len(), 1); - assert_eq!( - cmd.get_current_dir().unwrap(), - executor.get_bundle_path().unwrap() - ); + assert_eq!(cmd.get_current_dir().unwrap(), get_bundle_path().unwrap()); executor.args.debug = true; let cmd = executor.new_command().unwrap(); assert_eq!(cmd.get_args().len(), 9); assert_eq!(cmd.get_envs().len(), 1); - assert_eq!( - cmd.get_current_dir().unwrap(), - executor.get_bundle_path().unwrap() - ); + assert_eq!(cmd.get_current_dir().unwrap(), get_bundle_path().unwrap()); } #[test] From 4be7185aa48f793428701b1b73f201e60b45d598 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Sat, 26 Mar 2022 17:33:41 +0800 Subject: [PATCH 0048/1953] runtime-rs: runtime part implement Fixes: #3785 Signed-off-by: Tim Zhang Signed-off-by: Zhongtao Hu Signed-off-by: Quanwei Zhou --- src/runtime-rs/Cargo.lock | 37 +- .../crates/runtimes/virt_container/Cargo.toml | 20 + .../src/container_manager/container.rs | 403 ++++++++++++++++++ .../src/container_manager/container_inner.rs | 261 ++++++++++++ .../src/container_manager/io/container_io.rs | 171 ++++++++ .../src/container_manager/io/mod.rs | 10 + .../src/container_manager/io/shim_io.rs | 135 ++++++ .../src/container_manager/manager.rs | 274 ++++++++++++ .../src/container_manager/mod.rs | 20 + .../src/container_manager/process.rs | 211 +++++++++ .../virt_container/src/health_check.rs | 123 ++++++ .../crates/runtimes/virt_container/src/lib.rs | 73 +++- .../runtimes/virt_container/src/sandbox.rs | 229 ++++++++++ 13 files changed, 1961 insertions(+), 6 deletions(-) create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/container_io.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/mod.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/shim_io.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/mod.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs create mode 100644 src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 2533ab1ebb4b..09abdaca5951 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -77,6 +77,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "awaitgroup" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc17ab023b4091c10ff099f9deebaeeb59b5189df07e554c4fef042b70745d68" + [[package]] name = "backtrace" version = "0.3.64" @@ -647,7 +653,7 @@ dependencies = [ "slog", "slog-scope", "thiserror", - "toml", + "toml 0.5.8", ] [[package]] @@ -1697,6 +1703,15 @@ dependencies = [ "vsock", ] +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.5.8" @@ -1848,11 +1863,31 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" name = "virt_container" version = "0.1.0" dependencies = [ + "agent", "anyhow", "async-trait", + "awaitgroup", "common", + "containerd-shim-protos", + "futures 0.3.21", + "hypervisor", + "kata-sys-util", "kata-types", + "lazy_static", + "libc", + "logging", + "nix 0.16.1", + "oci", + "protobuf", + "resource", + "serde", + "serde_derive", + "serde_json", + "slog", + "slog-scope", "tokio", + "toml 0.4.10", + "url", ] [[package]] diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml index 8e63c01c1ad1..e34e2fd5b720 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml @@ -7,7 +7,27 @@ edition = "2018" [dependencies] anyhow = "^1.0" async-trait = "0.1.48" +awaitgroup = "0.6.0" +containerd-shim-protos = { version = "0.2.0", features = ["async"]} +futures = "0.3.19" +lazy_static = "1.4.0" +libc = ">=0.2.39" +nix = "0.16.0" +protobuf = "2.23.0" +serde = { version = "1.0.100", features = ["derive"] } +serde_derive = "1.0.27" +serde_json = "1.0.39" +slog = "2.5.2" +slog-scope = "4.4.0" tokio = { version = "1.8.0" } +toml = "0.4.2" +url = "2.1.1" +agent = { path = "../../agent" } common = { path = "../common" } +hypervisor = { path = "../../hypervisor" } +kata-sys-util = { path = "../../../../libs/kata-sys-util" } kata-types = { path = "../../../../libs/kata-types" } +logging = { path = "../../../../libs/logging"} +oci = { path = "../../../../libs/oci" } +resource = { path = "../../resource" } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs new file mode 100644 index 000000000000..cb1c7b2b1cba --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -0,0 +1,403 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use agent::Agent; +use anyhow::{anyhow, Context, Result}; +use common::{ + error::Error, + types::{ + ContainerConfig, ContainerID, ContainerProcess, ProcessStateInfo, ProcessStatus, + ProcessType, + }, +}; +use oci::{LinuxResources, Process as OCIProcess}; +use resource::ResourceManager; +use tokio::sync::RwLock; + +use super::{ + process::{Process, ProcessWatcher}, + ContainerInner, +}; +use crate::container_manager::logger_with_process; + +pub struct Exec { + pub(crate) process: Process, + pub(crate) oci_process: OCIProcess, +} + +pub struct Container { + pid: u32, + pub container_id: ContainerID, + config: ContainerConfig, + inner: Arc>, + agent: Arc, + resource_manager: Arc, + logger: slog::Logger, +} + +impl Container { + pub fn new( + pid: u32, + config: ContainerConfig, + agent: Arc, + resource_manager: Arc, + ) -> Result { + let container_id = ContainerID::new(&config.container_id).context("new container id")?; + let logger = sl!().new(o!("container_id" => config.container_id.clone())); + let process = ContainerProcess::new(&config.container_id, "")?; + let init_process = Process::new( + &process, + pid, + &config.bundle, + config.stdin.clone(), + config.stdout.clone(), + config.stderr.clone(), + config.terminal, + ); + + Ok(Self { + pid, + container_id, + config, + inner: Arc::new(RwLock::new(ContainerInner::new( + agent.clone(), + init_process, + logger.clone(), + ))), + agent, + resource_manager, + logger, + }) + } + + pub async fn create(&self, mut spec: oci::Spec) -> Result<()> { + // process oci spec + let mut inner = self.inner.write().await; + let config = &self.config; + amend_spec(&mut spec).context("load spec")?; + + // handler rootfs + let rootfs = self + .resource_manager + .handler_rootfs(&config.container_id, &config.bundle, &config.rootfs_mounts) + .await + .context("handler rootfs")?; + + // update rootfs + match spec.root.as_mut() { + Some(spec) => { + spec.path = rootfs + .get_guest_rootfs_path() + .await + .context("get guest rootfs path")? + } + None => return Err(anyhow!("spec miss root field")), + }; + inner.rootfs.push(rootfs); + + // handler volumes + let volumes = self + .resource_manager + .handler_volumes(&config.container_id, &spec.mounts) + .await + .context("handler volumes")?; + let mut oci_mounts = vec![]; + let mut storages = vec![]; + for v in volumes { + let mut volume_mounts = v.get_volume_mount().context("get volume mount")?; + if !volume_mounts.is_empty() { + oci_mounts.append(&mut volume_mounts); + } + + let mut s = v.get_storage().context("get storage")?; + if !s.is_empty() { + storages.append(&mut s); + } + inner.volumes.push(v); + } + spec.mounts = oci_mounts; + + // TODO: handler device + + // update cgroups + self.resource_manager + .update_cgroups( + &config.container_id, + spec.linux + .as_ref() + .map(|linux| linux.resources.as_ref()) + .flatten(), + ) + .await?; + + // create container + let r = agent::CreateContainerRequest { + process_id: agent::ContainerProcessID::new(&config.container_id, ""), + string_user: None, + devices: vec![], + storages, + oci: Some(spec), + guest_hooks: None, + sandbox_pidns: false, + rootfs_mounts: vec![], + }; + + self.agent + .create_container(r) + .await + .context("agent create container")?; + self.resource_manager.dump().await; + Ok(()) + } + + pub async fn start(&self, process: &ContainerProcess) -> Result<()> { + let mut inner = self.inner.write().await; + match process.process_type { + ProcessType::Container => { + if let Err(err) = inner.start_container(&process.container_id).await { + let _ = inner.stop_process(process, true).await; + return Err(err); + } + + let container_io = inner.new_container_io(process).await?; + inner + .init_process + .start_io_and_wait(self.agent.clone(), container_io) + .await?; + } + ProcessType::Exec => { + if let Err(e) = inner.start_exec_process(process).await { + let _ = inner.stop_process(process, true).await; + return Err(e).context("enter process"); + } + + let container_io = inner.new_container_io(process).await.context("io stream")?; + + { + let exec = inner + .exec_processes + .get(&process.exec_id) + .ok_or_else(|| Error::ProcessNotFound(process.clone()))?; + if exec.process.height != 0 && exec.process.width != 0 { + inner + .win_resize_process(process, exec.process.height, exec.process.width) + .await + .context("win resize")?; + } + } + + // start io and wait + { + let exec = inner + .exec_processes + .get_mut(&process.exec_id) + .ok_or_else(|| Error::ProcessNotFound(process.clone()))?; + + exec.process + .start_io_and_wait(self.agent.clone(), container_io) + .await + .context("start io and wait")?; + } + } + } + + Ok(()) + } + + pub async fn delete_exec_process(&self, container_process: &ContainerProcess) -> Result<()> { + let mut inner = self.inner.write().await; + inner + .delete_exec_process(&container_process.exec_id) + .await + .context("delete process") + } + + pub async fn state_process( + &self, + container_process: &ContainerProcess, + ) -> Result { + let inner = self.inner.read().await; + match container_process.process_type { + ProcessType::Container => inner.init_process.state().await, + ProcessType::Exec => { + let exec = inner + .exec_processes + .get(&container_process.exec_id) + .ok_or_else(|| Error::ProcessNotFound(container_process.clone()))?; + exec.process.state().await + } + } + } + + pub async fn wait_process( + &self, + container_process: &ContainerProcess, + ) -> Result { + let logger = logger_with_process(container_process); + info!(logger, "start wait process"); + + let inner = self.inner.read().await; + inner + .fetch_exit_watcher(container_process) + .context("fetch exit watcher") + } + + pub async fn kill_process( + &self, + container_process: &ContainerProcess, + signal: u32, + all: bool, + ) -> Result<()> { + let inner = self.inner.read().await; + inner.signal_process(container_process, signal, all).await + } + + pub async fn exec_process( + &self, + container_process: &ContainerProcess, + stdin: Option, + stdout: Option, + stderr: Option, + terminal: bool, + oci_process: OCIProcess, + ) -> Result<()> { + let process = Process::new( + container_process, + self.pid, + &self.config.bundle, + stdin, + stdout, + stderr, + terminal, + ); + let exec = Exec { + process, + oci_process, + }; + let mut inner = self.inner.write().await; + inner.add_exec_process(&container_process.exec_id, exec); + Ok(()) + } + + pub async fn close_io(&self, container_process: &ContainerProcess) -> Result<()> { + let mut inner = self.inner.write().await; + inner.close_io(container_process).await + } + + pub async fn stop_process(&self, container_process: &ContainerProcess) -> Result<()> { + let mut inner = self.inner.write().await; + inner + .stop_process(container_process, true) + .await + .context("stop process") + } + + pub async fn pause(&self) -> Result<()> { + let inner = self.inner.read().await; + if inner.init_process.status == ProcessStatus::Paused { + warn!(self.logger, "container is paused no need to pause"); + return Ok(()); + } + self.agent + .pause_container(self.container_id.clone().into()) + .await + .context("agent pause container")?; + Ok(()) + } + + pub async fn resume(&self) -> Result<()> { + let inner = self.inner.read().await; + if inner.init_process.status == ProcessStatus::Running { + warn!(self.logger, "container is running no need to resume"); + return Ok(()); + } + self.agent + .resume_container(self.container_id.clone().into()) + .await + .context("agent pause container")?; + Ok(()) + } + + pub async fn resize_pty( + &self, + process: &ContainerProcess, + width: u32, + height: u32, + ) -> Result<()> { + let logger = logger_with_process(process); + let inner = self.inner.read().await; + if inner.init_process.status != ProcessStatus::Running { + warn!(logger, "container is running no need to resume"); + return Ok(()); + } + self.agent + .tty_win_resize(agent::TtyWinResizeRequest { + process_id: process.clone().into(), + row: height, + column: width, + }) + .await + .context("resize pty")?; + Ok(()) + } + + pub async fn stats(&self) -> Result> { + let stats_resp = self + .agent + .stats_container(self.container_id.clone().into()) + .await + .context("agent stats container")?; + Ok(Some(stats_resp)) + } + + pub async fn update(&self, resources: &LinuxResources) -> Result<()> { + self.resource_manager + .update_cgroups(&self.config.container_id, Some(resources)) + .await?; + + let req = agent::UpdateContainerRequest { + container_id: self.container_id.container_id.clone(), + resources: resources.clone(), + mounts: Vec::new(), + }; + self.agent + .update_container(req) + .await + .context("agent update container")?; + Ok(()) + } +} + +fn amend_spec(spec: &mut oci::Spec) -> Result<()> { + // hook should be done on host + spec.hooks = None; + + if let Some(linux) = spec.linux.as_mut() { + linux.seccomp = None; + + if let Some(resource) = linux.resources.as_mut() { + resource.devices = Vec::new(); + resource.pids = None; + resource.block_io = None; + resource.hugepage_limits = Vec::new(); + resource.network = None; + } + + let mut ns: Vec = Vec::new(); + for n in linux.namespaces.iter() { + match n.r#type.as_str() { + oci::PIDNAMESPACE | oci::NETWORKNAMESPACE => continue, + _ => ns.push(n.clone()), + } + } + + linux.namespaces = ns; + } + + Ok(()) +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs new file mode 100644 index 000000000000..2920b23ff877 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -0,0 +1,261 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{collections::HashMap, sync::Arc}; + +use agent::Agent; +use anyhow::{anyhow, Context, Result}; +use common::{ + error::Error, + types::{ContainerID, ContainerProcess, ProcessExitStatus, ProcessStatus, ProcessType}, +}; +use nix::sys::signal::Signal; +use resource::{rootfs::Rootfs, volume::Volume}; +use tokio::sync::RwLock; + +use crate::container_manager::logger_with_process; + +use super::{ + io::ContainerIo, + process::{Process, ProcessWatcher}, + Exec, +}; + +pub struct ContainerInner { + agent: Arc, + logger: slog::Logger, + pub(crate) init_process: Process, + pub(crate) exec_processes: HashMap, + pub(crate) rootfs: Vec>, + pub(crate) volumes: Vec>, +} + +impl ContainerInner { + pub(crate) fn new(agent: Arc, init_process: Process, logger: slog::Logger) -> Self { + Self { + agent, + logger, + init_process, + exec_processes: HashMap::new(), + rootfs: vec![], + volumes: vec![], + } + } + + fn container_id(&self) -> &str { + self.init_process.process.container_id() + } + + pub(crate) fn check_state(&self, states: Vec) -> Result<()> { + let state = self.init_process.status; + if states.contains(&state) { + return Ok(()); + } + + Err(anyhow!( + "failed to check state {:?} for {:?}", + state, + states + )) + } + + pub(crate) fn set_state(&mut self, state: ProcessStatus) { + self.init_process.status = state; + } + + pub(crate) async fn start_exec_process(&mut self, process: &ContainerProcess) -> Result<()> { + let exec = self + .exec_processes + .get_mut(&process.exec_id) + .ok_or_else(|| Error::ProcessNotFound(process.clone()))?; + + self.agent + .exec_process(agent::ExecProcessRequest { + process_id: process.clone().into(), + string_user: None, + process: Some(exec.oci_process.clone()), + }) + .await + .map(|_| { + exec.process.status = ProcessStatus::Running; + }) + } + + pub(crate) async fn win_resize_process( + &self, + process: &ContainerProcess, + height: u32, + width: u32, + ) -> Result<()> { + self.check_state(vec![ProcessStatus::Created, ProcessStatus::Running]) + .context("check state")?; + + self.agent + .tty_win_resize(agent::TtyWinResizeRequest { + process_id: process.clone().into(), + row: height, + column: width, + }) + .await?; + Ok(()) + } + + pub fn fetch_exit_watcher(&self, process: &ContainerProcess) -> Result { + match process.process_type { + ProcessType::Container => self.init_process.fetch_exit_watcher(), + ProcessType::Exec => { + let exec = self + .exec_processes + .get(&process.exec_id) + .ok_or_else(|| Error::ProcessNotFound(process.clone()))?; + exec.process.fetch_exit_watcher() + } + } + } + + pub(crate) async fn start_container(&mut self, cid: &ContainerID) -> Result<()> { + self.check_state(vec![ProcessStatus::Created, ProcessStatus::Stopped]) + .context("check state")?; + + self.agent + .start_container(agent::ContainerID { + container_id: cid.container_id.clone(), + }) + .await + .context("start container")?; + + self.set_state(ProcessStatus::Running); + + Ok(()) + } + + async fn get_exit_status(&self) -> Arc> { + self.init_process.exit_status.clone() + } + + pub(crate) fn add_exec_process(&mut self, id: &str, exec: Exec) -> Option { + self.exec_processes.insert(id.to_string(), exec) + } + + pub(crate) async fn delete_exec_process(&mut self, eid: &str) -> Result<()> { + match self.exec_processes.remove(eid) { + Some(_) => { + debug!(self.logger, " delete process eid {}", eid); + Ok(()) + } + None => Err(anyhow!( + "failed to find cid {} eid {}", + self.container_id(), + eid + )), + } + } + + async fn cleanup_container(&mut self, cid: &str, force: bool) -> Result<()> { + // wait until the container process + // terminated and the status write lock released. + info!(self.logger, "wait on container terminated"); + let exit_status = self.get_exit_status().await; + let _locked_exit_status = exit_status.read().await; + info!(self.logger, "container terminated"); + let timeout: u32 = 10; + self.agent + .remove_container(agent::RemoveContainerRequest::new(cid, timeout)) + .await + .or_else(|e| { + if force { + warn!( + self.logger, + "stop container: agent remove container failed: {}", e + ); + Ok(agent::Empty::new()) + } else { + Err(e) + } + })?; + + // close the exit channel to wakeup wait service + // send to notify watchers who are waiting for the process exit + self.init_process.stop(); + Ok(()) + } + + pub(crate) async fn stop_process( + &mut self, + process: &ContainerProcess, + force: bool, + ) -> Result<()> { + let logger = logger_with_process(process); + info!(logger, "begin to stop process"); + // do not stop again when state stopped, may cause multi cleanup resource + self.check_state(vec![ProcessStatus::Running]) + .context("check state")?; + + // if use force mode to stop container, stop always successful + // send kill signal to container + // ignore the error of sending signal, since the process would + // have been killed and exited yet. + self.signal_process(process, Signal::SIGKILL as u32, false) + .await + .map_err(|e| { + warn!(logger, "failed to signal kill. {:?}", e); + }) + .ok(); + + match process.process_type { + ProcessType::Container => self + .cleanup_container(&process.container_id.container_id, force) + .await + .context("stop container")?, + ProcessType::Exec => { + let exec = self + .exec_processes + .get_mut(&process.exec_id) + .ok_or_else(|| anyhow!("failed to find exec"))?; + exec.process.stop(); + } + } + + Ok(()) + } + + pub(crate) async fn signal_process( + &self, + process: &ContainerProcess, + signal: u32, + all: bool, + ) -> Result<()> { + let mut process_id: agent::ContainerProcessID = process.clone().into(); + if all { + // force signal init process + process_id.exec_id.clear(); + }; + + self.agent + .signal_process(agent::SignalProcessRequest { process_id, signal }) + .await?; + Ok(()) + } + + pub async fn new_container_io(&self, process: &ContainerProcess) -> Result { + Ok(ContainerIo::new(self.agent.clone(), process.clone())) + } + + pub async fn close_io(&mut self, process: &ContainerProcess) -> Result<()> { + match process.process_type { + ProcessType::Container => self.init_process.close_io().await, + ProcessType::Exec => { + let exec = self + .exec_processes + .get_mut(&process.exec_id) + .ok_or_else(|| Error::ProcessNotFound(process.clone()))?; + exec.process.close_io().await; + } + }; + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/container_io.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/container_io.rs new file mode 100644 index 000000000000..c211e8bca43a --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/container_io.rs @@ -0,0 +1,171 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + future::Future, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use agent::Agent; +use anyhow::Result; +use common::types::ContainerProcess; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +struct ContainerIoInfo { + pub agent: Arc, + pub process: ContainerProcess, +} + +pub struct ContainerIo { + pub stdin: Box, + pub stdout: Box, + pub stderr: Box, +} + +impl ContainerIo { + pub fn new(agent: Arc, process: ContainerProcess) -> Self { + let info = Arc::new(ContainerIoInfo { agent, process }); + + Self { + stdin: Box::new(ContainerIoWrite::new(info.clone())), + stdout: Box::new(ContainerIoRead::new(info.clone(), true)), + stderr: Box::new(ContainerIoRead::new(info, false)), + } + } +} + +struct ContainerIoWrite<'inner> { + pub info: Arc, + write_future: + Option> + Send + 'inner>>>, +} + +impl<'inner> ContainerIoWrite<'inner> { + pub fn new(info: Arc) -> Self { + Self { + info, + write_future: Default::default(), + } + } + + fn poll_write_inner( + &'inner mut self, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let mut write_future = self.write_future.take(); + if write_future.is_none() { + let req = agent::WriteStreamRequest { + process_id: self.info.process.clone().into(), + data: buf.to_vec(), + }; + write_future = Some(Box::pin(self.info.agent.write_stdin(req))); + } + + let mut write_future = write_future.unwrap(); + match write_future.as_mut().poll(cx) { + Poll::Ready(v) => match v { + Ok(resp) => Poll::Ready(Ok(resp.length as usize)), + Err(err) => Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::Other, err))), + }, + Poll::Pending => { + self.write_future = Some(write_future); + Poll::Pending + } + } + } +} + +impl<'inner> AsyncWrite for ContainerIoWrite<'inner> { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let me = unsafe { + std::mem::transmute::<&mut ContainerIoWrite<'_>, &mut ContainerIoWrite<'inner>>( + &mut *self, + ) + }; + me.poll_write_inner(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +type ResultBuffer = Result; +struct ContainerIoRead<'inner> { + pub info: Arc, + is_stdout: bool, + read_future: Option + Send + 'inner>>>, +} + +impl<'inner> ContainerIoRead<'inner> { + pub fn new(info: Arc, is_stdout: bool) -> Self { + Self { + info, + is_stdout, + read_future: Default::default(), + } + } + fn poll_read_inner( + &'inner mut self, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let mut read_future = self.read_future.take(); + if read_future.is_none() { + let req = agent::ReadStreamRequest { + process_id: self.info.process.clone().into(), + len: buf.remaining() as u32, + }; + read_future = if self.is_stdout { + Some(Box::pin(self.info.agent.read_stdout(req))) + } else { + Some(Box::pin(self.info.agent.read_stderr(req))) + }; + } + + let mut read_future = read_future.unwrap(); + match read_future.as_mut().poll(cx) { + Poll::Ready(v) => match v { + Ok(resp) => { + buf.put_slice(&resp.data); + Poll::Ready(Ok(())) + } + Err(err) => Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::Other, err))), + }, + Poll::Pending => { + self.read_future = Some(read_future); + Poll::Pending + } + } + } +} + +impl<'inner> AsyncRead for ContainerIoRead<'inner> { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let me = unsafe { + std::mem::transmute::<&mut ContainerIoRead<'_>, &mut ContainerIoRead<'inner>>( + &mut *self, + ) + }; + me.poll_read_inner(cx, buf) + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/mod.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/mod.rs new file mode 100644 index 000000000000..3c6ca719bcba --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/mod.rs @@ -0,0 +1,10 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod container_io; +pub use container_io::ContainerIo; +mod shim_io; +pub use shim_io::ShimIo; diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/shim_io.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/shim_io.rs new file mode 100644 index 000000000000..03a4c387b465 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/io/shim_io.rs @@ -0,0 +1,135 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + io, + os::unix::{io::FromRawFd, net::UnixStream as StdUnixStream}, + pin::Pin, + task::Context as TaskContext, + task::Poll, +}; + +use anyhow::{anyhow, Context, Result}; +use nix::{ + fcntl::{self, OFlag}, + sys::stat::Mode, +}; +use tokio::{ + fs::File, + io::{AsyncRead, AsyncWrite}, + net::UnixStream as AsyncUnixStream, +}; +use url::Url; + +fn open_fifo(path: &str) -> Result { + let fd = fcntl::open(path, OFlag::O_RDWR, Mode::from_bits(0).unwrap())?; + + let std_stream = unsafe { StdUnixStream::from_raw_fd(fd) }; + std_stream + .set_nonblocking(true) + .context("set nonblocking")?; + + AsyncUnixStream::from_std(std_stream).map_err(|e| anyhow!(e)) +} + +pub struct ShimIo { + pub stdin: Option>, + pub stdout: Option>, + pub stderr: Option>, +} + +impl ShimIo { + pub async fn new( + stdin: &Option, + stdout: &Option, + stderr: &Option, + ) -> Result { + let stdin_fd: Option> = if let Some(stdin) = stdin { + match File::open(&stdin).await { + Ok(file) => Some(Box::new(file)), + Err(err) => { + error!(sl!(), "failed to open {} error {:?}", &stdin, err); + None + } + } + } else { + None + }; + + let get_url = |url: &Option| -> Option { + match url { + None => None, + Some(out) => match Url::parse(out.as_str()) { + Err(url::ParseError::RelativeUrlWithoutBase) => { + let out = "fifo://".to_owned() + out.as_str(); + let u = Url::parse(out.as_str()).unwrap(); + Some(u) + } + Err(err) => { + warn!(sl!(), "unable to parse stdout uri: {}", err); + None + } + Ok(u) => Some(u), + }, + } + }; + + let stdout_url = get_url(stdout); + let get_fd = |url: &Option| -> Option> { + if let Some(url) = url { + if url.scheme() == "fifo" { + let path = url.path(); + match open_fifo(path) { + Ok(s) => { + return Some(Box::new(ShimIoWrite::Stream(s))); + } + Err(err) => { + error!(sl!(), "failed to open file {} error {:?}", url.path(), err); + } + } + } + } + None + }; + + let stderr_url = get_url(stderr); + Ok(Self { + stdin: stdin_fd, + stdout: get_fd(&stdout_url), + stderr: get_fd(&stderr_url), + }) + } +} + +#[derive(Debug)] +enum ShimIoWrite { + Stream(AsyncUnixStream), + // TODO: support other type +} + +impl AsyncWrite for ShimIoWrite { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut TaskContext<'_>, + buf: &[u8], + ) -> Poll> { + match *self { + ShimIoWrite::Stream(ref mut s) => Pin::new(s).poll_write(cx, buf), + } + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + match *self { + ShimIoWrite::Stream(ref mut s) => Pin::new(s).poll_flush(cx), + } + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + match *self { + ShimIoWrite::Stream(ref mut s) => Pin::new(s).poll_shutdown(cx), + } + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs new file mode 100644 index 000000000000..155cf0a9c4d2 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -0,0 +1,274 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Context, Result}; + +use std::{collections::HashMap, sync::Arc}; + +use agent::Agent; +use async_trait::async_trait; +use common::{ + error::Error, + types::{ + ContainerConfig, ContainerID, ContainerProcess, ExecProcessRequest, KillRequest, + ProcessExitStatus, ProcessStateInfo, ProcessType, ResizePTYRequest, ShutdownRequest, + StatsInfo, UpdateRequest, PID, + }, + ContainerManager, +}; +use oci::Process as OCIProcess; +use resource::ResourceManager; +use tokio::sync::RwLock; + +use super::{logger_with_process, Container}; + +unsafe impl Send for VirtContainerManager {} +unsafe impl Sync for VirtContainerManager {} +pub struct VirtContainerManager { + sid: String, + pid: u32, + containers: Arc>>, + resource_manager: Arc, + agent: Arc, +} + +impl VirtContainerManager { + pub fn new( + sid: &str, + pid: u32, + agent: Arc, + resource_manager: Arc, + ) -> Self { + Self { + sid: sid.to_string(), + pid, + containers: Default::default(), + resource_manager, + agent, + } + } +} + +#[async_trait] +impl ContainerManager for VirtContainerManager { + async fn create_container(&self, config: ContainerConfig, spec: oci::Spec) -> Result { + let container = Container::new( + self.pid, + config, + self.agent.clone(), + self.resource_manager.clone(), + ) + .context("new container")?; + + let mut containers = self.containers.write().await; + container.create(spec).await.context("create")?; + containers.insert(container.container_id.to_string(), container); + + Ok(PID { pid: self.pid }) + } + + async fn close_process_io(&self, process: &ContainerProcess) -> Result<()> { + let containers = self.containers.read().await; + let container_id = &process.container_id.to_string(); + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + + c.close_io(process).await.context("close io")?; + Ok(()) + } + + async fn delete_process(&self, process: &ContainerProcess) -> Result { + let container_id = &process.container_id.container_id; + match process.process_type { + ProcessType::Container => { + let mut containers = self.containers.write().await; + let c = containers + .remove(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.to_string()))?; + c.state_process(process).await.context("state process") + } + ProcessType::Exec => { + let containers = self.containers.read().await; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.to_string()))?; + c.delete_exec_process(process) + .await + .context("delete process")?; + c.state_process(process).await.context("state process") + } + } + } + + async fn exec_process(&self, req: ExecProcessRequest) -> Result<()> { + if req.spec_type_url.is_empty() { + return Err(anyhow!("invalid type url")); + } + let oci_process: OCIProcess = + serde_json::from_slice(&req.spec_value).context("serde from slice")?; + + let containers = self.containers.read().await; + let container_id = &req.process.container_id.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + c.exec_process( + &req.process, + req.stdin, + req.stdout, + req.stderr, + req.terminal, + oci_process, + ) + .await + .context("exec")?; + Ok(()) + } + + async fn kill_process(&self, req: &KillRequest) -> Result<()> { + let containers = self.containers.read().await; + let container_id = &req.process.container_id.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + c.kill_process(&req.process, req.signal, req.all) + .await + .map_err(|err| { + warn!( + sl!(), + "failed to signal process {:?} {:?}", &req.process, err + ); + err + }) + .ok(); + Ok(()) + } + + async fn wait_process(&self, process: &ContainerProcess) -> Result { + let logger = logger_with_process(process); + + let containers = self.containers.read().await; + let container_id = &process.container_id.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + let (watcher, status) = c.wait_process(process).await.context("wait")?; + drop(containers); + + match watcher { + Some(mut watcher) => { + info!(logger, "begin wait exit"); + while watcher.changed().await.is_ok() {} + info!(logger, "end wait exited"); + } + None => { + warn!(logger, "failed to find watcher for wait process"); + } + } + + let status = status.read().await; + + info!(logger, "wait process exit status {:?}", status); + + // stop process + let containers = self.containers.read().await; + let container_id = &process.container_id.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + c.stop_process(process).await.context("stop container")?; + Ok(status.clone()) + } + + async fn start_process(&self, process: &ContainerProcess) -> Result { + let containers = self.containers.read().await; + let container_id = &process.container_id.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + c.start(process).await.context("start")?; + Ok(PID { pid: self.pid }) + } + + async fn state_process(&self, process: &ContainerProcess) -> Result { + let containers = self.containers.read().await; + let container_id = &process.container_id.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.clone()))?; + let state = c.state_process(process).await.context("state process")?; + Ok(state) + } + + async fn pause_container(&self, id: &ContainerID) -> Result<()> { + let containers = self.containers.read().await; + let c = containers + .get(&id.container_id) + .ok_or_else(|| Error::ContainerNotFound(id.container_id.clone()))?; + c.pause().await.context("pause")?; + Ok(()) + } + + async fn resume_container(&self, id: &ContainerID) -> Result<()> { + let containers = self.containers.read().await; + let c = containers + .get(&id.container_id) + .ok_or_else(|| Error::ContainerNotFound(id.container_id.clone()))?; + c.resume().await.context("resume")?; + Ok(()) + } + + async fn resize_process_pty(&self, req: &ResizePTYRequest) -> Result<()> { + let containers = self.containers.read().await; + let c = containers + .get(&req.process.container_id.container_id) + .ok_or_else(|| { + Error::ContainerNotFound(req.process.container_id.container_id.clone()) + })?; + c.resize_pty(&req.process, req.width, req.height) + .await + .context("resize pty")?; + Ok(()) + } + + async fn stats_container(&self, id: &ContainerID) -> Result { + let containers = self.containers.read().await; + let c = containers + .get(&id.container_id) + .ok_or_else(|| Error::ContainerNotFound(id.container_id.clone()))?; + let stats = c.stats().await.context("stats")?; + Ok(StatsInfo::from(stats)) + } + + async fn update_container(&self, req: UpdateRequest) -> Result<()> { + let resource = serde_json::from_slice::(&req.value) + .context("deserialize LinuxResource")?; + let containers = self.containers.read().await; + let container_id = &req.container_id; + let c = containers + .get(container_id) + .ok_or_else(|| Error::ContainerNotFound(container_id.to_string()))?; + c.update(&resource).await.context("stats") + } + + async fn pid(&self) -> Result { + Ok(PID { pid: self.pid }) + } + + async fn connect_container(&self, _id: &ContainerID) -> Result { + Ok(PID { pid: self.pid }) + } + + async fn need_shutdown_sandbox(&self, req: &ShutdownRequest) -> bool { + req.is_now || self.containers.read().await.is_empty() || self.sid == req.container_id + } + + async fn is_sandbox_container(&self, process: &ContainerProcess) -> bool { + process.process_type == ProcessType::Container + && process.container_id.container_id == self.sid + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/mod.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/mod.rs new file mode 100644 index 000000000000..3c615517fd5c --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/mod.rs @@ -0,0 +1,20 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod container; +use container::{Container, Exec}; +mod container_inner; +mod io; +use container_inner::ContainerInner; +mod manager; +pub use manager::VirtContainerManager; +mod process; + +use common::types::ContainerProcess; + +fn logger_with_process(container_process: &ContainerProcess) -> slog::Logger { + sl!().new(o!("container_id" => container_process.container_id.container_id.clone(), "exec_id" => container_process.exec_id.clone())) +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs new file mode 100644 index 000000000000..28b9b023c7fd --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs @@ -0,0 +1,211 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use agent::Agent; +use anyhow::{Context, Result}; +use awaitgroup::{WaitGroup, Worker as WaitGroupWorker}; +use common::types::{ContainerProcess, ProcessExitStatus, ProcessStateInfo, ProcessStatus, PID}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::{watch, RwLock}, +}; + +use super::{ + io::{ContainerIo, ShimIo}, + logger_with_process, +}; + +pub type ProcessWatcher = ( + Option>, + Arc>, +); + +#[derive(Debug)] +pub struct Process { + pub process: ContainerProcess, + pub pid: u32, + logger: slog::Logger, + pub bundle: String, + + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, + pub terminal: bool, + + pub height: u32, + pub width: u32, + pub status: ProcessStatus, + + pub exit_status: Arc>, + pub exit_watcher_rx: Option>, + pub exit_watcher_tx: Option>, + // used to sync between stdin io copy thread(tokio) and the close it call. + // close io call should wait until the stdin io copy finished to + // prevent stdin data lost. + pub wg_stdin: WaitGroup, +} + +impl Process { + pub fn new( + process: &ContainerProcess, + pid: u32, + bundle: &str, + stdin: Option, + stdout: Option, + stderr: Option, + terminal: bool, + ) -> Process { + let (sender, receiver) = watch::channel(false); + + Process { + process: process.clone(), + pid, + logger: logger_with_process(process), + bundle: bundle.to_string(), + stdin, + stdout, + stderr, + terminal, + height: 0, + width: 0, + status: ProcessStatus::Created, + exit_status: Arc::new(RwLock::new(ProcessExitStatus::new())), + exit_watcher_rx: Some(receiver), + exit_watcher_tx: Some(sender), + wg_stdin: WaitGroup::new(), + } + } + + pub async fn start_io_and_wait( + &mut self, + agent: Arc, + container_io: ContainerIo, + ) -> Result<()> { + info!(self.logger, "start io and wait"); + + // new shim io + let shim_io = ShimIo::new(&self.stdin, &self.stdout, &self.stderr) + .await + .context("new shim io")?; + + // start io copy for stdin + let wgw_stdin = self.wg_stdin.worker(); + if let Some(stdin) = shim_io.stdin { + self.run_io_copy("stdin", wgw_stdin, stdin, container_io.stdin) + .await?; + } + + // prepare for wait group for stdout, stderr + let wg = WaitGroup::new(); + let wgw = wg.worker(); + + // start io copy for stdout + if let Some(stdout) = shim_io.stdout { + self.run_io_copy("stdout", wgw.clone(), container_io.stdout, stdout) + .await?; + } + + // start io copy for stderr + if !self.terminal { + if let Some(stderr) = shim_io.stderr { + self.run_io_copy("stderr", wgw, container_io.stderr, stderr) + .await?; + } + } + + self.run_io_wait(agent, wg).await.context("run io thread")?; + Ok(()) + } + + async fn run_io_copy<'a>( + &'a self, + io_name: &'a str, + wgw: WaitGroupWorker, + mut reader: Box, + mut writer: Box, + ) -> Result<()> { + let io_name = io_name.to_string(); + let logger = self.logger.new(o!("io name" => io_name)); + let _ = tokio::spawn(async move { + match tokio::io::copy(&mut reader, &mut writer).await { + Err(e) => warn!(logger, "io: failed to copy stdin stream {}", e), + Ok(length) => warn!(logger, "io: stop to copy stdin stream length {}", length), + }; + + wgw.done(); + }); + + Ok(()) + } + + async fn run_io_wait(&mut self, agent: Arc, mut wg: WaitGroup) -> Result<()> { + let logger = self.logger.clone(); + info!(logger, "start run io wait"); + let process = self.process.clone(); + let status = self.exit_status.clone(); + let exit_notifier = self.exit_watcher_tx.take(); + + let _ = tokio::spawn(async move { + //wait on all of the container's io stream terminated + info!(logger, "begin wait group io",); + wg.wait().await; + info!(logger, "end wait group for io"); + + let req = agent::WaitProcessRequest { + process_id: process.clone().into(), + }; + + info!(logger, "begin wait process"); + let resp = match agent.wait_process(req).await { + Ok(ret) => ret, + Err(e) => { + error!(logger, "failed to wait process {:?}", e); + return; + } + }; + + info!(logger, "end wait process exit code {}", resp.status); + + let mut locked_status = status.write().await; + locked_status.update_exit_code(resp.status); + + drop(exit_notifier); + info!(logger, "end io wait thread"); + }); + Ok(()) + } + + pub fn fetch_exit_watcher(&self) -> Result { + Ok((self.exit_watcher_rx.clone(), self.exit_status.clone())) + } + + pub async fn state(&self) -> Result { + let exit_status = self.exit_status.read().await; + Ok(ProcessStateInfo { + container_id: self.process.container_id.container_id.clone(), + exec_id: self.process.exec_id.clone(), + pid: PID { pid: self.pid }, + bundle: self.bundle.clone(), + stdin: self.stdin.clone(), + stdout: self.stdout.clone(), + stderr: self.stderr.clone(), + terminal: self.terminal, + status: self.status, + exit_status: exit_status.exit_code, + exited_at: exit_status.exit_time, + }) + } + + pub fn stop(&mut self) { + self.status = ProcessStatus::Stopped; + } + + pub async fn close_io(&mut self) { + self.wg_stdin.wait().await; + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs b/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs new file mode 100644 index 000000000000..3a7703ac3b47 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use agent::Agent; +use anyhow::Context; +use tokio::sync::{mpsc, Mutex}; + +/// monitor check interval 30s +const HEALTH_CHECK_TIMER_INTERVAL: u64 = 30; + +/// version check threshold 5min +const VERSION_CHECK_THRESHOLD: u64 = 5 * 60 / HEALTH_CHECK_TIMER_INTERVAL; + +/// health check stop channel buffer size +const HEALTH_CHECK_STOP_CHANNEL_BUFFER_SIZE: usize = 1; + +pub struct HealthCheck { + pub keep_alive: bool, + keep_vm: bool, + stop_tx: mpsc::Sender<()>, + stop_rx: Arc>>, +} + +impl HealthCheck { + pub fn new(keep_alive: bool, keep_vm: bool) -> HealthCheck { + let (tx, rx) = mpsc::channel(HEALTH_CHECK_STOP_CHANNEL_BUFFER_SIZE); + HealthCheck { + keep_alive, + keep_vm, + stop_tx: tx, + stop_rx: Arc::new(Mutex::new(rx)), + } + } + + pub fn start(&self, id: &str, agent: Arc) { + if !self.keep_alive { + return; + } + let id = id.to_string(); + + info!(sl!(), "start runtime keep alive"); + + let stop_rx = self.stop_rx.clone(); + let keep_vm = self.keep_vm; + let _ = tokio::spawn(async move { + let mut version_check_threshold_count = 0; + + loop { + tokio::time::sleep(std::time::Duration::from_secs(HEALTH_CHECK_TIMER_INTERVAL)) + .await; + let mut stop_rx = stop_rx.lock().await; + match stop_rx.try_recv() { + Ok(_) => { + info!(sl!(), "revive stop {} monitor signal", id); + break; + } + + Err(mpsc::error::TryRecvError::Empty) => { + // check agent + match agent + .check(agent::CheckRequest::new("")) + .await + .context("check health") + { + Ok(_) => { + debug!(sl!(), "check {} agent health successfully", id); + version_check_threshold_count += 1; + if version_check_threshold_count >= VERSION_CHECK_THRESHOLD { + // need to check version + version_check_threshold_count = 0; + if let Ok(v) = agent + .version(agent::CheckRequest::new("")) + .await + .context("check version") + { + info!(sl!(), "agent {}", v.agent_version) + } + } + continue; + } + Err(e) => { + error!(sl!(), "failed to do {} agent health check: {}", id, e); + if let Err(mpsc::error::TryRecvError::Empty) = stop_rx.try_recv() { + error!(sl!(), "failed to receive stop monitor signal"); + if !keep_vm { + ::std::process::exit(1); + } + } else { + info!(sl!(), "wait to exit exit {}", id); + break; + } + } + } + } + + Err(mpsc::error::TryRecvError::Disconnected) => { + warn!(sl!(), "{} monitor channel has broken", id); + break; + } + } + } + }); + } + + pub async fn stop(&self) { + if !self.keep_alive { + return; + } + info!(sl!(), "stop runtime keep alive"); + self.stop_tx + .send(()) + .await + .map_err(|e| { + warn!(sl!(), "failed send monitor channel. {:?}", e); + }) + .ok(); + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs index 8419a9382838..1710a8370193 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -3,12 +3,25 @@ // // SPDX-License-Identifier: Apache-2.0 // + +#[macro_use] +extern crate slog; + +logging::logger_with_subsystem!(sl, "virt-container"); + +mod container_manager; +pub mod health_check; +pub mod sandbox; + use std::sync::Arc; -use anyhow::Result; +use agent::kata::KataAgent; +use anyhow::{Context, Result}; use async_trait::async_trait; use common::{message::Message, RuntimeHandler, RuntimeInstance}; -use kata_types::config::{hypervisor::register_hypervisor_plugin, DragonballConfig}; +use hypervisor::Hypervisor; +use kata_types::config::{hypervisor::register_hypervisor_plugin, DragonballConfig, TomlConfig}; +use resource::ResourceManager; use tokio::sync::mpsc::Sender; unsafe impl Send for VirtContainer {} @@ -34,10 +47,55 @@ impl RuntimeHandler for VirtContainer { async fn new_instance( &self, - _sid: &str, - _msg_sender: Sender, + sid: &str, + msg_sender: Sender, ) -> Result { - todo!() + let (toml_config, _) = TomlConfig::load_from_file("").context("load config")?; + + // TODO: new sandbox and container manager + // TODO: get from hypervisor + let hypervisor = new_hypervisor(&toml_config) + .await + .context("new hypervisor")?; + + // get uds from hypervisor and get config from toml_config + let agent = Arc::new(KataAgent::new(kata_types::config::Agent { + debug: true, + enable_tracing: false, + server_port: 1024, + log_port: 1025, + dial_timeout_ms: 10, + reconnect_timeout_ms: 3_000, + request_timeout_ms: 30_000, + health_check_request_timeout_ms: 90_000, + kernel_modules: Default::default(), + container_pipe_size: 0, + debug_console_enabled: false, + })); + + let resource_manager = Arc::new(ResourceManager::new( + sid, + agent.clone(), + hypervisor.clone(), + &toml_config, + )?); + let pid = std::process::id(); + + let sandbox = sandbox::VirtSandbox::new( + sid, + msg_sender, + agent.clone(), + hypervisor, + resource_manager.clone(), + ) + .await + .context("new virt sandbox")?; + let container_manager = + container_manager::VirtContainerManager::new(sid, pid, agent, resource_manager); + Ok(RuntimeInstance { + sandbox: Arc::new(sandbox), + container_manager: Arc::new(container_manager), + }) } fn cleanup(&self, _id: &str) -> Result<()> { @@ -45,3 +103,8 @@ impl RuntimeHandler for VirtContainer { Ok(()) } } + +async fn new_hypervisor(_toml_config: &TomlConfig) -> Result> { + // TODO: implement ready hypervisor + todo!() +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs new file mode 100644 index 000000000000..b98492af47c7 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -0,0 +1,229 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use agent::{self, Agent}; +use anyhow::{Context, Result}; +use async_trait::async_trait; +use common::{ + message::{Action, Message}, + Sandbox, +}; +use containerd_shim_protos::events::task::TaskOOM; +use hypervisor::Hypervisor; +use kata_types::config::TomlConfig; +use resource::{ResourceConfig, ResourceManager}; +use tokio::sync::{mpsc::Sender, Mutex, RwLock}; + +use crate::health_check::HealthCheck; + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum SandboxState { + Init, + Running, + Stopped, +} + +struct SandboxInner { + state: SandboxState, +} + +impl SandboxInner { + pub fn new() -> Self { + Self { + state: SandboxState::Init, + } + } +} + +unsafe impl Send for VirtSandbox {} +unsafe impl Sync for VirtSandbox {} +#[derive(Clone)] +pub struct VirtSandbox { + sid: String, + msg_sender: Arc>>, + inner: Arc>, + resource_manager: Arc, + agent: Arc, + hypervisor: Arc, + monitor: Arc, +} + +impl VirtSandbox { + pub async fn new( + sid: &str, + msg_sender: Sender, + agent: Arc, + hypervisor: Arc, + resource_manager: Arc, + ) -> Result { + Ok(Self { + sid: sid.to_string(), + msg_sender: Arc::new(Mutex::new(msg_sender)), + inner: Arc::new(RwLock::new(SandboxInner::new())), + agent, + hypervisor, + resource_manager, + monitor: Arc::new(HealthCheck::new(true, true)), + }) + } + + async fn prepare_for_start_sandbox( + &self, + netns: Option, + _config: &TomlConfig, + ) -> Result> { + let mut resource_configs = vec![]; + + if let Some(_netns_path) = netns { + // TODO: support network + } + + let hypervisor_config = self.hypervisor.hypervisor_config().await; + let virtio_fs_config = ResourceConfig::ShareFs(hypervisor_config.shared_fs); + resource_configs.push(virtio_fs_config); + + Ok(resource_configs) + } +} + +#[async_trait] +impl Sandbox for VirtSandbox { + async fn start(&self, netns: Option, config: &TomlConfig) -> Result<()> { + let id = &self.sid; + + // if sandbox running, return + // if sandbox not running try to start sandbox + let mut inner = self.inner.write().await; + if inner.state == SandboxState::Running { + warn!(sl!(), "sandbox is running, no need to start"); + return Ok(()); + } + + self.hypervisor + .prepare_vm(id, netns.clone()) + .await + .context("prepare vm")?; + + // generate device and setup before start vm + // should after hypervisor.prepare_vm + let resources = self.prepare_for_start_sandbox(netns, config).await?; + self.resource_manager + .prepare_before_start_vm(resources) + .await + .context("set up device before start vm")?; + + // start vm + self.hypervisor.start_vm(10_000).await.context("start vm")?; + info!(sl!(), "start vm"); + + // connect agent + // set agent socket + let address = self + .hypervisor + .get_agent_socket() + .await + .context("get agent socket")?; + self.agent.start(&address).await.context("connect")?; + + self.resource_manager + .setup_after_start_vm() + .await + .context("setup device after start vm")?; + + // create sandbox in vm + let req = agent::CreateSandboxRequest { + hostname: "".to_string(), + dns: vec![], + storages: self + .resource_manager + .get_storage_for_sandbox() + .await + .context("get storages for sandbox")?, + sandbox_pidns: false, + sandbox_id: id.to_string(), + guest_hook_path: "".to_string(), + kernel_modules: vec![], + }; + + self.agent + .create_sandbox(req) + .await + .context("create sandbox")?; + + inner.state = SandboxState::Running; + let agent = self.agent.clone(); + let sender = self.msg_sender.clone(); + info!(sl!(), "oom watcher start"); + let _ = tokio::spawn(async move { + loop { + match agent + .get_oom_event(agent::Empty::new()) + .await + .context("get oom event") + { + Ok(resp) => { + let cid = &resp.container_id; + warn!(sl!(), "send oom event for container {}", &cid); + let event = TaskOOM { + container_id: cid.to_string(), + ..Default::default() + }; + let msg = Message::new(Action::Event(Arc::new(event))); + let lock_sender = sender.lock().await; + if let Err(err) = lock_sender.send(msg).await.context("send event") { + error!( + sl!(), + "failed to send oom event for {} error {:?}", cid, err + ); + } + } + Err(err) => { + warn!(sl!(), "failed to get oom event error {:?}", err); + break; + } + } + } + }); + self.monitor.start(id, self.agent.clone()); + Ok(()) + } + + async fn stop(&self) -> Result<()> { + info!(sl!(), "begin stop sandbox"); + // TODO: stop sandbox + Ok(()) + } + + async fn shutdown(&self) -> Result<()> { + info!(sl!(), "shutdown"); + + self.resource_manager + .delete_cgroups() + .await + .context("delete cgroups")?; + + info!(sl!(), "stop monitor"); + self.monitor.stop().await; + + info!(sl!(), "stop agent"); + self.agent.stop().await; + + // stop server + info!(sl!(), "send shutdown message"); + let msg = Message::new(Action::Shutdown); + let sender = self.msg_sender.clone(); + let sender = sender.lock().await; + sender.send(msg).await.context("send shutdown msg")?; + Ok(()) + } + + async fn cleanup(&self, _id: &str) -> Result<()> { + // TODO: cleanup + Ok(()) + } +} From fd4c26f9c191b726667a87f2b4ebd725a76d6fe9 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Thu, 14 Apr 2022 12:21:35 +0800 Subject: [PATCH 0049/1953] runtime-rs: support network resource Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/libs/kata-sys-util/Cargo.toml | 4 +- .../kata-sys-util/src/rand/random_bytes.rs | 13 +- src/libs/kata-sys-util/src/rand/uuid.rs | 19 +- src/runtime-rs/Cargo.lock | 107 ++++++- src/runtime-rs/crates/hypervisor/src/lib.rs | 1 + src/runtime-rs/crates/resource/Cargo.toml | 8 +- src/runtime-rs/crates/resource/src/lib.rs | 3 + .../crates/resource/src/manager_inner.rs | 63 +++- .../resource/src/network/endpoint/mod.rs | 22 ++ .../src/network/endpoint/physical_endpoint.rs | 145 +++++++++ .../src/network/endpoint/veth_endpoint.rs | 84 +++++ .../crates/resource/src/network/mod.rs | 48 +++ .../resource/src/network/network_entity.rs | 24 ++ .../resource/src/network/network_info/mod.rs | 18 ++ .../network_info/network_info_from_link.rs | 203 ++++++++++++ .../resource/src/network/network_model/mod.rs | 46 +++ .../src/network/network_model/none_model.rs | 35 +++ .../src/network/network_model/route_model.rs | 88 ++++++ .../network/network_model/tc_filter_model.rs | 95 ++++++ .../resource/src/network/network_pair.rs | 178 +++++++++++ .../src/network/network_with_netns.rs | 211 +++++++++++++ .../resource/src/network/utils/address.rs | 109 +++++++ .../resource/src/network/utils/link/create.rs | 129 ++++++++ .../src/network/utils/link/driver_info.rs | 102 +++++++ .../resource/src/network/utils/link/macros.rs | 48 +++ .../src/network/utils/link/manager.rs | 288 ++++++++++++++++++ .../resource/src/network/utils/link/mod.rs | 145 +++++++++ .../crates/resource/src/network/utils/mod.rs | 35 +++ .../resource/src/network/utils/netns.rs | 51 ++++ .../crates/resource/src/rootfs/mod.rs | 4 +- .../resource/src/volume/share_fs_volume.rs | 2 +- 31 files changed, 2311 insertions(+), 17 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs create mode 100644 src/runtime-rs/crates/resource/src/network/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_entity.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_info/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_model/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_model/none_model.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_model/route_model.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_pair.rs create mode 100644 src/runtime-rs/crates/resource/src/network/network_with_netns.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/address.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/link/create.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/link/driver_info.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/link/macros.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/link/manager.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/link/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/mod.rs create mode 100644 src/runtime-rs/crates/resource/src/network/utils/netns.rs diff --git a/src/libs/kata-sys-util/Cargo.toml b/src/libs/kata-sys-util/Cargo.toml index 7eba88a420f9..781d0a5bb524 100644 --- a/src/libs/kata-sys-util/Cargo.toml +++ b/src/libs/kata-sys-util/Cargo.toml @@ -11,7 +11,7 @@ license = "Apache-2.0" edition = "2018" [dependencies] -byteorder = "~1" +byteorder = "1.4.3" cgroups = { package = "cgroups-rs", version = "0.2.7" } chrono = "0.4.0" common-path = "=1.0.0" @@ -24,7 +24,7 @@ serde_json = "1.0.73" slog = "2.5.2" slog-scope = "4.4.0" subprocess = "0.2.8" -rand = "^0.7.2" +rand = "0.7.2" thiserror = "1.0.30" kata-types = { path = "../kata-types" } diff --git a/src/libs/kata-sys-util/src/rand/random_bytes.rs b/src/libs/kata-sys-util/src/rand/random_bytes.rs index 44f792962053..183856d6b5e6 100644 --- a/src/libs/kata-sys-util/src/rand/random_bytes.rs +++ b/src/libs/kata-sys-util/src/rand/random_bytes.rs @@ -46,6 +46,17 @@ mod tests { fn random_bytes() { let b = RandomBytes::new(16); assert_eq!(b.bytes.len(), 16); - println!("{:?}", b.bytes); + + // check lower hex + let lower_hex = format!("{:x}", b); + assert_eq!(lower_hex, lower_hex.to_lowercase()); + + // check upper hex + let upper_hex = format!("{:X}", b); + assert_eq!(upper_hex, upper_hex.to_uppercase()); + + // check new random bytes + let b1 = RandomBytes::new(16); + assert_ne!(b.bytes, b1.bytes); } } diff --git a/src/libs/kata-sys-util/src/rand/uuid.rs b/src/libs/kata-sys-util/src/rand/uuid.rs index a257c9480390..905ba05e24ae 100644 --- a/src/libs/kata-sys-util/src/rand/uuid.rs +++ b/src/libs/kata-sys-util/src/rand/uuid.rs @@ -27,6 +27,7 @@ impl UUID { } } +/// From: convert UUID to string impl From<&UUID> for String { fn from(from: &UUID) -> Self { let time_low = BigEndian::read_u32(&from.0[..4]); @@ -57,13 +58,17 @@ mod tests { #[test] fn test_uuid() { - let uuid = UUID::new(); - let sss: String = String::from(&uuid); - println!("{}", sss); + let uuid1 = UUID::new(); + let s1: String = String::from(&uuid1); - let uuid2 = UUID([0u8, 1u8, 2u8, 3u8, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - let sss2 = String::from(&uuid2); - println!("Display: {}", uuid2); - assert_eq!(&sss2, "00010203-0405-0607-0809-0a0b0c0d0e0f"); + let uuid2 = UUID::new(); + let s2: String = String::from(&uuid2); + + assert_eq!(s1.len(), s2.len()); + assert_ne!(s1, s2); + + let uuid3 = UUID([0u8, 1u8, 2u8, 3u8, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let s3 = String::from(&uuid3); + assert_eq!(&s3, "00010203-0405-0607-0809-0a0b0c0d0e0f"); } } diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 09abdaca5951..8a59c5df06ef 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -792,6 +792,71 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "netlink-packet-core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297" +dependencies = [ + "anyhow", + "byteorder", + "libc", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-route" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733ea73609acfd7fa7ddadfb7bf709b0471668c456ad9513685af543a06342b2" +dependencies = [ + "anyhow", + "bitflags", + "byteorder", + "libc", + "netlink-packet-core", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-utils" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e" +dependencies = [ + "anyhow", + "byteorder", + "paste", + "thiserror", +] + +[[package]] +name = "netlink-proto" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8785b8141e8432aa45fceb922a7e876d7da3fad37fa7e7ec702ace3aa0826b" +dependencies = [ + "bytes 1.1.0", + "futures 0.3.21", + "log", + "netlink-packet-core", + "netlink-sys", + "tokio", +] + +[[package]] +name = "netlink-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c9f9547a08241bee7b6558b9b98e1f290d187de8b7cfca2bbb4937bcaa8f8" +dependencies = [ + "bytes 1.1.0", + "futures 0.3.21", + "libc", + "log", + "tokio", +] + [[package]] name = "nix" version = "0.16.1" @@ -805,6 +870,19 @@ dependencies = [ "void", ] +[[package]] +name = "nix" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset", +] + [[package]] name = "nix" version = "0.23.1" @@ -921,6 +999,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1254,22 +1338,43 @@ dependencies = [ "agent", "anyhow", "async-trait", + "bitflags", "cgroups-rs", + "futures 0.3.21", "hypervisor", "kata-sys-util", "kata-types", "lazy_static", "libc", - "log", "logging", + "netlink-packet-route", + "netlink-sys", "nix 0.16.1", "oci", + "rand 0.7.3", + "rtnetlink", + "scopeguard", "slog", "slog-scope", "tokio", "uuid", ] +[[package]] +name = "rtnetlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f54290e54521dac3de4149d83ddf9f62a359b3cc93bcb494a794a41e6f4744b" +dependencies = [ + "futures 0.3.21", + "log", + "netlink-packet-route", + "netlink-proto", + "nix 0.22.3", + "thiserror", + "tokio", +] + [[package]] name = "runtimes" version = "0.1.0" diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index de8c172375b8..0889f3322eb7 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -10,6 +10,7 @@ extern crate slog; logging::logger_with_subsystem!(sl, "hypervisor"); pub mod device; +pub use device::*; use std::collections::HashMap; diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 115d4fc56bb7..e6365b1705c0 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -7,11 +7,17 @@ edition = "2018" [dependencies] anyhow = "^1.0" async-trait = "0.1.48" +bitflags = "1.2.1" cgroups-rs = "0.2.9" +futures = "0.3.11" lazy_static = "1.4.0" libc = ">=0.2.39" -log = "^0.4.0" +netlink-sys = "0.8.2" +netlink-packet-route = "0.11.0" nix = "0.16.0" +rand = "^0.7.2" +rtnetlink = "0.9.1" +scopeguard = "1.0.0" slog = "2.5.2" slog-scope = "4.4.0" tokio = { version = "1.8.0", features = ["process"] } diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs index 86dc71fe787e..28ffc5601968 100644 --- a/src/runtime-rs/crates/resource/src/lib.rs +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -15,6 +15,8 @@ logging::logger_with_subsystem!(sl, "resource"); pub mod cgroups; pub mod manager; mod manager_inner; +pub mod network; +use network::NetworkConfig; pub mod rootfs; pub mod share_fs; pub mod volume; @@ -24,5 +26,6 @@ use kata_types::config::hypervisor::SharedFsInfo; #[derive(Debug)] pub enum ResourceConfig { + Network(NetworkConfig), ShareFs(SharedFsInfo), } diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index 7b7cfea4fdf4..c254352429c3 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -15,6 +15,7 @@ use oci::LinuxResources; use crate::{ cgroups::CgroupsResource, + network::{self, Network}, rootfs::{RootFsResource, Rootfs}, share_fs::{self, ShareFs}, volume::{Volume, VolumeResource}, @@ -23,10 +24,9 @@ use crate::{ pub(crate) struct ResourceManagerInner { sid: String, - // TODO: remove - #[allow(dead_code)] agent: Arc, hypervisor: Arc, + network: Option>, share_fs: Option>, pub rootfs_resource: RootFsResource, @@ -45,6 +45,7 @@ impl ResourceManagerInner { sid: sid.to_string(), agent, hypervisor, + network: None, share_fs: None, rootfs_resource: RootFsResource::new(), volume_resource: VolumeResource::new(), @@ -66,12 +67,60 @@ impl ResourceManagerInner { .context("setup share fs device before start vm")?; self.share_fs = Some(share_fs); } + ResourceConfig::Network(c) => { + let d = network::new(&c).await.context("new network")?; + d.setup(self.hypervisor.as_ref()) + .await + .context("setup network")?; + self.network = Some(d) + } }; } Ok(()) } + async fn handle_interfaces(&self, network: &dyn Network) -> Result<()> { + for i in network.interfaces().await.context("get interfaces")? { + // update interface + info!(sl!(), "update interface {:?}", i); + self.agent + .update_interface(agent::UpdateInterfaceRequest { interface: Some(i) }) + .await + .context("update interface")?; + } + + Ok(()) + } + + async fn handle_neighbours(&self, network: &dyn Network) -> Result<()> { + let neighbors = network.neighs().await.context("neighs")?; + if !neighbors.is_empty() { + info!(sl!(), "update neighbors {:?}", neighbors); + self.agent + .add_arp_neighbors(agent::AddArpNeighborRequest { + neighbors: Some(agent::ARPNeighbors { neighbors }), + }) + .await + .context("update neighbors")?; + } + Ok(()) + } + + async fn handle_routes(&self, network: &dyn Network) -> Result<()> { + let routes = network.routes().await.context("routes")?; + if !routes.is_empty() { + info!(sl!(), "update routes {:?}", routes); + self.agent + .update_routes(agent::UpdateRoutesRequest { + route: Some(agent::Routes { routes }), + }) + .await + .context("update routes")?; + } + Ok(()) + } + pub async fn setup_after_start_vm(&mut self) -> Result<()> { if let Some(share_fs) = self.share_fs.as_ref() { share_fs @@ -80,6 +129,16 @@ impl ResourceManagerInner { .context("setup share fs device after start vm")?; } + if let Some(network) = self.network.as_ref() { + let network = network.as_ref(); + self.handle_interfaces(network) + .await + .context("handle interfaces")?; + self.handle_neighbours(network) + .await + .context("handle neighbors")?; + self.handle_routes(network).await.context("handle routes")?; + } Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs new file mode 100644 index 000000000000..caaa69dca611 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod physical_endpoint; +pub use physical_endpoint::PhysicalEndpoint; +mod veth_endpoint; +pub use veth_endpoint::VethEndpoint; + +use anyhow::Result; +use async_trait::async_trait; +use hypervisor::Hypervisor; + +#[async_trait] +pub trait Endpoint: std::fmt::Debug + Send + Sync { + async fn name(&self) -> String; + async fn hardware_addr(&self) -> String; + async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()>; + async fn detach(&self, hypervisor: &dyn Hypervisor) -> Result<()>; +} diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs new file mode 100644 index 000000000000..78387a5ce488 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs @@ -0,0 +1,145 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::Path; + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use hypervisor::{device, Hypervisor}; + +use super::Endpoint; +use crate::network::utils::{self, link}; + +pub const SYS_PCI_DEVICES_PATH: &str = "/sys/bus/pci/devices"; + +#[derive(Debug)] +pub struct VendorDevice { + vendor_id: String, + device_id: String, +} + +impl VendorDevice { + pub fn new(vendor_id: &str, device_id: &str) -> Result { + if vendor_id.is_empty() || device_id.is_empty() { + return Err(anyhow!( + "invalid parameters vendor_id {} device_id {}", + vendor_id, + device_id + )); + } + Ok(Self { + vendor_id: vendor_id.to_string(), + device_id: device_id.to_string(), + }) + } + + pub fn vendor_device_id(&self) -> String { + format!("{}_{}", &self.vendor_id, &self.device_id) + } +} + +#[derive(Debug)] +pub struct PhysicalEndpoint { + iface_name: String, + hard_addr: String, + bdf: String, + driver: String, + vendor_device_id: VendorDevice, +} + +impl PhysicalEndpoint { + pub fn new(name: &str, hardware_addr: &[u8]) -> Result { + let driver_info = link::get_driver_info(name).context("get driver info")?; + let bdf = driver_info.bus_info; + let sys_pci_devices_path = Path::new(SYS_PCI_DEVICES_PATH); + // get driver by following symlink /sys/bus/pci/devices/$bdf/driver + let driver_path = sys_pci_devices_path.join(&bdf).join("driver"); + let link = driver_path.read_link().context("read link")?; + let driver = link + .file_name() + .map_or(String::new(), |v| v.to_str().unwrap().to_owned()); + + // get vendor and device id from pci space (sys/bus/pci/devices/$bdf) + let iface_device_path = sys_pci_devices_path.join(&bdf).join("device"); + let device_id = std::fs::read_to_string(&iface_device_path) + .context(format!("read device path {:?}", &iface_device_path))?; + + let iface_vendor_path = sys_pci_devices_path.join(&bdf).join("vendor"); + let vendor_id = std::fs::read_to_string(&iface_vendor_path) + .context(format!("read vendor path {:?}", &iface_vendor_path))?; + + Ok(Self { + iface_name: name.to_string(), + hard_addr: utils::get_mac_addr(hardware_addr).context("get mac addr")?, + vendor_device_id: VendorDevice::new(&vendor_id, &device_id) + .context("new vendor device")?, + driver, + bdf, + }) + } +} + +#[async_trait] +impl Endpoint for PhysicalEndpoint { + async fn name(&self) -> String { + self.iface_name.clone() + } + + async fn hardware_addr(&self) -> String { + self.hard_addr.clone() + } + + async fn attach(&self, hypervisor: &dyn Hypervisor) -> Result<()> { + // bind physical interface from host driver and bind to vfio + device::bind_device_to_vfio( + &self.bdf, + &self.driver, + &self.vendor_device_id.vendor_device_id(), + ) + .context(format!( + "bind physical endpoint from {} to vfio", + &self.driver + ))?; + + // set vfio's bus type, pci or mmio. Mostly use pci by default. + let mode = match self.driver.as_str() { + "virtio-pci" => "mmio", + _ => "pci", + }; + + // add vfio device + let d = device::Device::Vfio(device::VfioConfig { + id: format!("physical_nic_{}", self.name().await), + sysfs_path: "".to_string(), + bus_slot_func: self.bdf.clone(), + mode: device::VfioBusMode::new(mode) + .context(format!("new vfio bus mode {:?}", mode))?, + }); + hypervisor.add_device(d).await.context("add device")?; + Ok(()) + } + + // detach for physical endpoint unbinds the physical network interface from vfio-pci + // and binds it back to the saved host driver. + async fn detach(&self, _hypervisor: &dyn Hypervisor) -> Result<()> { + // bind back the physical network interface to host. + // we need to do this even if a new network namespace has not + // been created by virt-containers. + + // we do not need to enter the network namespace to bind back the + // physical interface to host driver. + device::bind_device_to_host( + &self.bdf, + &self.driver, + &self.vendor_device_id.vendor_device_id(), + ) + .context(format!( + "bind physical endpoint device from vfio to {}", + &self.driver + ))?; + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs new file mode 100644 index 000000000000..c1bfb4c46450 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/veth_endpoint.rs @@ -0,0 +1,84 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{self, Error}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use hypervisor::{device::NetworkConfig, Device, Hypervisor}; + +use super::Endpoint; +use crate::network::{utils, NetworkPair}; + +#[derive(Debug)] +pub struct VethEndpoint { + net_pair: NetworkPair, +} + +impl VethEndpoint { + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + idx: u32, + model: &str, + queues: usize, + ) -> Result { + let net_pair = NetworkPair::new(handle, idx, name, model, queues) + .await + .context("new networkInterfacePair")?; + Ok(VethEndpoint { net_pair }) + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + Ok(NetworkConfig { + id: self.net_pair.virt_iface.name.clone(), + host_dev_name: iface.name.clone(), + guest_mac: Some(guest_mac), + }) + } +} + +#[async_trait] +impl Endpoint for VethEndpoint { + async fn name(&self) -> String { + self.net_pair.virt_iface.name.clone() + } + + async fn hardware_addr(&self) -> String { + self.net_pair.tap.tap_iface.hard_addr.clone() + } + + async fn attach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .add_network_model() + .await + .context("add network model")?; + let config = self.get_network_config().context("get network config")?; + h.add_device(Device::Network(config)) + .await + .context("Error add device")?; + Ok(()) + } + + async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .del_network_model() + .await + .context("del network model")?; + let config = self.get_network_config().context("get network config")?; + h.remove_device(Device::Network(config)) + .await + .context("remove device")?; + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/mod.rs b/src/runtime-rs/crates/resource/src/network/mod.rs new file mode 100644 index 000000000000..7193a6d92194 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/mod.rs @@ -0,0 +1,48 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod endpoint; +pub use endpoint::Endpoint; +mod network_entity; +mod network_info; +pub use network_info::NetworkInfo; +mod network_model; +pub use network_model::NetworkModel; +mod network_with_netns; +pub use network_with_netns::NetworkWithNetNsConfig; +use network_with_netns::NetworkWithNetns; +mod network_pair; +use network_pair::NetworkPair; +mod utils; + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use hypervisor::Hypervisor; + +#[derive(Debug)] +pub enum NetworkConfig { + NetworkResourceWithNetNs(NetworkWithNetNsConfig), +} + +#[async_trait] +pub trait Network: Send + Sync { + async fn setup(&self, h: &dyn Hypervisor) -> Result<()>; + async fn interfaces(&self) -> Result>; + async fn routes(&self) -> Result>; + async fn neighs(&self) -> Result>; +} + +pub async fn new(config: &NetworkConfig) -> Result> { + match config { + NetworkConfig::NetworkResourceWithNetNs(c) => Ok(Arc::new( + NetworkWithNetns::new(c) + .await + .context("new network with netns")?, + )), + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_entity.rs b/src/runtime-rs/crates/resource/src/network/network_entity.rs new file mode 100644 index 000000000000..5182dfe4b0ac --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_entity.rs @@ -0,0 +1,24 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::Arc; + +use super::{Endpoint, NetworkInfo}; + +#[derive(Debug)] +pub(crate) struct NetworkEntity { + pub(crate) endpoint: Arc, + pub(crate) network_info: Arc, +} + +impl NetworkEntity { + pub fn new(endpoint: Arc, network_info: Arc) -> Self { + Self { + endpoint, + network_info, + } + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_info/mod.rs b/src/runtime-rs/crates/resource/src/network/network_info/mod.rs new file mode 100644 index 000000000000..1500d5179ea0 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_info/mod.rs @@ -0,0 +1,18 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub(crate) mod network_info_from_link; + +use agent::{ARPNeighbor, Interface, Route}; +use anyhow::Result; +use async_trait::async_trait; + +#[async_trait] +pub trait NetworkInfo: std::fmt::Debug + Send + Sync { + async fn interface(&self) -> Result; + async fn routes(&self) -> Result>; + async fn neighs(&self) -> Result>; +} diff --git a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs new file mode 100644 index 000000000000..fa341a1ae619 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs @@ -0,0 +1,203 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{convert::TryFrom, net::Ipv4Addr}; + +use agent::{ARPNeighbor, IPAddress, IPFamily, Interface, Route}; +use anyhow::{Context, Result}; +use async_trait::async_trait; +use futures::stream::TryStreamExt; +use netlink_packet_route::{ + self, neighbour::NeighbourMessage, nlas::neighbour::Nla, route::RouteMessage, +}; + +use super::NetworkInfo; +use crate::network::utils::{ + address::Address, + link::{self, LinkAttrs}, +}; + +#[derive(Debug)] +pub(crate) struct NetworkInfoFromLink { + interface: Interface, + neighs: Vec, + routes: Vec, +} + +impl NetworkInfoFromLink { + pub async fn new( + handle: &rtnetlink::Handle, + link: &dyn link::Link, + hw_addr: &str, + ) -> Result { + let attrs = link.attrs(); + let name = &attrs.name; + + Ok(Self { + interface: Interface { + device: name.clone(), + name: name.clone(), + ip_addresses: handle_addresses(handle, attrs) + .await + .context("handle addresses")?, + mtu: attrs.mtu as u64, + hw_addr: hw_addr.to_string(), + pci_addr: Default::default(), + field_type: link.r#type().to_string(), + raw_flags: attrs.flags & libc::IFF_NOARP as u32, + }, + neighs: handle_neighbors(handle, attrs) + .await + .context("handle neighbours")?, + routes: handle_routes(handle, attrs) + .await + .context("handle routes")?, + }) + } +} + +async fn handle_addresses(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result> { + let mut addr_msg_list = handle + .address() + .get() + .set_link_index_filter(attrs.index) + .execute(); + + let mut addresses = Vec::new(); + while let Some(addr_msg) = addr_msg_list.try_next().await? { + if addr_msg.header.family as i32 != libc::AF_INET { + warn!(sl!(), "unsupported ipv6 addr. {:?}", addr_msg); + continue; + } + let a = Address::try_from(addr_msg).context("get addr from msg")?; + if a.addr.is_loopback() { + continue; + } + + addresses.push(IPAddress { + family: if a.addr.is_ipv4() { + IPFamily::V4 + } else { + IPFamily::V6 + }, + address: a.addr.to_string(), + mask: a.perfix_len.to_string(), + }); + } + Ok(addresses) +} + +fn generate_neigh(name: &str, n: &NeighbourMessage) -> Result { + let mut neigh = ARPNeighbor { + device: name.to_string(), + state: n.header.state as i32, + ..Default::default() + }; + for nla in &n.nlas { + match nla { + Nla::Destination(addr) => { + if addr.len() != 4 { + continue; + } + let dest = Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); + let addr = Some(IPAddress { + family: IPFamily::V4, + address: dest.to_string(), + mask: "".to_string(), + }); + neigh.to_ip_address = addr; + } + Nla::LinkLocalAddress(addr) => { + if addr.len() < 6 { + continue; + } + let lladdr = format!( + "{:<02x}:{:<02x}:{:<02x}:{:<02x}:{:<02x}:{:<02x}", + addr[0], addr[1], addr[2], addr[3], addr[4], addr[5] + ); + neigh.ll_addr = lladdr; + } + _ => { + // skip the unused Nla + } + } + } + + Ok(neigh) +} + +async fn handle_neighbors( + handle: &rtnetlink::Handle, + attrs: &LinkAttrs, +) -> Result> { + let name = &attrs.name; + let mut neighs = vec![]; + let mut neigh_msg_list = handle.neighbours().get().execute(); + while let Some(neigh) = neigh_msg_list.try_next().await? { + // get neigh filter with index + if neigh.header.ifindex == attrs.index { + neighs.push(generate_neigh(name, &neigh).context("generate neigh")?) + } + } + Ok(neighs) +} + +fn generate_route(name: &str, route: &RouteMessage) -> Result> { + if route.header.protocol == libc::RTPROT_KERNEL { + return Ok(None); + } + + Ok(Some(Route { + dest: route + .destination_prefix() + .map(|(addr, _)| addr.to_string()) + .unwrap_or_default(), + gateway: route.gateway().map(|v| v.to_string()).unwrap_or_default(), + device: name.to_string(), + source: route + .source_prefix() + .map(|(addr, _)| addr.to_string()) + .unwrap_or_default(), + scope: route.header.scope as u32, + family: if route.header.address_family == libc::AF_INET as u8 { + IPFamily::V4 + } else { + IPFamily::V6 + }, + })) +} + +async fn handle_routes(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result> { + let name = &attrs.name; + let mut routes = vec![]; + let mut route_msg_list = handle.route().get(rtnetlink::IpVersion::V4).execute(); + while let Some(route) = route_msg_list.try_next().await? { + // get route filter with index + if let Some(index) = route.output_interface() { + if index == attrs.index { + if let Some(route) = generate_route(name, &route).context("generate route")? { + routes.push(route); + } + } + } + } + Ok(routes) +} + +#[async_trait] +impl NetworkInfo for NetworkInfoFromLink { + async fn interface(&self) -> Result { + Ok(self.interface.clone()) + } + + async fn routes(&self) -> Result> { + Ok(self.routes.clone()) + } + + async fn neighs(&self) -> Result> { + Ok(self.neighs.clone()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_model/mod.rs b/src/runtime-rs/crates/resource/src/network/network_model/mod.rs new file mode 100644 index 000000000000..11cda538ca4d --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_model/mod.rs @@ -0,0 +1,46 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub mod none_model; +pub mod route_model; +pub mod tc_filter_model; + +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use super::NetworkPair; + +const TC_FILTER_NET_MODEL_STR: &str = "tcfilter"; +const ROUTE_NET_MODEL_STR: &str = "route"; + +pub enum NetworkModelType { + NoneModel, + TcFilter, + Route, +} + +#[async_trait] +pub trait NetworkModel: std::fmt::Debug + Send + Sync { + fn model_type(&self) -> NetworkModelType; + async fn add(&self, net_pair: &NetworkPair) -> Result<()>; + async fn del(&self, net_pair: &NetworkPair) -> Result<()>; +} + +pub fn new(model: &str) -> Result> { + match model { + TC_FILTER_NET_MODEL_STR => Ok(Arc::new( + tc_filter_model::TcFilterModel::new().context("new tc filter model")?, + )), + ROUTE_NET_MODEL_STR => Ok(Arc::new( + route_model::RouteModel::new().context("new route model")?, + )), + _ => Ok(Arc::new( + none_model::NoneModel::new().context("new none model")?, + )), + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_model/none_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/none_model.rs new file mode 100644 index 000000000000..f68b4d3e223f --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_model/none_model.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::Result; +use async_trait::async_trait; + +use super::{NetworkModel, NetworkModelType}; +use crate::network::NetworkPair; + +#[derive(Debug)] +pub(crate) struct NoneModel {} + +impl NoneModel { + pub fn new() -> Result { + Ok(Self {}) + } +} + +#[async_trait] +impl NetworkModel for NoneModel { + fn model_type(&self) -> NetworkModelType { + NetworkModelType::NoneModel + } + + async fn add(&self, _pair: &NetworkPair) -> Result<()> { + Ok(()) + } + + async fn del(&self, _pair: &NetworkPair) -> Result<()> { + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs new file mode 100644 index 000000000000..64b1da2e7497 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use tokio::process::Command; + +use super::{NetworkModel, NetworkModelType}; +use crate::network::NetworkPair; + +#[derive(Debug)] +pub(crate) struct RouteModel {} + +impl RouteModel { + pub fn new() -> Result { + Ok(Self {}) + } +} + +#[async_trait] +impl NetworkModel for RouteModel { + fn model_type(&self) -> NetworkModelType { + NetworkModelType::Route + } + + async fn add(&self, pair: &NetworkPair) -> Result<()> { + let tap_name = &pair.tap.tap_iface.name; + let virt_name = &pair.virt_iface.name; + let virt_iface_addr = pair.virt_iface.addrs[0].addr.to_string(); + + let commands_args = vec![ + vec![ + "rule", "add", "pref", "10", "from", "all", "lookup", "local", + ], + vec!["rule", "del", "pref", "0", "from", "all"], + vec!["rule", "add", "pref", "5", "iif", virt_name, "table", "10"], + vec![ + "route", "replace", "default", "dev", tap_name, "table", "10", + ], + vec![ + "neigh", + "replace", + &virt_iface_addr, + "lladdr", + &pair.virt_iface.hard_addr, + "dev", + tap_name, + ], + ]; + + for ca in commands_args { + let output = Command::new("/sbin/ip") + .args(&ca) + .output() + .await + .context(format!("run command ip args {:?}", &ca))?; + if !output.status.success() { + return Err(anyhow!( + "run command ip args {:?} error {}", + &ca, + String::from_utf8(output.stderr)? + )); + } + } + + // TODO: support ipv6 + // change sysctl for tap0_kata + // echo 1 > /proc/sys/net/ipv4/conf/tap0_kata/accept_local + let accept_local_path = format!("/proc/sys/net/ipv4/conf/{}/accept_local", &tap_name); + std::fs::write(&accept_local_path, "1".to_string()) + .with_context(|| format!("Failed to echo 1 > {}", &accept_local_path))?; + + // echo 1 > /proc/sys/net/ipv4/conf/eth0/proxy_arp + // This enabled ARP reply on peer eth0 to prevent without any reply on VPC + let proxy_arp_path = format!("/proc/sys/net/ipv4/conf/{}/proxy_arp", &virt_name); + std::fs::write(&proxy_arp_path, "1".to_string()) + .with_context(|| format!("Failed to echo 1 > {}", &proxy_arp_path))?; + + Ok(()) + } + + async fn del(&self, _pair: &NetworkPair) -> Result<()> { + todo!() + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs new file mode 100644 index 000000000000..ae347e717c42 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs @@ -0,0 +1,95 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use tokio::process::Command; + +use super::{NetworkModel, NetworkModelType}; +use crate::network::NetworkPair; + +#[derive(Debug)] +pub(crate) struct TcFilterModel {} + +impl TcFilterModel { + pub fn new() -> Result { + Ok(Self {}) + } +} + +#[async_trait] +impl NetworkModel for TcFilterModel { + fn model_type(&self) -> NetworkModelType { + NetworkModelType::TcFilter + } + + async fn add(&self, pair: &NetworkPair) -> Result<()> { + let tap_name = &pair.tap.tap_iface.name; + let virt_name = &pair.virt_iface.name; + + add_qdisc_ingress(tap_name) + .await + .context("add qdisc ingress for tap link")?; + add_qdisc_ingress(virt_name) + .await + .context("add qdisc ingress")?; + + add_redirect_tcfilter(tap_name, virt_name) + .await + .context("add tc filter for tap")?; + add_redirect_tcfilter(virt_name, tap_name) + .await + .context("add tc filter")?; + Ok(()) + } + + async fn del(&self, pair: &NetworkPair) -> Result<()> { + del_qdisc(&pair.virt_iface.name) + .await + .context("del qdisc")?; + Ok(()) + } +} + +// TODO: use netlink replace tc command +async fn add_qdisc_ingress(dev: &str) -> Result<()> { + let output = Command::new("/sbin/tc") + .args(&["qdisc", "add", "dev", dev, "handle", "ffff:", "ingress"]) + .output() + .await + .context("add tc")?; + if !output.status.success() { + return Err(anyhow!("{}", String::from_utf8(output.stderr)?)); + } + Ok(()) +} + +async fn add_redirect_tcfilter(src: &str, dst: &str) -> Result<()> { + let output = Command::new("/sbin/tc") + .args(&[ + "filter", "add", "dev", src, "parent", "ffff:", "protocol", "all", "u32", "match", + "u8", "0", "0", "action", "mirred", "egress", "redirect", "dev", dst, + ]) + .output() + .await + .context("add redirect tcfilter")?; + if !output.status.success() { + return Err(anyhow!("{}", String::from_utf8(output.stderr)?)); + } + Ok(()) +} + +async fn del_qdisc(dev: &str) -> Result<()> { + let output = Command::new("/sbin/tc") + .args(&["qdisc", "del", "dev", dev, "handle", "ffff:", "ingress"]) + .output() + .await + .context("del qdisc")?; + if !output.status.success() { + return Err(anyhow!("{}", String::from_utf8(output.stderr)?)); + } + Ok(()) +} diff --git a/src/runtime-rs/crates/resource/src/network/network_pair.rs b/src/runtime-rs/crates/resource/src/network/network_pair.rs new file mode 100644 index 000000000000..71e212907eb6 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_pair.rs @@ -0,0 +1,178 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{convert::TryFrom, sync::Arc, usize}; + +use anyhow::{anyhow, Context, Result}; +use futures::stream::TryStreamExt; + +use super::{ + network_model, + utils::{self, address::Address, link}, +}; + +const TAP_SUFFIX: &str = "_kata"; + +#[derive(Default, Copy, Clone, Debug, PartialEq, Eq)] +pub struct NetInterworkingModel(u32); + +#[derive(Default, Debug, Clone)] +pub struct NetworkInterface { + pub name: String, + pub hard_addr: String, + pub addrs: Vec
, +} + +#[derive(Default, Debug)] +pub struct TapInterface { + pub id: String, + pub name: String, + pub tap_iface: NetworkInterface, +} +#[derive(Debug)] +pub struct NetworkPair { + pub tap: TapInterface, + pub virt_iface: NetworkInterface, + pub model: Arc, + pub network_qos: bool, +} +impl NetworkPair { + pub(crate) async fn new( + handle: &rtnetlink::Handle, + idx: u32, + name: &str, + model: &str, + queues: usize, + ) -> Result { + let unique_id = kata_sys_util::rand::UUID::new(); + let model = network_model::new(model).context("new network model")?; + let tap_iface_name = format!("tap{}{}", idx, TAP_SUFFIX); + let virt_iface_name = format!("eth{}", idx); + let tap_link = create_link(handle, &tap_iface_name, queues) + .await + .context("create link")?; + let virt_link = get_link_by_name(handle, virt_iface_name.clone().as_str()) + .await + .context("get link by name")?; + + let mut virt_addr_msg_list = handle + .address() + .get() + .set_link_index_filter(virt_link.attrs().index) + .execute(); + + let mut virt_address = vec![]; + while let Some(addr_msg) = virt_addr_msg_list.try_next().await? { + let addr = Address::try_from(addr_msg).context("get address from msg")?; + virt_address.push(addr); + } + + // Save the veth MAC address to the TAP so that it can later be used + // to build the hypervisor command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + let tap_hard_addr = + utils::get_mac_addr(&virt_link.attrs().hardware_addr).context("get mac addr")?; + + // Save the TAP Mac address to the virt_iface so that it can later updated + // the guest's gateway IP's mac as this TAP device. This MAC address has + // to be inside the VM in order to the network reach to the gateway. + let virt_hard_addr = + utils::get_mac_addr(&tap_link.attrs().hardware_addr).context("get mac addr")?; + + handle + .link() + .set(tap_link.attrs().index) + .mtu(virt_link.attrs().mtu) + .execute() + .await + .context("set link mtu")?; + + handle + .link() + .set(tap_link.attrs().index) + .up() + .execute() + .await + .context("set link up")?; + + let mut net_pair = NetworkPair { + tap: TapInterface { + id: String::from(&unique_id), + name: format!("br{}{}", idx, TAP_SUFFIX), + tap_iface: NetworkInterface { + name: tap_iface_name, + hard_addr: tap_hard_addr, + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: virt_iface_name, + hard_addr: virt_hard_addr, + addrs: virt_address, + }, + model, + network_qos: false, + }; + + if !name.is_empty() { + net_pair.virt_iface.name = String::from(name); + } + + Ok(net_pair) + } + + pub(crate) async fn add_network_model(&self) -> Result<()> { + let model = self.model.clone(); + model.add(self).await.context("add")?; + Ok(()) + } + + pub(crate) async fn del_network_model(&self) -> Result<()> { + let model = self.model.clone(); + model.del(self).await.context("del")?; + Ok(()) + } +} + +pub async fn create_link( + handle: &rtnetlink::Handle, + name: &str, + queues: usize, +) -> Result> { + link::create_link(name, link::LinkType::Tap, queues)?; + + let link = get_link_by_name(handle, name) + .await + .context("get link by name")?; + + let base = link.attrs(); + if base.master_index != 0 { + handle + .link() + .set(base.index) + .master(base.master_index) + .execute() + .await + .context("set index")?; + } + Ok(link) +} + +pub async fn get_link_by_name( + handle: &rtnetlink::Handle, + name: &str, +) -> Result> { + let mut link_msg_list = handle.link().get().match_name(name.to_string()).execute(); + let msg = if let Some(msg) = link_msg_list.try_next().await? { + msg + } else { + return Err(anyhow!("failed to find link by name {}", name)); + }; + + Ok(link::get_link_from_message(msg)) +} diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs new file mode 100644 index 000000000000..c228a7c8a91b --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -0,0 +1,211 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, +}; + +use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; +use futures::stream::TryStreamExt; +use hypervisor::Hypervisor; +use scopeguard::defer; +use tokio::sync::RwLock; + +use super::{ + endpoint::{Endpoint, PhysicalEndpoint, VethEndpoint}, + network_entity::NetworkEntity, + network_info::network_info_from_link::NetworkInfoFromLink, + utils::{link, netns}, + Network, +}; +use crate::network::NetworkInfo; + +#[derive(Debug)] +pub struct NetworkWithNetNsConfig { + pub network_model: String, + pub netns_path: String, + pub queues: usize, +} + +struct NetworkWithNetnsInner { + netns_path: String, + entity_list: Vec, +} + +impl NetworkWithNetnsInner { + async fn new(config: &NetworkWithNetNsConfig) -> Result { + let entity_list = if config.netns_path.is_empty() { + warn!(sl!(), "skip to scan for empty netns"); + vec![] + } else { + // get endpoint + get_entity_from_netns(config) + .await + .context("get entity from netns")? + }; + Ok(Self { + netns_path: config.netns_path.to_string(), + entity_list, + }) + } +} + +pub(crate) struct NetworkWithNetns { + inner: Arc>, +} + +impl NetworkWithNetns { + pub(crate) async fn new(config: &NetworkWithNetNsConfig) -> Result { + Ok(Self { + inner: Arc::new(RwLock::new(NetworkWithNetnsInner::new(config).await?)), + }) + } +} + +#[async_trait] +impl Network for NetworkWithNetns { + async fn setup(&self, h: &dyn Hypervisor) -> Result<()> { + let inner = self.inner.read().await; + let _netns_guard = netns::NetnsGuard::new(&inner.netns_path).context("net netns guard")?; + for e in &inner.entity_list { + e.endpoint.attach(h).await.context("attach")?; + } + Ok(()) + } + + async fn interfaces(&self) -> Result> { + let inner = self.inner.read().await; + let mut interfaces = vec![]; + for e in &inner.entity_list { + interfaces.push(e.network_info.interface().await.context("interface")?); + } + Ok(interfaces) + } + + async fn routes(&self) -> Result> { + let inner = self.inner.read().await; + let mut routes = vec![]; + for e in &inner.entity_list { + let mut list = e.network_info.routes().await.context("routes")?; + routes.append(&mut list); + } + Ok(routes) + } + + async fn neighs(&self) -> Result> { + let inner = self.inner.read().await; + let mut neighs = vec![]; + for e in &inner.entity_list { + let mut list = e.network_info.neighs().await.context("neighs")?; + neighs.append(&mut list); + } + Ok(neighs) + } +} + +async fn get_entity_from_netns(config: &NetworkWithNetNsConfig) -> Result> { + info!( + sl!(), + "get network entity for config {:?} tid {:?}", + config, + nix::unistd::gettid() + ); + let mut entity_list = vec![]; + let _netns_guard = netns::NetnsGuard::new(&config.netns_path) + .context("net netns guard") + .unwrap(); + let (connection, handle, _) = rtnetlink::new_connection().context("new connection")?; + let thread_handler = tokio::spawn(connection); + defer!({ + thread_handler.abort(); + }); + + let mut links = handle.link().get().execute(); + + let idx = AtomicU32::new(0); + while let Some(link) = links.try_next().await? { + let link = link::get_link_from_message(link); + let attrs = link.attrs(); + + if (attrs.flags & libc::IFF_LOOPBACK as u32) != 0 { + continue; + } + + let idx = idx.fetch_add(1, Ordering::Relaxed); + let (endpoint, network_info) = create_endpoint(&handle, link.as_ref(), idx, config) + .await + .context("create endpoint")?; + + entity_list.push(NetworkEntity::new(endpoint, network_info)); + } + + Ok(entity_list) +} + +async fn create_endpoint( + handle: &rtnetlink::Handle, + link: &dyn link::Link, + idx: u32, + config: &NetworkWithNetNsConfig, +) -> Result<(Arc, Arc)> { + let _netns_guard = netns::NetnsGuard::new(&config.netns_path) + .context("net netns guard") + .unwrap(); + let attrs = link.attrs(); + let link_type = link.r#type(); + let endpoint: Arc = if is_physical_iface(&attrs.name)? { + info!( + sl!(), + "physical network interface found: {} {:?}", + &attrs.name, + nix::unistd::gettid() + ); + let t = PhysicalEndpoint::new(&attrs.name, &attrs.hardware_addr) + .context("new physical endpoint")?; + Arc::new(t) + } else { + info!( + sl!(), + "{} network interface found: {}", &link_type, &attrs.name + ); + match link_type { + "veth" => { + let ret = VethEndpoint::new( + handle, + &attrs.name, + idx, + &config.network_model, + config.queues, + ) + .await + .context("veth endpoint")?; + Arc::new(ret) + } + _ => return Err(anyhow!("unsupported link type: {}", link_type)), + } + }; + + let network_info = Arc::new( + NetworkInfoFromLink::new(handle, link, &endpoint.hardware_addr().await) + .await + .context("network info from link")?, + ); + + Ok((endpoint, network_info)) +} + +fn is_physical_iface(name: &str) -> Result { + if name == "lo" { + return Ok(false); + } + let driver_info = link::get_driver_info(name)?; + if driver_info.bus_info.split(':').count() != 3 { + return Ok(false); + } + Ok(true) +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/address.rs b/src/runtime-rs/crates/resource/src/network/utils/address.rs new file mode 100644 index 000000000000..916d011d585e --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/address.rs @@ -0,0 +1,109 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + convert::TryFrom, + net::{IpAddr, Ipv4Addr, Ipv6Addr}, +}; + +use anyhow::{anyhow, Result}; +use netlink_packet_route::{nlas::address::Nla, AddressMessage, AF_INET, AF_INET6}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Address { + pub addr: IpAddr, + pub label: String, + pub flags: u32, + pub scope: u8, + pub perfix_len: u8, + pub peer: IpAddr, + pub broadcast: IpAddr, + pub prefered_lft: u32, + pub valid_ltf: u32, +} + +impl TryFrom for Address { + type Error = anyhow::Error; + fn try_from(msg: AddressMessage) -> Result { + let AddressMessage { header, nlas } = msg; + let mut addr = Address { + addr: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + peer: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + broadcast: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + label: String::default(), + flags: 0, + scope: header.scope, + perfix_len: header.prefix_len, + prefered_lft: 0, + valid_ltf: 0, + }; + + let mut local = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); + let mut dst = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); + + for nla in nlas.into_iter() { + match nla { + Nla::Address(a) => { + dst = parse_ip(a, header.family)?; + } + Nla::Local(a) => { + local = parse_ip(a, header.family)?; + } + Nla::Broadcast(b) => { + addr.broadcast = parse_ip(b, header.family)?; + } + Nla::Label(l) => { + addr.label = l; + } + Nla::Flags(f) => { + addr.flags = f; + } + Nla::CacheInfo(_c) => {} + _ => {} + } + } + + // IPv6 sends the local address as IFA_ADDRESS with no + // IFA_LOCAL, IPv4 sends both IFA_LOCAL and IFA_ADDRESS + // with IFA_ADDRESS being the peer address if they differ + // + // But obviously, as there are IPv6 PtP addresses, too, + // IFA_LOCAL should also be handled for IPv6. + if local.is_unspecified() { + if header.family == AF_INET as u8 && local == dst { + addr.addr = dst; + } else { + addr.addr = local; + addr.peer = dst; + } + } else { + addr.addr = dst; + } + Ok(addr) + } +} + +fn parse_ip(ip: Vec, family: u8) -> Result { + let support_len = if family as u16 == AF_INET { 4 } else { 16 }; + if ip.len() != support_len { + return Err(anyhow!( + "invalid ip addresses {:?} support {}", + &ip, + support_len + )); + } + match family as u16 { + AF_INET => Ok(IpAddr::V4(Ipv4Addr::new(ip[0], ip[1], ip[2], ip[3]))), + AF_INET6 => { + let mut octets = [0u8; 16]; + octets.copy_from_slice(&ip[..16]); + Ok(IpAddr::V6(Ipv6Addr::from(octets))) + } + _ => { + return Err(anyhow!("unknown IP network family {}", family)); + } + } +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/create.rs b/src/runtime-rs/crates/resource/src/network/utils/link/create.rs new file mode 100644 index 000000000000..06bedf79b9a1 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/link/create.rs @@ -0,0 +1,129 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + fs::{File, OpenOptions}, + os::unix::io::AsRawFd, + path::Path, + {io, mem}, +}; + +use anyhow::{Context, Result}; +use nix::ioctl_write_ptr; + +use super::macros::{get_name, set_name}; + +type IfName = [u8; libc::IFNAMSIZ]; + +#[derive(Copy, Clone, Debug)] +#[repr(C)] +struct CreateLinkMap { + pub mem_start: libc::c_ulong, + pub mem_end: libc::c_ulong, + pub base_addr: libc::c_ushort, + pub irq: libc::c_uchar, + pub dma: libc::c_uchar, + pub port: libc::c_uchar, +} + +#[repr(C)] +union CreateLinkIfru { + pub ifr_addr: libc::sockaddr, + pub ifr_dst_addr: libc::sockaddr, + pub ifr_broad_addr: libc::sockaddr, + pub ifr_netmask: libc::sockaddr, + pub ifr_hw_addr: libc::sockaddr, + pub ifr_flags: libc::c_short, + pub ifr_if_index: libc::c_int, + pub ifr_metric: libc::c_int, + pub ifr_mtu: libc::c_int, + pub ifr_map: CreateLinkMap, + pub ifr_slave: IfName, + pub ifr_new_name: IfName, + pub ifr_data: *mut libc::c_char, +} + +#[repr(C)] +struct CreateLinkReq { + pub ifr_name: IfName, + pub ifr_ifru: CreateLinkIfru, +} + +impl CreateLinkReq { + pub fn from_name(name: &str) -> io::Result { + let mut req: CreateLinkReq = unsafe { mem::zeroed() }; + req.set_name(name)?; + Ok(req) + } + + pub fn set_name(&mut self, name: &str) -> io::Result<()> { + set_name!(self.ifr_name, name) + } + + pub fn get_name(&self) -> io::Result { + get_name!(self.ifr_name) + } + + pub unsafe fn set_raw_flags(&mut self, raw_flags: libc::c_short) { + self.ifr_ifru.ifr_flags = raw_flags; + } +} + +const DEVICE_PATH: &str = "/dev/net/tun"; + +ioctl_write_ptr!(tun_set_iff, b'T', 202, libc::c_int); +ioctl_write_ptr!(tun_set_persist, b'T', 203, libc::c_int); + +#[derive(Clone, Copy, Debug)] +pub enum LinkType { + #[allow(dead_code)] + Tun, + Tap, +} + +pub fn create_link(name: &str, link_type: LinkType, queues: usize) -> Result<()> { + let mut flags = libc::IFF_VNET_HDR; + flags |= match link_type { + LinkType::Tun => libc::IFF_TUN, + LinkType::Tap => libc::IFF_TAP, + }; + + let queues = if queues == 0 { 1 } else { queues }; + if queues > 1 { + flags |= libc::IFF_MULTI_QUEUE | libc::IFF_NO_PI; + } else { + flags |= libc::IFF_ONE_QUEUE; + }; + + // create first queue + let mut files = vec![]; + let (file, result_name) = create_queue(name, flags)?; + unsafe { + tun_set_persist(file.as_raw_fd(), &1).context("tun set persist")?; + } + files.push(file); + + // create other queues + if queues > 1 { + for _ in 0..queues - 1 { + files.push(create_queue(&result_name, flags)?.0); + } + } + + info!(sl!(), "create link with fds {:?}", files); + Ok(()) +} + +fn create_queue(name: &str, flags: libc::c_int) -> Result<(File, String)> { + let path = Path::new(DEVICE_PATH); + let file = OpenOptions::new().read(true).write(true).open(&path)?; + let mut req = CreateLinkReq::from_name(name)?; + unsafe { + req.set_raw_flags(flags as libc::c_short); + tun_set_iff(file.as_raw_fd(), &mut req as *mut _ as *mut _).context("tun set iff")?; + }; + Ok((file, req.get_name()?)) +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/driver_info.rs b/src/runtime-rs/crates/resource/src/network/utils/link/driver_info.rs new file mode 100644 index 000000000000..a7269d013ad6 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/link/driver_info.rs @@ -0,0 +1,102 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{io, mem}; + +use anyhow::{Context, Result}; +use nix::sys::socket::{socket, AddressFamily, SockFlag, SockType}; +use scopeguard::defer; + +use super::macros::{get_name, set_name}; + +/// FW version length +const ETHTOOL_FW_VERSION_LEN: usize = 32; + +/// bus info length +const ETHTOOL_BUS_INFO_LEN: usize = 32; + +/// erom version length +const ETHTOOL_EROM_VERSION_LEN: usize = 32; + +/// driver info +const ETHTOOL_DRIVER_INFO: u32 = 0x00000003; + +/// Ethtool interface define 0x8946 +const IOCTL_ETHTOOL_INTERFACE: u32 = 0x8946; + +nix::ioctl_readwrite_bad!(ioctl_ethtool, IOCTL_ETHTOOL_INTERFACE, DeviceInfoReq); + +#[repr(C)] +pub union DeviceInfoIfru { + pub ifr_addr: libc::sockaddr, + pub ifr_data: *mut libc::c_char, +} + +type IfName = [u8; libc::IFNAMSIZ]; + +#[repr(C)] +pub struct DeviceInfoReq { + pub ifr_name: IfName, + pub ifr_ifru: DeviceInfoIfru, +} + +impl DeviceInfoReq { + pub fn from_name(name: &str) -> io::Result { + let mut req: DeviceInfoReq = unsafe { mem::zeroed() }; + req.set_name(name)?; + Ok(req) + } + + pub fn set_name(&mut self, name: &str) -> io::Result<()> { + set_name!(self.ifr_name, name) + } +} + +#[repr(C)] +#[derive(Debug, Clone)] +struct Driver { + pub cmd: u32, + pub driver: [u8; 32], + pub version: [u8; 32], + pub fw_version: [u8; ETHTOOL_FW_VERSION_LEN], + pub bus_info: [u8; ETHTOOL_BUS_INFO_LEN], + pub erom_version: [u8; ETHTOOL_EROM_VERSION_LEN], + pub reserved2: [u8; 12], + pub n_priv_flags: u32, + pub n_stats: u32, + pub test_info_len: u32, + pub eedump_len: u32, + pub regdump_len: u32, +} + +#[derive(Debug, Clone)] +pub struct DriverInfo { + pub driver: String, + pub bus_info: String, +} + +pub fn get_driver_info(name: &str) -> Result { + let mut req = DeviceInfoReq::from_name(name).context(format!("ifreq from name {}", name))?; + let mut ereq: Driver = unsafe { mem::zeroed() }; + ereq.cmd = ETHTOOL_DRIVER_INFO; + req.ifr_ifru.ifr_data = &mut ereq as *mut _ as *mut _; + + let fd = socket( + AddressFamily::Inet, + SockType::Datagram, + SockFlag::empty(), + None, + ) + .context("new socket")?; + defer!({ + let _ = nix::unistd::close(fd); + }); + unsafe { ioctl_ethtool(fd, &mut req).context("ioctl ethtool")? }; + Ok(DriverInfo { + driver: get_name!(ereq.driver).context("get driver name")?, + bus_info: get_name!(ereq.bus_info).context("get bus info name")?, + }) +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/macros.rs b/src/runtime-rs/crates/resource/src/network/utils/link/macros.rs new file mode 100644 index 000000000000..128a76bb29a7 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/link/macros.rs @@ -0,0 +1,48 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +macro_rules! set_name { + ($name_field:expr, $name_str:expr) => {{ + let name_c = &::std::ffi::CString::new($name_str.to_owned()).map_err(|_| { + ::std::io::Error::new( + ::std::io::ErrorKind::InvalidInput, + "malformed interface name", + ) + })?; + let name_slice = name_c.as_bytes_with_nul(); + if name_slice.len() > libc::IFNAMSIZ { + return Err(io::Error::new(::std::io::ErrorKind::InvalidInput, "").into()); + } + $name_field[..name_slice.len()].clone_from_slice(name_slice); + + Ok(()) + }}; +} + +macro_rules! get_name { + ($name_field:expr) => {{ + let nul_pos = match $name_field.iter().position(|x| *x == 0) { + Some(p) => p, + None => { + return Err(::std::io::Error::new( + ::std::io::ErrorKind::InvalidData, + "malformed interface name", + ) + .into()) + } + }; + + std::ffi::CString::new(&$name_field[..nul_pos]) + .unwrap() + .into_string() + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "malformed interface name") + }) + }}; +} + +pub(crate) use get_name; +pub(crate) use set_name; diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs b/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs new file mode 100644 index 000000000000..fd4a8b1c99fe --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs @@ -0,0 +1,288 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use netlink_packet_route::{ + link::nlas::{Info, InfoBridge, InfoData, InfoKind, Nla}, + LinkMessage, +}; + +use super::{Link, LinkAttrs}; + +pub fn get_link_from_message(mut msg: LinkMessage) -> Box { + let mut base = LinkAttrs { + index: msg.header.index, + flags: msg.header.flags, + link_layer_type: msg.header.link_layer_type, + ..Default::default() + }; + if msg.header.flags & libc::IFF_PROMISC as u32 != 0 { + base.promisc = 1; + } + let mut link: Option> = None; + while let Some(attr) = msg.nlas.pop() { + match attr { + Nla::Info(infos) => { + link = Some(link_info(infos)); + } + Nla::Address(a) => { + base.hardware_addr = a; + } + Nla::IfName(i) => { + base.name = i; + } + Nla::Mtu(m) => { + base.mtu = m; + } + Nla::Link(l) => { + base.parent_index = l; + } + Nla::Master(m) => { + base.master_index = m; + } + Nla::TxQueueLen(t) => { + base.txq_len = t; + } + Nla::IfAlias(a) => { + base.alias = a; + } + Nla::Stats(_s) => {} + Nla::Stats64(_s) => {} + Nla::Xdp(_x) => {} + Nla::ProtoInfo(_) => {} + Nla::OperState(_) => {} + Nla::NetnsId(n) => { + base.net_ns_id = n; + } + Nla::GsoMaxSize(i) => { + base.gso_max_size = i; + } + Nla::GsoMaxSegs(e) => { + base.gso_max_seqs = e; + } + Nla::VfInfoList(_) => {} + Nla::NumTxQueues(t) => { + base.num_tx_queues = t; + } + Nla::NumRxQueues(r) => { + base.num_rx_queues = r; + } + Nla::Group(g) => { + base.group = g; + } + _ => { + // skip unused attr + } + } + } + + let mut ret = link.unwrap_or_else(|| Box::new(Device::default())); + ret.set_attrs(base); + ret +} + +fn link_info(mut infos: Vec) -> Box { + let mut link: Option> = None; + while let Some(info) = infos.pop() { + match info { + Info::Kind(kind) => match kind { + InfoKind::Tun => { + if link.is_none() { + link = Some(Box::new(Tuntap::default())); + } + } + InfoKind::Veth => { + if link.is_none() { + link = Some(Box::new(Veth::default())); + } + } + InfoKind::IpVlan => { + if link.is_none() { + link = Some(Box::new(IpVlan::default())); + } + } + InfoKind::Vlan => { + if link.is_none() { + link = Some(Box::new(Vlan::default())); + } + } + InfoKind::Bridge => { + if link.is_none() { + link = Some(Box::new(Bridge::default())); + } + } + _ => { + if link.is_none() { + link = Some(Box::new(Device::default())); + } + } + }, + Info::Data(data) => match data { + InfoData::Tun(_) => { + link = Some(Box::new(Tuntap::default())); + } + InfoData::Veth(_) => { + link = Some(Box::new(Veth::default())); + } + InfoData::IpVlan(_) => { + link = Some(Box::new(IpVlan::default())); + } + InfoData::Vlan(_) => { + link = Some(Box::new(Vlan::default())); + } + InfoData::Bridge(ibs) => { + link = Some(Box::new(parse_bridge(ibs))); + } + _ => { + link = Some(Box::new(Device::default())); + } + }, + Info::SlaveKind(_sk) => { + if link.is_none() { + link = Some(Box::new(Device::default())); + } + } + Info::SlaveData(_sd) => { + link = Some(Box::new(Device::default())); + } + _ => { + link = Some(Box::new(Device::default())); + } + } + } + link.unwrap() +} + +fn parse_bridge(mut ibs: Vec) -> Bridge { + let mut bridge = Bridge::default(); + while let Some(ib) = ibs.pop() { + match ib { + InfoBridge::HelloTime(ht) => { + bridge.hello_time = ht; + } + InfoBridge::MulticastSnooping(m) => { + bridge.multicast_snooping = m == 1; + } + InfoBridge::VlanFiltering(v) => { + bridge.vlan_filtering = v == 1; + } + _ => {} + } + } + bridge +} +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct Device { + attrs: Option, +} + +impl Link for Device { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr); + } + fn r#type(&self) -> &'static str { + "device" + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct Tuntap { + pub attrs: Option, +} + +impl Link for Tuntap { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr); + } + fn r#type(&self) -> &'static str { + "tuntap" + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct Veth { + attrs: Option, + + /// on create only + pub peer_name: String, +} + +impl Link for Veth { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr); + } + fn r#type(&self) -> &'static str { + "veth" + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct IpVlan { + attrs: Option, + + /// on create only + pub peer_name: String, +} + +impl Link for IpVlan { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr); + } + fn r#type(&self) -> &'static str { + "ipvlan" + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct Vlan { + attrs: Option, + + /// on create only + pub peer_name: String, +} + +impl Link for Vlan { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr); + } + fn r#type(&self) -> &'static str { + "vlan" + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct Bridge { + attrs: Option, + pub multicast_snooping: bool, + pub hello_time: u32, + pub vlan_filtering: bool, +} + +impl Link for Bridge { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr); + } + fn r#type(&self) -> &'static str { + "bridge" + } +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs b/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs new file mode 100644 index 000000000000..9fcc2b640508 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/link/mod.rs @@ -0,0 +1,145 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod create; +pub use create::{create_link, LinkType}; +mod driver_info; +pub use driver_info::{get_driver_info, DriverInfo}; +mod macros; +mod manager; +pub use manager::get_link_from_message; + +use std::os::unix::io::RawFd; + +use netlink_packet_route::link::nlas::State; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Namespace { + NetNsPid(u32), + #[allow(dead_code)] + NetNsFd(RawFd), +} +impl Default for Namespace { + fn default() -> Self { + Self::NetNsPid(0) + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum LinkStatistics { + #[allow(dead_code)] + Stats(LinkStatistics32), + Stats64(LinkStatistics64), +} +impl Default for LinkStatistics { + fn default() -> Self { + Self::Stats64(LinkStatistics64::default()) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct LinkStatistics32 { + pub rx_packets: u32, + pub tx_packets: u32, + pub rx_bytes: u32, + pub tx_bytes: u32, + pub rx_errors: u32, + pub tx_errors: u32, + pub rx_dropped: u32, + pub tx_dropped: u32, + pub multicast: u32, + pub collisions: u32, + pub rx_length_errors: u32, + pub rx_over_errors: u32, + pub rx_crc_errors: u32, + pub rx_frame_errors: u32, + pub rx_fifo_errors: u32, + pub rx_missed_errors: u32, + pub tx_aborted_errors: u32, + pub tx_carrier_errors: u32, + pub tx_fifo_errors: u32, + pub tx_heartbeat_errors: u32, + pub tx_window_errors: u32, + pub rx_compressed: u32, + pub tx_compressed: u32, +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct LinkStatistics64 { + pub rx_packets: u64, + pub tx_packets: u64, + pub rx_bytes: u64, + pub tx_bytes: u64, + pub rx_errors: u64, + pub tx_errors: u64, + pub rx_dropped: u64, + pub tx_dropped: u64, + pub multicast: u64, + pub collisions: u64, + pub rx_length_errors: u64, + pub rx_over_errors: u64, + pub rx_crc_errors: u64, + pub rx_frame_errors: u64, + pub rx_fifo_errors: u64, + pub rx_missed_errors: u64, + pub tx_aborted_errors: u64, + pub tx_carrier_errors: u64, + pub tx_fifo_errors: u64, + pub tx_heartbeat_errors: u64, + pub tx_window_errors: u64, + pub rx_compressed: u64, + pub tx_compressed: u64, +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct LinkXdp { + pub fd: RawFd, + pub attached: bool, + pub flags: u32, + pub prog_id: u32, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct OperState(State); +impl Default for OperState { + fn default() -> Self { + Self(State::Unknown) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct LinkAttrs { + pub index: u32, + pub mtu: u32, + pub txq_len: u32, + + pub name: String, + pub hardware_addr: Vec, + pub flags: u32, + pub parent_index: u32, + pub master_index: u32, + pub namespace: Namespace, + pub alias: String, + pub statistics: LinkStatistics, + pub promisc: u32, + pub xdp: LinkXdp, + pub link_layer_type: u16, + pub proto_info: Vec, + pub oper_state: OperState, + pub net_ns_id: i32, + pub num_tx_queues: u32, + pub num_rx_queues: u32, + pub gso_max_size: u32, + pub gso_max_seqs: u32, + pub vfs: Vec, + pub group: u32, +} + +pub trait Link: Send + Sync { + fn attrs(&self) -> &LinkAttrs; + fn set_attrs(&mut self, attr: LinkAttrs); + fn r#type(&self) -> &str; +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/mod.rs b/src/runtime-rs/crates/resource/src/network/utils/mod.rs new file mode 100644 index 000000000000..574178c3de4d --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/mod.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +pub(crate) mod address; +pub(crate) mod link; +pub(crate) mod netns; + +use anyhow::{anyhow, Result}; + +pub(crate) fn parse_mac(s: &str) -> Option { + let v: Vec<_> = s.split(':').collect(); + if v.len() != 6 { + return None; + } + let mut bytes = [0u8; 6]; + for i in 0..6 { + bytes[i] = u8::from_str_radix(v[i], 16).ok()?; + } + + Some(hypervisor::Address(bytes)) +} + +pub(crate) fn get_mac_addr(b: &[u8]) -> Result { + if b.len() != 6 { + return Err(anyhow!("invalid mac address {:?}", b)); + } else { + Ok(format!( + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + b[0], b[1], b[2], b[3], b[4], b[5] + )) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/utils/netns.rs b/src/runtime-rs/crates/resource/src/network/utils/netns.rs new file mode 100644 index 000000000000..1377a785943d --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/utils/netns.rs @@ -0,0 +1,51 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{fs::File, os::unix::io::AsRawFd}; + +use anyhow::{Context, Result}; +use nix::sched::{setns, CloneFlags}; +use nix::unistd::{getpid, gettid}; + +pub(crate) struct NetnsGuard { + old_netns: Option, +} + +impl NetnsGuard { + pub(crate) fn new(new_netns_path: &str) -> Result { + let old_netns = if !new_netns_path.is_empty() { + let current_netns_path = format!("/proc/{}/task/{}/ns/{}", getpid(), gettid(), "net"); + let old_netns = File::open(¤t_netns_path) + .context(format!("open current netns path {}", ¤t_netns_path))?; + let new_netns = File::open(&new_netns_path) + .context(format!("open new netns path {}", &new_netns_path))?; + setns(new_netns.as_raw_fd(), CloneFlags::CLONE_NEWNET) + .context("set netns to new netns")?; + info!( + sl!(), + "set netns from old {:?} to new {:?} tid {}", + old_netns, + new_netns, + gettid().to_string() + ); + Some(old_netns) + } else { + warn!(sl!(), "skip to set netns for empty netns path"); + None + }; + Ok(Self { old_netns }) + } +} + +impl Drop for NetnsGuard { + fn drop(&mut self) { + if let Some(old_netns) = self.old_netns.as_ref() { + let old_netns_fd = old_netns.as_raw_fd(); + setns(old_netns_fd, CloneFlags::CLONE_NEWNET).unwrap(); + info!(sl!(), "set netns to old {:?}", old_netns_fd); + } + } +} diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs index 4063c8bf8205..7ea27fe0d611 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs @@ -11,7 +11,6 @@ use std::{sync::Arc, vec::Vec}; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use kata_types::mount::Mount; -use log::{error, info}; use nix::sys::stat::{self, SFlag}; use tokio::sync::RwLock; @@ -90,6 +89,7 @@ impl RootFsResource { let inner = self.inner.read().await; for r in &inner.rootfs { info!( + sl!(), "rootfs {:?}: count {}", r.get_guest_rootfs_path().await, Arc::strong_count(r) @@ -114,7 +114,7 @@ fn get_block_device(file_path: &str) -> Option { } } Err(err) => { - error!("failed to stat for {} {:?}", file_path, err); + error!(sl!(), "failed to stat for {} {:?}", file_path, err); return None; } }; diff --git a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs index 4fa6b341f183..9bf02ddc4f4a 100644 --- a/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs @@ -7,7 +7,6 @@ use std::{path::Path, sync::Arc}; use anyhow::{anyhow, Context, Result}; -use log::debug; use nix::sys::stat::{stat, SFlag}; use super::Volume; @@ -53,6 +52,7 @@ impl ShareFsVolume { | SFlag::S_IFREG; if !file_type.contains(SFlag::from_bits_truncate(stat.st_mode)) { debug!( + sl!(), "Ignoring non-regular file as FS sharing not supported. mount: {:?}", m ); From 06f398a34f2d10ef2009d0326e1b05a764ad4bcb Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Thu, 19 May 2022 14:52:31 +0800 Subject: [PATCH 0050/1953] runtime-rs: use withContext to evaluate lazily Fixes: #4129 Signed-off-by: Zhongtao Hu --- .../crates/hypervisor/src/device/vfio.rs | 4 ++-- .../src/network/endpoint/physical_endpoint.rs | 21 +++++++++---------- .../src/network/network_model/route_model.rs | 2 +- .../resource/src/network/utils/netns.rs | 6 +++--- .../crates/resource/src/share_fs/utils.rs | 2 +- .../crates/resource/src/volume/mod.rs | 8 +++---- versions.yaml | 10 +++++++++ 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/vfio.rs index db1a99fb3fad..5e62d4549c9a 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/vfio.rs @@ -11,7 +11,7 @@ use anyhow::{anyhow, Context, Result}; fn override_driver(bdf: &str, driver: &str) -> Result<()> { let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf); fs::write(&driver_override, driver) - .context(format!("echo {} > {}", driver, &driver_override))?; + .with_context(|| format!("echo {} > {}", driver, &driver_override))?; info!(sl!(), "echo {} > {}", driver, driver_override); Ok(()) } @@ -138,7 +138,7 @@ pub fn bind_device_to_host(bdf: &str, host_driver: &str, _vendor_device_id: &str // echo bdf > /sys/bus/pci/drivers_probe std::fs::write(PCI_DRIVER_PROBE, bdf) - .context(format!("echo {} > {}", bdf, PCI_DRIVER_PROBE))?; + .with_context(|| format!("echo {} > {}", bdf, PCI_DRIVER_PROBE))?; info!(sl!(), "echo {} > {}", bdf, PCI_DRIVER_PROBE); Ok(()) diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs index 78387a5ce488..ffdfb5848b90 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/physical_endpoint.rs @@ -65,11 +65,11 @@ impl PhysicalEndpoint { // get vendor and device id from pci space (sys/bus/pci/devices/$bdf) let iface_device_path = sys_pci_devices_path.join(&bdf).join("device"); let device_id = std::fs::read_to_string(&iface_device_path) - .context(format!("read device path {:?}", &iface_device_path))?; + .with_context(|| format!("read device path {:?}", &iface_device_path))?; let iface_vendor_path = sys_pci_devices_path.join(&bdf).join("vendor"); let vendor_id = std::fs::read_to_string(&iface_vendor_path) - .context(format!("read vendor path {:?}", &iface_vendor_path))?; + .with_context(|| format!("read vendor path {:?}", &iface_vendor_path))?; Ok(Self { iface_name: name.to_string(), @@ -99,10 +99,7 @@ impl Endpoint for PhysicalEndpoint { &self.driver, &self.vendor_device_id.vendor_device_id(), ) - .context(format!( - "bind physical endpoint from {} to vfio", - &self.driver - ))?; + .with_context(|| format!("bind physical endpoint from {} to vfio", &self.driver))?; // set vfio's bus type, pci or mmio. Mostly use pci by default. let mode = match self.driver.as_str() { @@ -116,7 +113,7 @@ impl Endpoint for PhysicalEndpoint { sysfs_path: "".to_string(), bus_slot_func: self.bdf.clone(), mode: device::VfioBusMode::new(mode) - .context(format!("new vfio bus mode {:?}", mode))?, + .with_context(|| format!("new vfio bus mode {:?}", mode))?, }); hypervisor.add_device(d).await.context("add device")?; Ok(()) @@ -136,10 +133,12 @@ impl Endpoint for PhysicalEndpoint { &self.driver, &self.vendor_device_id.vendor_device_id(), ) - .context(format!( - "bind physical endpoint device from vfio to {}", - &self.driver - ))?; + .with_context(|| { + format!( + "bind physical endpoint device from vfio to {}", + &self.driver + ) + })?; Ok(()) } } diff --git a/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs index 64b1da2e7497..5955af0ff41d 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs @@ -56,7 +56,7 @@ impl NetworkModel for RouteModel { .args(&ca) .output() .await - .context(format!("run command ip args {:?}", &ca))?; + .with_context(|| format!("run command ip args {:?}", &ca))?; if !output.status.success() { return Err(anyhow!( "run command ip args {:?} error {}", diff --git a/src/runtime-rs/crates/resource/src/network/utils/netns.rs b/src/runtime-rs/crates/resource/src/network/utils/netns.rs index 1377a785943d..a2a29dc97107 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/netns.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/netns.rs @@ -19,11 +19,11 @@ impl NetnsGuard { let old_netns = if !new_netns_path.is_empty() { let current_netns_path = format!("/proc/{}/task/{}/ns/{}", getpid(), gettid(), "net"); let old_netns = File::open(¤t_netns_path) - .context(format!("open current netns path {}", ¤t_netns_path))?; + .with_context(|| format!("open current netns path {}", ¤t_netns_path))?; let new_netns = File::open(&new_netns_path) - .context(format!("open new netns path {}", &new_netns_path))?; + .with_context(|| format!("open new netns path {}", &new_netns_path))?; setns(new_netns.as_raw_fd(), CloneFlags::CLONE_NEWNET) - .context("set netns to new netns")?; + .with_context(|| "set netns to new netns")?; info!( sl!(), "set netns from old {:?} to new {:?} tid {}", diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs index 3a4b0c74394d..bd90d6bd9b51 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -30,7 +30,7 @@ pub(crate) fn share_to_guest( ) -> Result { let host_dest = do_get_host_path(target, sid, cid, is_volume, false); mount::bind_mount_unchecked(source, &host_dest, readonly) - .context(format!("failed to bind mount {} to {}", source, &host_dest))?; + .with_context(|| format!("failed to bind mount {} to {}", source, &host_dest))?; // bind mount remount event is not propagated to mount subtrees, so we have // to remount the read only dir mount point directly. diff --git a/src/runtime-rs/crates/resource/src/volume/mod.rs b/src/runtime-rs/crates/resource/src/volume/mod.rs index 2ad12f119608..53c737c79cbb 100644 --- a/src/runtime-rs/crates/resource/src/volume/mod.rs +++ b/src/runtime-rs/crates/resource/src/volume/mod.rs @@ -49,18 +49,18 @@ impl VolumeResource { let shm_size = shm_volume::DEFAULT_SHM_SIZE; Arc::new( shm_volume::ShmVolume::new(m, shm_size) - .context(format!("new shm volume {:?}", m))?, + .with_context(|| format!("new shm volume {:?}", m))?, ) } else if share_fs_volume::is_share_fs_volume(m) { Arc::new( share_fs_volume::ShareFsVolume::new(share_fs, m, cid) .await - .context(format!("new share fs volume {:?}", m))?, + .with_context(|| format!("new share fs volume {:?}", m))?, ) } else if block_volume::is_block_volume(m) { Arc::new( block_volume::BlockVolume::new(m) - .context(format!("new block volume {:?}", m))?, + .with_context(|| format!("new block volume {:?}", m))?, ) } else if is_skip_volume(m) { info!(sl!(), "skip volume {:?}", m); @@ -68,7 +68,7 @@ impl VolumeResource { } else { Arc::new( default_volume::DefaultVolume::new(m) - .context(format!("new default volume {:?}", m))?, + .with_context(|| format!("new default volume {:?}", m))?, ) }; diff --git a/versions.yaml b/versions.yaml index 34723fd857ab..07f1bcce5761 100644 --- a/versions.yaml +++ b/versions.yaml @@ -282,6 +282,16 @@ languages: building Kata newest-version: "1.58.1" + golangci-lint: + description: "golangci-lint" + notes: "'version' is the default minimum version used by this project." + version: "1.41.1" + meta: + description: | + 'newest-version' is the latest version known to work when + building Kata + newest-version: "1.46.2" + specs: description: "Details of important specifications" From ff7874bc2305e545626356431cbf768f4dfee6c4 Mon Sep 17 00:00:00 2001 From: Fupan Li Date: Sat, 11 Jun 2022 09:53:54 +0800 Subject: [PATCH 0051/1953] protobuf: upgrade the protobuf version to 2.27.0 Signed-off-by: Fupan Li --- src/agent/Cargo.toml | 2 +- src/agent/rustjail/Cargo.toml | 2 +- src/libs/protocols/Cargo.toml | 2 +- src/libs/protocols/build.rs | 1 + src/runtime-rs/crates/agent/Cargo.toml | 2 +- src/runtime-rs/crates/runtimes/common/Cargo.toml | 2 +- src/runtime-rs/crates/runtimes/virt_container/Cargo.toml | 2 +- src/runtime-rs/crates/shim/Cargo.toml | 2 +- src/tools/agent-ctl/Cargo.toml | 2 +- src/tools/trace-forwarder/Cargo.toml | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 041fcdb5d95e..77a5e4189155 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -10,7 +10,7 @@ rustjail = { path = "rustjail" } protocols = { path = "../libs/protocols", features = ["async"] } lazy_static = "1.3.0" ttrpc = { version = "0.6.0", features = ["async"], default-features = false } -protobuf = "2.23.0" +protobuf = "2.27.0" libc = "0.2.58" nix = "0.23.0" capctl = "0.2.0" diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index 628b0275bdbe..80ed60db2120 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -16,7 +16,7 @@ scopeguard = "1.0.0" capctl = "0.2.0" lazy_static = "1.3.0" libc = "0.2.58" -protobuf = "2.23.0" +protobuf = "2.27.0" slog = "2.5.2" slog-scope = "4.1.2" scan_fmt = "0.2.6" diff --git a/src/libs/protocols/Cargo.toml b/src/libs/protocols/Cargo.toml index cf2559b2e0e4..6853e9c25935 100644 --- a/src/libs/protocols/Cargo.toml +++ b/src/libs/protocols/Cargo.toml @@ -12,7 +12,7 @@ async = ["ttrpc/async", "async-trait"] [dependencies] ttrpc = { version = "0.6.0" } async-trait = { version = "0.1.42", optional = true } -protobuf = { version = "=2.14.0", features = ["with-serde"] } +protobuf = { version = "2.27.0", features = ["with-serde"] } serde = { version = "1.0.130", features = ["derive"], optional = true } serde_json = { version = "1.0.68", optional = true } oci = { path = "../oci" } diff --git a/src/libs/protocols/build.rs b/src/libs/protocols/build.rs index 8a2725ae0527..ebb6ef126954 100644 --- a/src/libs/protocols/build.rs +++ b/src/libs/protocols/build.rs @@ -149,6 +149,7 @@ fn real_main() -> Result<(), std::io::Error> { "protos/google/protobuf/empty.proto", "protos/oci.proto", "protos/types.proto", + "protos/csi.proto", ], false, )?; diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml index 7fde8bf85ce1..f9350bd0659e 100644 --- a/src/runtime-rs/crates/agent/Cargo.toml +++ b/src/runtime-rs/crates/agent/Cargo.toml @@ -11,7 +11,7 @@ futures = "0.1.27" anyhow = "1.0.26" async-trait = "0.1.48" log = "0.4.14" -protobuf = "2.23.0" +protobuf = "2.27.0" serde = { version = "^1.0", features = ["derive"] } serde_json = ">=1.0.9" slog = "2.5.2" diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index bfa08452b7cf..af2ea082b12c 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -12,7 +12,7 @@ async-trait = "0.1.48" containerd-shim-protos = { version = "0.2.0", features = ["async"]} lazy_static = "1.4.0" nix = "0.23.1" -protobuf = "2.23.0" +protobuf = "2.27.0" serde_json = "1.0.39" slog = "2.5.2" slog-scope = "4.4.0" diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml index e34e2fd5b720..8ce387f2afa1 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml @@ -13,7 +13,7 @@ futures = "0.3.19" lazy_static = "1.4.0" libc = ">=0.2.39" nix = "0.16.0" -protobuf = "2.23.0" +protobuf = "2.27.0" serde = { version = "1.0.100", features = ["derive"] } serde_derive = "1.0.27" serde_json = "1.0.39" diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml index 67fac11c13fa..c1a1d1d79d0f 100644 --- a/src/runtime-rs/crates/shim/Cargo.toml +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -20,7 +20,7 @@ go-flag = "0.1.0" libc = "0.2.108" log = "0.4.14" nix = "0.23.1" -protobuf = "2.23.0" +protobuf = "2.27.0" sha2 = "=0.9.3" slog = {version = "2.7.0", features = ["std", "release_max_level_trace", "max_level_trace"]} slog-async = "2.7.0" diff --git a/src/tools/agent-ctl/Cargo.toml b/src/tools/agent-ctl/Cargo.toml index 4d5845f1c31a..3b5f9d073fd6 100644 --- a/src/tools/agent-ctl/Cargo.toml +++ b/src/tools/agent-ctl/Cargo.toml @@ -26,7 +26,7 @@ logging = { path = "../../libs/logging" } slog = "2.7.0" slog-scope = "4.4.0" rand = "0.8.4" -protobuf = "2.14.0" +protobuf = "2.27.0" nix = "0.23.0" libc = "0.2.112" diff --git a/src/tools/trace-forwarder/Cargo.toml b/src/tools/trace-forwarder/Cargo.toml index 8a520a26a0b1..2579ae5b6c04 100644 --- a/src/tools/trace-forwarder/Cargo.toml +++ b/src/tools/trace-forwarder/Cargo.toml @@ -22,7 +22,7 @@ serde_json = "1.0.44" anyhow = "1.0.31" opentelemetry = { version = "0.14.0", features=["serialize"] } opentelemetry-jaeger = "0.13.0" -protobuf = "=2.14.0" +protobuf = "2.27.0" tracing-opentelemetry = "0.16.0" tracing = "0.1.29" tracing-subscriber = "0.3.3" From 9cb15ab4c5c05e1c73c819eb931f599a675cb47b Mon Sep 17 00:00:00 2001 From: Fupan Li Date: Sat, 11 Jun 2022 11:30:51 +0800 Subject: [PATCH 0052/1953] agent: add the FSGroup support Signed-off-by: Fupan Li --- src/runtime-rs/crates/agent/src/kata/trans.rs | 32 +++++++++++++++---- src/runtime-rs/crates/agent/src/lib.rs | 2 +- src/runtime-rs/crates/agent/src/types.rs | 19 +++++++++++ .../src/share_fs/share_virtio_fs_inline.rs | 1 + .../crates/resource/src/volume/shm_volume.rs | 1 + 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 033f0bd9796b..e7fdaa9448f8 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -16,13 +16,14 @@ use crate::{ ARPNeighbor, ARPNeighbors, AddArpNeighborRequest, AgentDetails, BlkioStats, BlkioStatsEntry, CgroupStats, CheckRequest, CloseStdinRequest, ContainerID, CopyFileRequest, CpuStats, CpuUsage, CreateContainerRequest, CreateSandboxRequest, Device, - Empty, ExecProcessRequest, GuestDetailsResponse, HealthCheckResponse, HugetlbStats, - IPAddress, IPFamily, Interface, Interfaces, KernelModule, MemHotplugByProbeRequest, - MemoryData, MemoryStats, NetworkStats, OnlineCPUMemRequest, PidsStats, ReadStreamRequest, - ReadStreamResponse, RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, - SetGuestDateTimeRequest, SignalProcessRequest, StatsContainerResponse, Storage, StringUser, - ThrottlingData, TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, - UpdateRoutesRequest, VersionCheckResponse, WaitProcessRequest, WriteStreamRequest, + Empty, ExecProcessRequest, FSGroup, FSGroupChangePolicy, GuestDetailsResponse, + HealthCheckResponse, HugetlbStats, IPAddress, IPFamily, Interface, Interfaces, + KernelModule, MemHotplugByProbeRequest, MemoryData, MemoryStats, NetworkStats, + OnlineCPUMemRequest, PidsStats, ReadStreamRequest, ReadStreamResponse, + RemoveContainerRequest, ReseedRandomDevRequest, Route, Routes, SetGuestDateTimeRequest, + SignalProcessRequest, StatsContainerResponse, Storage, StringUser, ThrottlingData, + TtyWinResizeRequest, UpdateContainerRequest, UpdateInterfaceRequest, UpdateRoutesRequest, + VersionCheckResponse, WaitProcessRequest, WriteStreamRequest, }, OomEventResponse, WaitProcessResponse, WriteStreamResponse, }; @@ -72,6 +73,22 @@ impl From for Empty { } } +impl From for agent::FSGroup { + fn from(from: FSGroup) -> Self { + let policy = match from.group_change_policy { + FSGroupChangePolicy::Always => types::FSGroupChangePolicy::Always, + FSGroupChangePolicy::OnRootMismatch => types::FSGroupChangePolicy::OnRootMismatch, + }; + + Self { + group_id: from.group_id, + group_change_policy: policy, + unknown_fields: Default::default(), + cached_size: Default::default(), + } + } +} + impl From for agent::StringUser { fn from(from: StringUser) -> Self { Self { @@ -105,6 +122,7 @@ impl From for agent::Storage { driver_options: from_vec(from.driver_options), source: from.source, fstype: from.fs_type, + fs_group: from_option(from.fs_group), options: from_vec(from.options), mount_point: from.mount_point, unknown_fields: Default::default(), diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index 9c72a76847e4..1e28cc2b8f0d 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -12,7 +12,7 @@ logging::logger_with_subsystem!(sl, "agent"); pub mod kata; mod log_forwarder; mod sock; -mod types; +pub mod types; pub use types::{ ARPNeighbor, ARPNeighbors, AddArpNeighborRequest, BlkioStatsEntry, CheckRequest, CloseStdinRequest, ContainerID, ContainerProcessID, CopyFileRequest, CreateContainerRequest, diff --git a/src/runtime-rs/crates/agent/src/types.rs b/src/runtime-rs/crates/agent/src/types.rs index caf507c9ae90..49319c2f07ea 100644 --- a/src/runtime-rs/crates/agent/src/types.rs +++ b/src/runtime-rs/crates/agent/src/types.rs @@ -15,6 +15,24 @@ impl Empty { } } +impl Default for FSGroupChangePolicy { + fn default() -> Self { + FSGroupChangePolicy::Always + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FSGroupChangePolicy { + Always = 0, + OnRootMismatch = 1, +} + +#[derive(Debug, PartialEq, Clone, Default)] +pub struct FSGroup { + pub group_id: u32, + pub group_change_policy: FSGroupChangePolicy, +} + #[derive(PartialEq, Clone, Default)] pub struct StringUser { pub uid: String, @@ -37,6 +55,7 @@ pub struct Storage { pub driver_options: Vec, pub source: String, pub fs_type: String, + pub fs_group: Option, pub options: Vec, pub mount_point: String, } diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs index 146efc609400..e903694ead6d 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs @@ -78,6 +78,7 @@ impl ShareFs for ShareVirtioFsInline { driver_options: Vec::new(), source: String::from(MOUNT_GUEST_TAG), fs_type: String::from(FS_TYPE_VIRTIO_FS), + fs_group: None, options: shared_options, mount_point: String::from(KATA_GUEST_SHARE_DIR), }; diff --git a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs index e26fe19046bd..c1c9df993fba 100644 --- a/src/runtime-rs/crates/resource/src/volume/shm_volume.rs +++ b/src/runtime-rs/crates/resource/src/volume/shm_volume.rs @@ -45,6 +45,7 @@ impl ShmVolume { driver_options: Vec::new(), source: String::from("shm"), fs_type: String::from("tmpfs"), + fs_group: None, options, mount_point: mount_path.to_string(), }; From 8835db6b0f8990398cc877ca43ea81a2c1c09078 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Fri, 29 Apr 2022 15:00:19 +0800 Subject: [PATCH 0053/1953] dragonball: initial commit The dragonball crate initial commit that includes dragonball README and basic code structure. Fixes: #4257 Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/.gitignore | 3 +++ src/dragonball/Cargo.toml | 15 +++++++++++++++ src/dragonball/LICENSE | 1 + src/dragonball/README.md | 28 ++++++++++++++++++++++++++++ src/dragonball/src/lib.rs | 5 +++++ 5 files changed, 52 insertions(+) create mode 100644 src/dragonball/.gitignore create mode 100644 src/dragonball/Cargo.toml create mode 120000 src/dragonball/LICENSE create mode 100644 src/dragonball/README.md create mode 100644 src/dragonball/src/lib.rs diff --git a/src/dragonball/.gitignore b/src/dragonball/.gitignore new file mode 100644 index 000000000000..64f40ab2960c --- /dev/null +++ b/src/dragonball/.gitignore @@ -0,0 +1,3 @@ +target +Cargo.lock +.idea diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml new file mode 100644 index 000000000000..2ffc85334b76 --- /dev/null +++ b/src/dragonball/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "dragonball" +version = "0.1.0" +authors = ["The Kata Containers community "] +description = "A secure sandbox for Kata Containers" +keywords = ["kata-containers", "sandbox", "vmm", "dragonball"] +homepage = "https://katacontainers.io/" +repository = "https://github.com/kata-containers/kata-containers.git" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +arc-swap = "1.5.0" +dbs-device = "0.1.0" +thiserror = "1" diff --git a/src/dragonball/LICENSE b/src/dragonball/LICENSE new file mode 120000 index 000000000000..30cff7403da0 --- /dev/null +++ b/src/dragonball/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/src/dragonball/README.md b/src/dragonball/README.md new file mode 100644 index 000000000000..dcc066424fcd --- /dev/null +++ b/src/dragonball/README.md @@ -0,0 +1,28 @@ +# Dragonball + +## Introduction +Dragonball Sandbox is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual Machine(KVM), +which is optimized for container workloads with: +- container image management and acceleration service +- flexible and high-performance virtual device drivers +- low cpu & memory overhead +- industry-leading startup time +- high concurrent startup throughput + +Dragonball Sandbox aims to provide a turnkey solution for the Kata Containers community. It is integrated into Kata 3.0 +runtime as a built-in VMM and gives users an out-of-the-box Kata Containers experience without complex environment setup +and configuration process. + +## Getting Start +[TODO] + +## Supported Architectures +- x86-64 +- AArch64 + +## Supported Kernel +[TODO] + +## License + +Dragonball is licensed under [Apache License](http://www.apache.org/licenses/LICENSE-2.0), Version 2.0. \ No newline at end of file diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs new file mode 100644 index 000000000000..df47da6aa8af --- /dev/null +++ b/src/dragonball/src/lib.rs @@ -0,0 +1,5 @@ +// Copyright (C) 2018-2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Dragonball is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual +//! Machine(KVM) which is optimized for container workloads. From aff6040555739c45fa80c68ccfb52ccde59f3dc0 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Fri, 29 Apr 2022 17:45:20 +0800 Subject: [PATCH 0054/1953] dragonball: add resource manager support. Resource manager manages all resources of a virtual machine instance. Fixes: #4257 Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 2 + src/dragonball/src/lib.rs | 3 + src/dragonball/src/resource_manager.rs | 785 +++++++++++++++++++++++++ 3 files changed, 790 insertions(+) create mode 100644 src/dragonball/src/resource_manager.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 2ffc85334b76..f88f5310dd07 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -11,5 +11,7 @@ edition = "2018" [dependencies] arc-swap = "1.5.0" +dbs-allocator = "0.1.0" +dbs-boot = "0.2.0" dbs-device = "0.1.0" thiserror = "1" diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index df47da6aa8af..f25b8c13d6b1 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -3,3 +3,6 @@ //! Dragonball is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual //! Machine(KVM) which is optimized for container workloads. + +/// Resource manager for virtual machines. +pub mod resource_manager; diff --git a/src/dragonball/src/resource_manager.rs b/src/dragonball/src/resource_manager.rs new file mode 100644 index 000000000000..2cb32c054612 --- /dev/null +++ b/src/dragonball/src/resource_manager.rs @@ -0,0 +1,785 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Mutex; + +use dbs_allocator::{Constraint, IntervalTree, Range}; +use dbs_boot::layout::{ + GUEST_MEM_END, GUEST_MEM_START, GUEST_PHYS_END, IRQ_BASE as LEGACY_IRQ_BASE, + IRQ_MAX as LEGACY_IRQ_MAX, MMIO_LOW_END, MMIO_LOW_START, +}; +use dbs_device::resources::{DeviceResources, MsiIrqType, Resource, ResourceConstraint}; + +// We reserve the LEGACY_IRQ_BASE(5) for shared IRQ. +const SHARED_IRQ: u32 = LEGACY_IRQ_BASE; +// Since ioapic2 have 24 pins for legacy devices, so irq number 0-23 are used. We will set MSI_IRQ_BASE at 24. +#[cfg(target_arch = "x86_64")] +const MSI_IRQ_BASE: u32 = 24; +#[cfg(target_arch = "aarch64")] +/// We define MSI_IRQ_BASE as LEGACY_IRQ_MAX for aarch64 in order not to conflict with legacy irq numbers. +const MSI_IRQ_BASE: u32 = LEGACY_IRQ_MAX + 1; + +// kvm max irq is defined in arch/x86/include/asm/kvm_host.h +const MSI_IRQ_MAX: u32 = 1023; +// x86's kvm user mem slots is defined in arch/x86/include/asm/kvm_host.h +#[cfg(target_arch = "x86_64")] +const KVM_USER_MEM_SLOTS: u32 = 509; +// aarch64's kvm user mem slots is defined in arch/arm64/include/asm/kvm_host.h +#[cfg(target_arch = "aarch64")] +const KVM_USER_MEM_SLOTS: u32 = 512; +const PIO_MIN: u16 = 0x0; +const PIO_MAX: u16 = 0xFFFF; +// Reserve the 64MB MMIO address range just below 4G, x86 systems have special +// devices, such as LAPIC, IOAPIC, HPET etc, in this range. And we don't explicitly +// allocate MMIO address for those devices. +const MMIO_SPACE_RESERVED: u64 = 0x400_0000; + +/// Errors associated with resource management operations +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum ResourceError { + /// Unknown/unsupported resource type. + #[error("unsupported resource type")] + UnknownResourceType, + + /// Invalid resource range. + #[error("invalid resource range for resource type : {0}")] + InvalidResourceRange(String), + + /// No resource available. + #[error("no resource available")] + NoAvailResource, +} + +#[derive(Default)] +struct ResourceManagerBuilder { + // IntervalTree for allocating legacy irq number. + legacy_irq_pool: IntervalTree<()>, + // IntervalTree for allocating message signal interrupt (MSI) irq number. + msi_irq_pool: IntervalTree<()>, + // IntervalTree for allocating port-mapped io (PIO) address. + pio_pool: IntervalTree<()>, + // IntervalTree for allocating memory-mapped io (MMIO) address. + mmio_pool: IntervalTree<()>, + // IntervalTree for allocating guest memory. + mem_pool: IntervalTree<()>, + // IntervalTree for allocating kvm memory slot. + kvm_mem_slot_pool: IntervalTree<()>, +} + +impl ResourceManagerBuilder { + /// init legacy_irq_pool with arch specific constants. + fn init_legacy_irq_pool(mut self) -> Self { + // The LEGACY_IRQ_BASE irq is reserved for shared IRQ and won't be allocated / reallocated, + // so we don't insert it into the legacy_irq interval tree. + self.legacy_irq_pool + .insert(Range::new(LEGACY_IRQ_BASE + 1, LEGACY_IRQ_MAX), None); + self + } + + /// init msi_irq_pool with arch specific constants. + fn init_msi_irq_pool(mut self) -> Self { + self.msi_irq_pool + .insert(Range::new(MSI_IRQ_BASE, MSI_IRQ_MAX), None); + self + } + + /// init pio_pool with arch specific constants. + fn init_pio_pool(mut self) -> Self { + self.pio_pool.insert(Range::new(PIO_MIN, PIO_MAX), None); + self + } + + /// Create mmio_pool with arch specific constants. + /// allow(clippy) is because `GUEST_MEM_START > MMIO_LOW_END`, we may modify GUEST_MEM_START or + /// MMIO_LOW_END in the future. + #[allow(clippy::absurd_extreme_comparisons)] + fn init_mmio_pool_helper(mmio: &mut IntervalTree<()>) { + mmio.insert(Range::new(MMIO_LOW_START, MMIO_LOW_END), None); + if !(*GUEST_MEM_END < MMIO_LOW_START + || GUEST_MEM_START > MMIO_LOW_END + || MMIO_LOW_START == MMIO_LOW_END) + { + #[cfg(target_arch = "x86_64")] + { + let constraint = Constraint::new(MMIO_SPACE_RESERVED) + .min(MMIO_LOW_END - MMIO_SPACE_RESERVED) + .max(0xffff_ffffu64); + let key = mmio.allocate(&constraint); + if let Some(k) = key.as_ref() { + mmio.update(k, ()); + } else { + panic!("failed to reserve MMIO address range for x86 system devices"); + } + } + } + + if *GUEST_MEM_END < *GUEST_PHYS_END { + mmio.insert(Range::new(*GUEST_MEM_END + 1, *GUEST_PHYS_END), None); + } + } + + /// init mmio_pool with helper function + fn init_mmio_pool(mut self) -> Self { + Self::init_mmio_pool_helper(&mut self.mmio_pool); + self + } + + /// Create mem_pool with arch specific constants. + /// deny(clippy) is because `GUEST_MEM_START > MMIO_LOW_END`, we may modify GUEST_MEM_START or + /// MMIO_LOW_END in the future. + #[allow(clippy::absurd_extreme_comparisons)] + pub(crate) fn init_mem_pool_helper(mem: &mut IntervalTree<()>) { + if *GUEST_MEM_END < MMIO_LOW_START + || GUEST_MEM_START > MMIO_LOW_END + || MMIO_LOW_START == MMIO_LOW_END + { + mem.insert(Range::new(GUEST_MEM_START, *GUEST_MEM_END), None); + } else { + if MMIO_LOW_START > GUEST_MEM_START { + mem.insert(Range::new(GUEST_MEM_START, MMIO_LOW_START - 1), None); + } + if MMIO_LOW_END < *GUEST_MEM_END { + mem.insert(Range::new(MMIO_LOW_END + 1, *GUEST_MEM_END), None); + } + } + } + + /// init mem_pool with helper function + fn init_mem_pool(mut self) -> Self { + Self::init_mem_pool_helper(&mut self.mem_pool); + self + } + + /// init kvm_mem_slot_pool with arch specific constants. + fn init_kvm_mem_slot_pool(mut self, max_kvm_mem_slot: Option) -> Self { + let max_slots = max_kvm_mem_slot.unwrap_or(KVM_USER_MEM_SLOTS as usize); + self.kvm_mem_slot_pool + .insert(Range::new(0, max_slots as u64), None); + self + } + + fn build(self) -> ResourceManager { + ResourceManager { + legacy_irq_pool: Mutex::new(self.legacy_irq_pool), + msi_irq_pool: Mutex::new(self.msi_irq_pool), + pio_pool: Mutex::new(self.pio_pool), + mmio_pool: Mutex::new(self.mmio_pool), + mem_pool: Mutex::new(self.mem_pool), + kvm_mem_slot_pool: Mutex::new(self.kvm_mem_slot_pool), + } + } +} + +/// Resource manager manages all resources for a virtual machine instance. +pub struct ResourceManager { + legacy_irq_pool: Mutex>, + msi_irq_pool: Mutex>, + pio_pool: Mutex>, + mmio_pool: Mutex>, + mem_pool: Mutex>, + kvm_mem_slot_pool: Mutex>, +} + +impl Default for ResourceManager { + fn default() -> Self { + ResourceManagerBuilder::default().build() + } +} + +impl ResourceManager { + /// Create a resource manager instance. + pub fn new(max_kvm_mem_slot: Option) -> Self { + let res_manager_builder = ResourceManagerBuilder::default(); + res_manager_builder + .init_legacy_irq_pool() + .init_msi_irq_pool() + .init_pio_pool() + .init_mmio_pool() + .init_mem_pool() + .init_kvm_mem_slot_pool(max_kvm_mem_slot) + .build() + } + + /// Init mem_pool with arch specific constants. + pub fn init_mem_pool(&self) { + let mut mem = self.mem_pool.lock().unwrap(); + ResourceManagerBuilder::init_mem_pool_helper(&mut mem); + } + + /// Check if mem_pool is empty. + pub fn is_mem_pool_empty(&self) -> bool { + self.mem_pool.lock().unwrap().is_empty() + } + + /// Allocate one legacy irq number. + /// + /// Allocate the specified irq number if `fixed` contains an irq number. + pub fn allocate_legacy_irq(&self, shared: bool, fixed: Option) -> Option { + // if shared_irq is used, just return the shared irq num. + if shared { + return Some(SHARED_IRQ); + } + + let mut constraint = Constraint::new(1u32); + if let Some(v) = fixed { + if v == SHARED_IRQ { + return None; + } + + constraint.min = v as u64; + constraint.max = v as u64; + } + // Safe to unwrap() because we don't expect poisoned lock here. + let mut legacy_irq_pool = self.legacy_irq_pool.lock().unwrap(); + let key = legacy_irq_pool.allocate(&constraint); + if let Some(k) = key.as_ref() { + legacy_irq_pool.update(k, ()); + } + key.map(|v| v.min as u32) + } + + /// Free a legacy irq number. + /// + /// Panic if the irq number is invalid. + pub fn free_legacy_irq(&self, irq: u32) -> Result<(), ResourceError> { + // if the irq number is shared_irq, we don't need to do anything. + if irq == SHARED_IRQ { + return Ok(()); + } + + if !(LEGACY_IRQ_BASE..=LEGACY_IRQ_MAX).contains(&irq) { + return Err(ResourceError::InvalidResourceRange( + "Legacy IRQ".to_string(), + )); + } + let key = Range::new(irq, irq); + // Safe to unwrap() because we don't expect poisoned lock here. + self.legacy_irq_pool.lock().unwrap().free(&key); + Ok(()) + } + + /// Allocate a group of MSI irq numbers. + /// + /// The allocated MSI irq numbers may or may not be naturally aligned. + pub fn allocate_msi_irq(&self, count: u32) -> Option { + let constraint = Constraint::new(count); + // Safe to unwrap() because we don't expect poisoned lock here. + let mut msi_irq_pool = self.msi_irq_pool.lock().unwrap(); + let key = msi_irq_pool.allocate(&constraint); + if let Some(k) = key.as_ref() { + msi_irq_pool.update(k, ()); + } + key.map(|v| v.min as u32) + } + + /// Allocate a group of MSI irq numbers, naturally aligned to `count`. + /// + /// This may be used to support PCI MSI, which requires the allocated irq number is naturally + /// aligned. + pub fn allocate_msi_irq_aligned(&self, count: u32) -> Option { + let constraint = Constraint::new(count).align(count); + // Safe to unwrap() because we don't expect poisoned lock here. + let mut msi_irq_pool = self.msi_irq_pool.lock().unwrap(); + let key = msi_irq_pool.allocate(&constraint); + if let Some(k) = key.as_ref() { + msi_irq_pool.update(k, ()); + } + key.map(|v| v.min as u32) + } + + /// Free a group of MSI irq numbers. + /// + /// Panic if `irq` or `count` is invalid. + pub fn free_msi_irq(&self, irq: u32, count: u32) -> Result<(), ResourceError> { + if irq < MSI_IRQ_BASE + || count == 0 + || irq.checked_add(count).is_none() + || irq + count - 1 > MSI_IRQ_MAX + { + return Err(ResourceError::InvalidResourceRange("MSI IRQ".to_string())); + } + let key = Range::new(irq, irq + count - 1); + // Safe to unwrap() because we don't expect poisoned lock here. + self.msi_irq_pool.lock().unwrap().free(&key); + Ok(()) + } + + /// Allocate a group of PIO address and returns the allocated PIO base address. + pub fn allocate_pio_address_simple(&self, size: u16) -> Option { + let constraint = Constraint::new(size); + self.allocate_pio_address(&constraint) + } + + /// Allocate a group of PIO address and returns the allocated PIO base address. + pub fn allocate_pio_address(&self, constraint: &Constraint) -> Option { + // Safe to unwrap() because we don't expect poisoned lock here. + let mut pio_pool = self.pio_pool.lock().unwrap(); + let key = pio_pool.allocate(constraint); + if let Some(k) = key.as_ref() { + pio_pool.update(k, ()); + } + key.map(|v| v.min as u16) + } + + /// Free PIO address range `[base, base + size - 1]`. + /// + /// Panic if `base` or `size` is invalid. + pub fn free_pio_address(&self, base: u16, size: u16) -> Result<(), ResourceError> { + if base.checked_add(size).is_none() { + return Err(ResourceError::InvalidResourceRange( + "PIO Address".to_string(), + )); + } + let key = Range::new(base, base + size - 1); + // Safe to unwrap() because we don't expect poisoned lock here. + self.pio_pool.lock().unwrap().free(&key); + Ok(()) + } + + /// Allocate a MMIO address range alinged to `align` and returns the allocated base address. + pub fn allocate_mmio_address_aligned(&self, size: u64, align: u64) -> Option { + let constraint = Constraint::new(size).align(align); + self.allocate_mmio_address(&constraint) + } + + /// Allocate a MMIO address range and returns the allocated base address. + pub fn allocate_mmio_address(&self, constraint: &Constraint) -> Option { + // Safe to unwrap() because we don't expect poisoned lock here. + let mut mmio_pool = self.mmio_pool.lock().unwrap(); + let key = mmio_pool.allocate(constraint); + key.map(|v| v.min) + } + + /// Free MMIO address range `[base, base + size - 1]` + pub fn free_mmio_address(&self, base: u64, size: u64) -> Result<(), ResourceError> { + if base.checked_add(size).is_none() { + return Err(ResourceError::InvalidResourceRange( + "MMIO Address".to_string(), + )); + } + let key = Range::new(base, base + size - 1); + // Safe to unwrap() because we don't expect poisoned lock here. + self.mmio_pool.lock().unwrap().free(&key); + Ok(()) + } + + /// Allocate guest memory address range and returns the allocated base memory address. + pub fn allocate_mem_address(&self, constraint: &Constraint) -> Option { + // Safe to unwrap() because we don't expect poisoned lock here. + let mut mem_pool = self.mem_pool.lock().unwrap(); + let key = mem_pool.allocate(constraint); + + key.map(|v| v.min) + } + + /// Free the guest memory address range `[base, base + size - 1]`. + /// + /// Panic if the guest memory address range is invalid. + /// allow(clippy) is because `base < GUEST_MEM_START`, we may modify GUEST_MEM_START in the future. + #[allow(clippy::absurd_extreme_comparisons)] + pub fn free_mem_address(&self, base: u64, size: u64) -> Result<(), ResourceError> { + if base.checked_add(size).is_none() + || base < GUEST_MEM_START + || base + size > *GUEST_MEM_END + { + return Err(ResourceError::InvalidResourceRange( + "MEM Address".to_string(), + )); + } + let key = Range::new(base, base + size - 1); + // Safe to unwrap() because we don't expect poisoned lock here. + self.mem_pool.lock().unwrap().free(&key); + Ok(()) + } + + /// Allocate a kvm memory slot number. + /// + /// Allocate the specified slot if `fixed` contains a slot number. + pub fn allocate_kvm_mem_slot(&self, size: u32, fixed: Option) -> Option { + let mut constraint = Constraint::new(size); + if let Some(v) = fixed { + constraint.min = v as u64; + constraint.max = v as u64; + } + // Safe to unwrap() because we don't expect poisoned lock here. + let mut kvm_mem_slot_pool = self.kvm_mem_slot_pool.lock().unwrap(); + let key = kvm_mem_slot_pool.allocate(&constraint); + if let Some(k) = key.as_ref() { + kvm_mem_slot_pool.update(k, ()); + } + key.map(|v| v.min as u32) + } + + /// Free a kvm memory slot number. + pub fn free_kvm_mem_slot(&self, slot: u32) -> Result<(), ResourceError> { + let key = Range::new(slot, slot); + // Safe to unwrap() because we don't expect poisoned lock here. + self.kvm_mem_slot_pool.lock().unwrap().free(&key); + Ok(()) + } + + /// Allocate requested resources for a device. + pub fn allocate_device_resources( + &self, + requests: &[ResourceConstraint], + shared_irq: bool, + ) -> std::result::Result { + let mut resources = DeviceResources::new(); + for resource in requests.iter() { + let res = match resource { + ResourceConstraint::PioAddress { range, align, size } => { + let mut constraint = Constraint::new(*size).align(*align); + if let Some(r) = range { + constraint.min = r.0 as u64; + constraint.max = r.1 as u64; + } + match self.allocate_pio_address(&constraint) { + Some(base) => Resource::PioAddressRange { + base: base as u16, + size: *size, + }, + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + } + } + ResourceConstraint::MmioAddress { range, align, size } => { + let mut constraint = Constraint::new(*size).align(*align); + if let Some(r) = range { + constraint.min = r.0; + constraint.max = r.1; + } + match self.allocate_mmio_address(&constraint) { + Some(base) => Resource::MmioAddressRange { base, size: *size }, + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + } + } + ResourceConstraint::MemAddress { range, align, size } => { + let mut constraint = Constraint::new(*size).align(*align); + if let Some(r) = range { + constraint.min = r.0; + constraint.max = r.1; + } + match self.allocate_mem_address(&constraint) { + Some(base) => Resource::MemAddressRange { base, size: *size }, + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + } + } + ResourceConstraint::LegacyIrq { irq } => { + match self.allocate_legacy_irq(shared_irq, *irq) { + Some(v) => Resource::LegacyIrq(v), + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + } + } + ResourceConstraint::PciMsiIrq { size } => { + match self.allocate_msi_irq_aligned(*size) { + Some(base) => Resource::MsiIrq { + ty: MsiIrqType::PciMsi, + base, + size: *size, + }, + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + } + } + ResourceConstraint::PciMsixIrq { size } => match self.allocate_msi_irq(*size) { + Some(base) => Resource::MsiIrq { + ty: MsiIrqType::PciMsix, + base, + size: *size, + }, + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + }, + ResourceConstraint::GenericIrq { size } => match self.allocate_msi_irq(*size) { + Some(base) => Resource::MsiIrq { + ty: MsiIrqType::GenericMsi, + base, + size: *size, + }, + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + }, + ResourceConstraint::KvmMemSlot { slot, size } => { + match self.allocate_kvm_mem_slot(*size, *slot) { + Some(v) => Resource::KvmMemSlot(v), + None => { + if let Err(e) = self.free_device_resources(&resources) { + return Err(e); + } else { + return Err(ResourceError::NoAvailResource); + } + } + } + } + }; + resources.append(res); + } + + Ok(resources) + } + + /// Free resources allocated for a device. + pub fn free_device_resources(&self, resources: &DeviceResources) -> Result<(), ResourceError> { + for res in resources.iter() { + let result = match res { + Resource::PioAddressRange { base, size } => self.free_pio_address(*base, *size), + Resource::MmioAddressRange { base, size } => self.free_mmio_address(*base, *size), + Resource::MemAddressRange { base, size } => self.free_mem_address(*base, *size), + Resource::LegacyIrq(base) => self.free_legacy_irq(*base), + Resource::MsiIrq { ty: _, base, size } => self.free_msi_irq(*base, *size), + Resource::KvmMemSlot(slot) => self.free_kvm_mem_slot(*slot), + Resource::MacAddresss(_) => Ok(()), + }; + if result.is_err() { + return result; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_allocate_legacy_irq() { + let mgr = ResourceManager::new(None); + + // Allocate/free shared IRQs multiple times. + assert_eq!(mgr.allocate_legacy_irq(true, None).unwrap(), SHARED_IRQ); + assert_eq!(mgr.allocate_legacy_irq(true, None).unwrap(), SHARED_IRQ); + mgr.free_legacy_irq(SHARED_IRQ); + mgr.free_legacy_irq(SHARED_IRQ); + mgr.free_legacy_irq(SHARED_IRQ); + + // Allocate specified IRQs. + assert_eq!( + mgr.allocate_legacy_irq(false, Some(LEGACY_IRQ_BASE + 10)) + .unwrap(), + LEGACY_IRQ_BASE + 10 + ); + mgr.free_legacy_irq(LEGACY_IRQ_BASE + 10); + assert_eq!( + mgr.allocate_legacy_irq(false, Some(LEGACY_IRQ_BASE + 10)) + .unwrap(), + LEGACY_IRQ_BASE + 10 + ); + assert!(mgr + .allocate_legacy_irq(false, Some(LEGACY_IRQ_BASE + 10)) + .is_none()); + + assert!(mgr.allocate_legacy_irq(false, None).is_some()); + + assert!(mgr + .allocate_legacy_irq(false, Some(LEGACY_IRQ_BASE - 1)) + .is_none()); + assert!(mgr + .allocate_legacy_irq(false, Some(LEGACY_IRQ_MAX + 1)) + .is_none()); + assert!(mgr.allocate_legacy_irq(false, Some(SHARED_IRQ)).is_none()); + } + + #[test] + fn test_invalid_free_legacy_irq() { + let mgr = ResourceManager::new(None); + assert_eq!( + mgr.free_legacy_irq(LEGACY_IRQ_MAX + 1), + Err(ResourceError::InvalidResourceRange( + "Legacy IRQ".to_string(), + )) + ); + } + + #[test] + fn test_allocate_msi_irq() { + let mgr = ResourceManager::new(None); + + let msi = mgr.allocate_msi_irq(3).unwrap(); + mgr.free_msi_irq(msi, 3); + let msi = mgr.allocate_msi_irq(3).unwrap(); + mgr.free_msi_irq(msi, 3); + + let irq = mgr.allocate_msi_irq_aligned(8).unwrap(); + assert_eq!(irq & 0x7, 0); + mgr.free_msi_irq(msi, 8); + let irq = mgr.allocate_msi_irq_aligned(8).unwrap(); + assert_eq!(irq & 0x7, 0); + + let irq = mgr.allocate_msi_irq_aligned(512).unwrap(); + assert_eq!(irq, 512); + mgr.free_msi_irq(irq, 512); + let irq = mgr.allocate_msi_irq_aligned(512).unwrap(); + assert_eq!(irq, 512); + + assert!(mgr.allocate_msi_irq(4099).is_none()); + } + + #[test] + fn test_invalid_free_msi_irq() { + let mgr = ResourceManager::new(None); + assert_eq!( + mgr.free_msi_irq(MSI_IRQ_MAX, 3), + Err(ResourceError::InvalidResourceRange("MSI IRQ".to_string())) + ); + } + + #[test] + fn test_allocate_pio_addr() { + let mgr = ResourceManager::new(None); + assert!(mgr.allocate_pio_address_simple(10).is_some()); + let mut requests = vec![ + ResourceConstraint::PioAddress { + range: None, + align: 0x1000, + size: 0x2000, + }, + ResourceConstraint::PioAddress { + range: Some((0x8000, 0x9000)), + align: 0x1000, + size: 0x1000, + }, + ResourceConstraint::PioAddress { + range: Some((0x9000, 0xa000)), + align: 0x1000, + size: 0x1000, + }, + ResourceConstraint::PioAddress { + range: Some((0xb000, 0xc000)), + align: 0x1000, + size: 0x1000, + }, + ]; + let resources = mgr.allocate_device_resources(&requests, false).unwrap(); + mgr.free_device_resources(&resources); + let resources = mgr.allocate_device_resources(&requests, false).unwrap(); + mgr.free_device_resources(&resources); + requests.push(ResourceConstraint::PioAddress { + range: Some((0xc000, 0xc000)), + align: 0x1000, + size: 0x1000, + }); + assert!(mgr.allocate_device_resources(&requests, false).is_err()); + let resources = mgr + .allocate_device_resources(&requests[0..requests.len() - 1], false) + .unwrap(); + mgr.free_device_resources(&resources); + } + + #[test] + fn test_invalid_free_pio_addr() { + let mgr = ResourceManager::new(None); + assert_eq!( + mgr.free_pio_address(u16::MAX, 3), + Err(ResourceError::InvalidResourceRange( + "PIO Address".to_string(), + )) + ); + } + + #[test] + fn test_allocate_kvm_mem_slot() { + let mgr = ResourceManager::new(None); + assert_eq!(mgr.allocate_kvm_mem_slot(1, None).unwrap(), 0); + assert_eq!(mgr.allocate_kvm_mem_slot(1, Some(200)).unwrap(), 200); + mgr.free_kvm_mem_slot(200); + assert_eq!(mgr.allocate_kvm_mem_slot(1, Some(200)).unwrap(), 200); + assert_eq!( + mgr.allocate_kvm_mem_slot(1, Some(KVM_USER_MEM_SLOTS)) + .unwrap(), + KVM_USER_MEM_SLOTS + ); + assert!(mgr + .allocate_kvm_mem_slot(1, Some(KVM_USER_MEM_SLOTS + 1)) + .is_none()); + } + + #[test] + fn test_allocate_mmio_address() { + let mgr = ResourceManager::new(None); + + #[cfg(target_arch = "x86_64")] + { + // Can't allocate from reserved region + let constraint = Constraint::new(0x100_0000u64) + .min(0x1_0000_0000u64 - 0x200_0000u64) + .max(0xffff_ffffu64); + assert!(mgr.allocate_mmio_address(&constraint).is_none()); + } + let constraint = Constraint::new(0x100_0000u64).min(0x1_0000_0000u64 - 0x200_0000u64); + assert!(mgr.allocate_mmio_address(&constraint).is_some()); + + #[cfg(target_arch = "x86_64")] + { + // Can't allocate from reserved region + let constraint = Constraint::new(0x100_0000u64) + .min(0x1_0000_0000u64 - 0x200_0000u64) + .max(0xffff_ffffu64); + assert!(mgr.allocate_mem_address(&constraint).is_none()); + } + #[cfg(target_arch = "aarch64")] + { + let constraint = Constraint::new(0x200_0000u64) + .min(0x1_0000_0000u64 - 0x200_0000u64) + .max(0xffff_fffeu64); + assert!(mgr.allocate_mem_address(&constraint).is_none()); + } + let constraint = Constraint::new(0x100_0000u64).min(0x1_0000_0000u64 - 0x200_0000u64); + assert!(mgr.allocate_mem_address(&constraint).is_some()); + } + + #[test] + #[should_panic] + fn test_allocate_duplicate_memory() { + let mgr = ResourceManager::new(None); + + let constraint_1 = Constraint::new(0x100_0000u64) + .min(0x1_0000_0000u64) + .max(0x1_0000_0000u64 + 0x100_0000u64); + let constraint_2 = Constraint::new(0x100_0000u64) + .min(0x1_0000_0000u64) + .max(0x1_0000_0000u64 + 0x100_0000u64); + + assert!(mgr.allocate_mem_address(&constraint_1).is_some()); + assert!(mgr.allocate_mem_address(&constraint_2).is_some()); + } +} From 3d38bb3005442712d8afe2abf8092ac128521fb0 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Fri, 29 Apr 2022 19:30:03 +0800 Subject: [PATCH 0055/1953] dragonball: add address space manager. Address space abstraction to manage virtual machine's physical address space. The AddressSpaceMgr Struct to manage address space. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 16 + src/dragonball/src/address_space_manager.rs | 890 ++++++++++++++++++++ src/dragonball/src/lib.rs | 4 + src/dragonball/src/vm/mod.rs | 17 + 4 files changed, 927 insertions(+) create mode 100644 src/dragonball/src/address_space_manager.rs create mode 100644 src/dragonball/src/vm/mod.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index f88f5310dd07..9b5998d91593 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -11,7 +11,23 @@ edition = "2018" [dependencies] arc-swap = "1.5.0" +dbs-address-space = "0.1.0" dbs-allocator = "0.1.0" dbs-boot = "0.2.0" dbs-device = "0.1.0" +kvm-bindings = "0.5.0" +kvm-ioctls = "0.11.0" +libc = "0.2.39" +log = "0.4.14" +nix = "0.23.1" +serde = "1.0.27" +serde_derive = "1.0.27" +serde_json = "1.0.9" +slog = "2.5.2" +slog-scope = "4.4.0" thiserror = "1" +vmm-sys-util = "0.9.0" +vm-memory = { version = "0.7.0", features = ["backend-mmap"] } + +[features] +atomic-guest-memory = [] diff --git a/src/dragonball/src/address_space_manager.rs b/src/dragonball/src/address_space_manager.rs new file mode 100644 index 000000000000..dc7650e97a09 --- /dev/null +++ b/src/dragonball/src/address_space_manager.rs @@ -0,0 +1,890 @@ +// Copyright (C) 2019-2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Address space abstraction to manage virtual machine's physical address space. +//! +//! The AddressSpace abstraction is introduced to manage virtual machine's physical address space. +//! The regions in virtual machine's physical address space may be used to: +//! 1) map guest virtual memory +//! 2) map MMIO ranges for emulated virtual devices, such as virtio-fs DAX window. +//! 3) map MMIO ranges for pass-through devices, such as PCI device BARs. +//! 4) map MMIO ranges for to vCPU, such as local APIC. +//! 5) not used/available +//! +//! A related abstraction, vm_memory::GuestMemory, is used to access guest virtual memory only. +//! In other words, AddressSpace is the resource owner, and GuestMemory is an accessor for guest +//! virtual memory. + +use std::collections::{BTreeMap, HashMap}; +use std::fs::File; +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; + +use dbs_address_space::{ + AddressSpace, AddressSpaceError, AddressSpaceLayout, AddressSpaceRegion, + AddressSpaceRegionType, NumaNode, NumaNodeInfo, MPOL_MF_MOVE, MPOL_PREFERRED, +}; +use dbs_allocator::Constraint; +use kvm_bindings::{kvm_userspace_memory_region, KVM_MEM_LOG_DIRTY_PAGES}; +use kvm_ioctls::VmFd; +use log::{debug, error, info, warn}; +use nix::sys::mman; +use nix::unistd::dup; +#[cfg(feature = "atomic-guest-memory")] +use vm_memory::atomic::GuestMemoryAtomic; +use vm_memory::{ + Address, FileOffset, GuestAddress, GuestAddressSpace, GuestMemoryMmap, GuestMemoryRegion, + GuestRegionMmap, GuestUsize, MemoryRegionAddress, MmapRegion, +}; + +use crate::resource_manager::ResourceManager; +use crate::vm::NumaRegionInfo; + +#[cfg(not(feature = "atomic-guest-memory"))] +/// Concrete GuestAddressSpace type used by the VMM. +pub type GuestAddressSpaceImpl = Arc; + +#[cfg(feature = "atomic-guest-memory")] +/// Concrete GuestAddressSpace type used by the VMM. +pub type GuestAddressSpaceImpl = GuestMemoryAtomic; + +/// Concrete GuestMemory type used by the VMM. +pub type GuestMemoryImpl = as GuestAddressSpace>::M; +/// Concrete GuestRegion type used by the VMM. +pub type GuestRegionImpl = GuestRegionMmap; + +// Maximum number of working threads for memory pre-allocation. +const MAX_PRE_ALLOC_THREAD: u64 = 16; + +// Control the actual number of pre-allocating threads. After several performance tests, we decide to use one thread to do pre-allocating for every 4G memory. +const PRE_ALLOC_GRANULARITY: u64 = 32; + +// We don't have plan to support mainframe computer and only focus on PC servers. +// 64 as max nodes should be enough for now. +const MAX_NODE: u32 = 64; + +// We will split the memory region if it conflicts with the MMIO hole. +// But if the space below the MMIO hole is smaller than the MINIMAL_SPLIT_SPACE, we won't split the memory region in order to enhance performance. +const MINIMAL_SPLIT_SPACE: u64 = 128 << 20; + +/// Errors associated with virtual machine address space management. +#[derive(Debug, thiserror::Error)] +pub enum AddressManagerError { + /// Invalid address space operation. + #[error("invalid address space operation")] + InvalidOperation, + + /// Invalid address range. + #[error("invalid address space region (0x{0:x}, 0x{1:x})")] + InvalidAddressRange(u64, GuestUsize), + + /// No available mem address. + #[error("no available mem address")] + NoAvailableMemAddress, + + /// No available kvm slotse. + #[error("no available kvm slots")] + NoAvailableKvmSlot, + + /// Address manager failed to create memfd to map anonymous memory. + #[error("address manager failed to create memfd to map anonymous memory")] + CreateMemFd(#[source] nix::Error), + + /// Address manager failed to open memory file. + #[error("address manager failed to open memory file")] + OpenFile(#[source] std::io::Error), + + /// Memory file provided is invalid due to empty file path, non-existent file path and other possible mistakes. + #[error("memory file provided to address manager {0} is invalid")] + FileInvalid(String), + + /// Memory file provided is invalid due to empty memory type + #[error("memory type provided to address manager {0} is invalid")] + TypeInvalid(String), + + /// Failed to set size for memory file. + #[error("address manager failed to set size for memory file")] + SetFileSize(#[source] std::io::Error), + + /// Failed to unlink memory file. + #[error("address manager failed to unlink memory file")] + UnlinkFile(#[source] nix::Error), + + /// Failed to duplicate fd of memory file. + #[error("address manager failed to duplicate memory file descriptor")] + DupFd(#[source] nix::Error), + + /// Failure in accessing the memory located at some address. + #[error("address manager failed to access guest memory located at 0x{0:x}")] + AccessGuestMemory(u64, #[source] vm_memory::mmap::Error), + + /// Failed to create GuestMemory + #[error("address manager failed to create guest memory object")] + CreateGuestMemory(#[source] vm_memory::Error), + + /// Failure in initializing guest memory. + #[error("address manager failed to initialize guest memory")] + GuestMemoryNotInitialized, + + /// Failed to mmap() guest memory + #[error("address manager failed to mmap() guest memory into current process")] + MmapGuestMemory(#[source] vm_memory::mmap::MmapRegionError), + + /// Failed to set KVM memory slot. + #[error("address manager failed to configure KVM memory slot")] + KvmSetMemorySlot(#[source] kvm_ioctls::Error), + + /// Failed to set madvise on AddressSpaceRegion + #[error("address manager failed to set madvice() on guest memory region")] + Madvise(#[source] nix::Error), + + /// join threads fail + #[error("address manager failed to join threads")] + JoinFail, + + /// Failed to create Address Space Region + #[error("address manager failed to create Address Space Region {0}")] + CreateAddressSpaceRegion(#[source] AddressSpaceError), +} + +type Result = std::result::Result; + +/// Parameters to configure address space creation operations. +pub struct AddressSpaceMgrBuilder<'a> { + mem_type: &'a str, + mem_file: &'a str, + mem_index: u32, + mem_suffix: bool, + mem_prealloc: bool, + dirty_page_logging: bool, + vmfd: Option>, +} + +impl<'a> AddressSpaceMgrBuilder<'a> { + /// Create a new [`AddressSpaceMgrBuilder`] object. + pub fn new(mem_type: &'a str, mem_file: &'a str) -> Result { + if mem_type.is_empty() { + return Err(AddressManagerError::TypeInvalid(mem_type.to_string())); + } + Ok(AddressSpaceMgrBuilder { + mem_type, + mem_file, + mem_index: 0, + mem_suffix: true, + mem_prealloc: false, + dirty_page_logging: false, + vmfd: None, + }) + } + + /// Enable/disable adding numbered suffix to memory file path. + /// This feature could be useful to generate hugetlbfs files with number suffix. (e.g. shmem0, shmem1) + pub fn toggle_file_suffix(&mut self, enabled: bool) { + self.mem_suffix = enabled; + } + + /// Enable/disable memory pre-allocation. + /// Enable this feature could improve performance stability at the start of workload by avoiding page fault. + /// Disable this feature may influence performance stability but the cpu resource consumption and start-up time will decrease. + pub fn toggle_prealloc(&mut self, prealloc: bool) { + self.mem_prealloc = prealloc; + } + + /// Enable/disable KVM dirty page logging. + pub fn toggle_dirty_page_logging(&mut self, logging: bool) { + self.dirty_page_logging = logging; + } + + /// Set KVM [`VmFd`] handle to configure memory slots. + pub fn set_kvm_vm_fd(&mut self, vmfd: Arc) -> Option> { + let mut existing_vmfd = None; + if self.vmfd.is_some() { + existing_vmfd = self.vmfd.clone(); + } + self.vmfd = Some(vmfd); + existing_vmfd + } + + /// Build a ['AddressSpaceMgr'] using the configured parameters. + pub fn build( + self, + res_mgr: &ResourceManager, + numa_region_infos: &[NumaRegionInfo], + ) -> Result { + let mut mgr = AddressSpaceMgr::default(); + mgr.create_address_space(res_mgr, numa_region_infos, self)?; + Ok(mgr) + } + + fn get_next_mem_file(&mut self) -> String { + if self.mem_suffix { + let path = format!("{}{}", self.mem_file, self.mem_index); + self.mem_index += 1; + path + } else { + self.mem_file.to_string() + } + } +} + +/// Struct to manage virtual machine's physical address space. +pub struct AddressSpaceMgr { + address_space: Option, + vm_as: Option, + base_to_slot: Arc>>, + prealloc_handlers: Vec>, + prealloc_exit: Arc, + numa_nodes: BTreeMap, +} + +impl AddressSpaceMgr { + /// Query address space manager is initialized or not + pub fn is_initialized(&self) -> bool { + self.address_space.is_some() + } + + /// Create the address space for a virtual machine. + /// + /// This method is designed to be called when starting up a virtual machine instead of at + /// runtime, so it's expected the virtual machine will be tore down and no strict error recover. + pub fn create_address_space( + &mut self, + res_mgr: &ResourceManager, + numa_region_infos: &[NumaRegionInfo], + mut param: AddressSpaceMgrBuilder, + ) -> Result<()> { + let mut regions = Vec::new(); + let mut start_addr = dbs_boot::layout::GUEST_MEM_START; + + // Create address space regions. + for info in numa_region_infos.iter() { + info!("numa_region_info {:?}", info); + // convert size_in_mib to bytes + let size = info + .size + .checked_shl(20) + .ok_or_else(|| AddressManagerError::InvalidOperation)?; + + // Guest memory does not intersect with the MMIO hole. + // TODO: make it work for ARM (issue #4307) + if start_addr > dbs_boot::layout::MMIO_LOW_END + || start_addr + size <= dbs_boot::layout::MMIO_LOW_START + { + let region = self.create_region(start_addr, size, info, &mut param)?; + regions.push(region); + start_addr = start_addr + .checked_add(size) + .ok_or_else(|| AddressManagerError::InvalidOperation)?; + } else { + // Add guest memory below the MMIO hole, avoid splitting the memory region + // if the available address region is small than MINIMAL_SPLIT_SPACE MiB. + let mut below_size = dbs_boot::layout::MMIO_LOW_START + .checked_sub(start_addr) + .ok_or_else(|| AddressManagerError::InvalidOperation)?; + if below_size < (MINIMAL_SPLIT_SPACE) { + below_size = 0; + } else { + let region = self.create_region(start_addr, below_size, info, &mut param)?; + regions.push(region); + } + + // Add guest memory above the MMIO hole + let above_start = dbs_boot::layout::MMIO_LOW_END + 1; + let above_size = size + .checked_sub(below_size) + .ok_or_else(|| AddressManagerError::InvalidOperation)?; + let region = self.create_region(above_start, above_size, info, &mut param)?; + regions.push(region); + start_addr = above_start + .checked_add(above_size) + .ok_or_else(|| AddressManagerError::InvalidOperation)?; + } + } + + // Create GuestMemory object + let mut vm_memory = GuestMemoryMmap::new(); + for reg in regions.iter() { + // Allocate used guest memory addresses. + // These addresses are statically allocated, resource allocation/update should not fail. + let constraint = Constraint::new(reg.len()) + .min(reg.start_addr().raw_value()) + .max(reg.last_addr().raw_value()); + let _key = res_mgr + .allocate_mem_address(&constraint) + .ok_or(AddressManagerError::NoAvailableMemAddress)?; + let mmap_reg = self.create_mmap_region(reg.clone())?; + + vm_memory = vm_memory + .insert_region(mmap_reg.clone()) + .map_err(AddressManagerError::CreateGuestMemory)?; + self.map_to_kvm(res_mgr, ¶m, reg, mmap_reg)?; + } + + #[cfg(feature = "atomic-guest-memory")] + { + self.vm_as = Some(AddressSpace::convert_into_vm_as(vm_memory)); + } + #[cfg(not(feature = "atomic-guest-memory"))] + { + self.vm_as = Some(Arc::new(vm_memory)); + } + + let layout = AddressSpaceLayout::new( + *dbs_boot::layout::GUEST_PHYS_END, + dbs_boot::layout::GUEST_MEM_START, + *dbs_boot::layout::GUEST_MEM_END, + ); + self.address_space = Some(AddressSpace::from_regions(regions, layout)); + + Ok(()) + } + + // size unit: Byte + fn create_region( + &mut self, + start_addr: u64, + size_bytes: u64, + info: &NumaRegionInfo, + param: &mut AddressSpaceMgrBuilder, + ) -> Result> { + let mem_file_path = param.get_next_mem_file(); + let region = AddressSpaceRegion::create_default_memory_region( + GuestAddress(start_addr), + size_bytes, + info.host_numa_node_id, + param.mem_type, + &mem_file_path, + param.mem_prealloc, + false, + ) + .map_err(AddressManagerError::CreateAddressSpaceRegion)?; + let region = Arc::new(region); + + self.insert_into_numa_nodes( + ®ion, + info.guest_numa_node_id.unwrap_or(0), + &info.vcpu_ids, + ); + info!( + "create new region: guest addr 0x{:x}-0x{:x} size {}", + start_addr, + start_addr + size_bytes, + size_bytes + ); + + Ok(region) + } + + fn map_to_kvm( + &mut self, + res_mgr: &ResourceManager, + param: &AddressSpaceMgrBuilder, + reg: &Arc, + mmap_reg: Arc, + ) -> Result<()> { + // Build mapping between GPA <-> HVA, by adding kvm memory slot. + let slot = res_mgr + .allocate_kvm_mem_slot(1, None) + .ok_or(AddressManagerError::NoAvailableKvmSlot)?; + + if let Some(vmfd) = param.vmfd.as_ref() { + let host_addr = mmap_reg + .get_host_address(MemoryRegionAddress(0)) + .map_err(|_e| AddressManagerError::InvalidOperation)?; + let flags = if param.dirty_page_logging { + KVM_MEM_LOG_DIRTY_PAGES + } else { + 0 + }; + let mem_region = kvm_userspace_memory_region { + slot: slot as u32, + guest_phys_addr: reg.start_addr().raw_value(), + memory_size: reg.len() as u64, + userspace_addr: host_addr as u64, + flags, + }; + + info!( + "VM: guest memory region {:x} starts at {:x?}", + reg.start_addr().raw_value(), + host_addr + ); + // Safe because the guest regions are guaranteed not to overlap. + unsafe { vmfd.set_user_memory_region(mem_region) } + .map_err(AddressManagerError::KvmSetMemorySlot)?; + } + + self.base_to_slot + .lock() + .unwrap() + .insert(reg.start_addr().raw_value(), slot as u32); + + Ok(()) + } + + /// Mmap the address space region into current process. + pub fn create_mmap_region( + &mut self, + region: Arc, + ) -> Result> { + // Special check for 32bit host with 64bit virtual machines. + if region.len() > usize::MAX as u64 { + return Err(AddressManagerError::InvalidAddressRange( + region.start_addr().raw_value(), + region.len(), + )); + } + // The device MMIO regions may not be backed by memory files, so refuse to mmap them. + if region.region_type() == AddressSpaceRegionType::DeviceMemory { + return Err(AddressManagerError::InvalidOperation); + } + + // The GuestRegionMmap/MmapRegion will take ownership of the FileOffset object, + // so we have to duplicate the fd here. It's really a dirty design. + let file_offset = match region.file_offset().as_ref() { + Some(fo) => { + let fd = dup(fo.file().as_raw_fd()).map_err(AddressManagerError::DupFd)?; + // Safe because we have just duplicated the raw fd. + let file = unsafe { File::from_raw_fd(fd) }; + let file_offset = FileOffset::new(file, fo.start()); + Some(file_offset) + } + None => None, + }; + let perm_flags = if (region.perm_flags() & libc::MAP_POPULATE) != 0 && region.is_hugepage() + { + // mmap(MAP_POPULATE) conflicts with madive(MADV_HUGEPAGE) because mmap(MAP_POPULATE) + // will pre-fault in all memory with normal pages before madive(MADV_HUGEPAGE) gets + // called. So remove the MAP_POPULATE flag and memory will be faulted in by working + // threads. + region.perm_flags() & (!libc::MAP_POPULATE) + } else { + region.perm_flags() + }; + let mmap_reg = MmapRegion::build( + file_offset, + region.len() as usize, + libc::PROT_READ | libc::PROT_WRITE, + perm_flags, + ) + .map_err(AddressManagerError::MmapGuestMemory)?; + + if region.is_anonpage() { + self.configure_anon_mem(&mmap_reg)?; + } + if let Some(node_id) = region.host_numa_node_id() { + self.configure_numa(&mmap_reg, node_id)?; + } + if region.is_hugepage() { + self.configure_thp_and_prealloc(®ion, &mmap_reg)?; + } + + let reg = GuestRegionImpl::new(mmap_reg, region.start_addr()) + .map_err(AddressManagerError::CreateGuestMemory)?; + Ok(Arc::new(reg)) + } + + fn configure_anon_mem(&self, mmap_reg: &MmapRegion) -> Result<()> { + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_DONTFORK, + ) + } + .map_err(AddressManagerError::Madvise) + } + + fn configure_numa(&self, mmap_reg: &MmapRegion, node_id: u32) -> Result<()> { + let nodemask = 1_u64 + .checked_shl(node_id) + .ok_or_else(|| AddressManagerError::InvalidOperation)?; + let res = unsafe { + libc::syscall( + libc::SYS_mbind, + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + MPOL_PREFERRED, + &nodemask as *const u64, + MAX_NODE, + MPOL_MF_MOVE, + ) + }; + if res < 0 { + warn!( + "failed to mbind memory to host_numa_node_id {}: this may affect performance", + node_id + ); + } + Ok(()) + } + + // We set Transparent Huge Page (THP) through mmap to increase performance. + // In order to reduce the impact of page fault on performance, we start several threads (up to MAX_PRE_ALLOC_THREAD) to touch every 4k page of the memory region to manually do memory pre-allocation. + // The reason why we don't use mmap to enable THP and pre-alloction is that THP setting won't take effect in this operation (tested in kernel 4.9) + fn configure_thp_and_prealloc( + &mut self, + region: &Arc, + mmap_reg: &MmapRegion, + ) -> Result<()> { + debug!( + "Setting MADV_HUGEPAGE on AddressSpaceRegion addr {:x?} len {:x?}", + mmap_reg.as_ptr(), + mmap_reg.size() + ); + + // Safe because we just create the MmapRegion + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_HUGEPAGE, + ) + } + .map_err(AddressManagerError::Madvise)?; + + if region.perm_flags() & libc::MAP_POPULATE > 0 { + // Touch every 4k page to trigger allocation. The step is 4K instead of 2M to ensure + // pre-allocation when running out of huge pages. + const PAGE_SIZE: u64 = 4096; + const PAGE_SHIFT: u32 = 12; + let addr = mmap_reg.as_ptr() as u64; + // Here we use >> PAGE_SHIFT to calculate how many 4K pages in the memory region. + let npage = (mmap_reg.size() as u64) >> PAGE_SHIFT; + + let mut touch_thread = ((mmap_reg.size() as u64) >> PRE_ALLOC_GRANULARITY) + 1; + if touch_thread > MAX_PRE_ALLOC_THREAD { + touch_thread = MAX_PRE_ALLOC_THREAD; + } + + let per_npage = npage / touch_thread; + for n in 0..touch_thread { + let start_npage = per_npage * n; + let end_npage = if n == (touch_thread - 1) { + npage + } else { + per_npage * (n + 1) + }; + let mut per_addr = addr + (start_npage * PAGE_SIZE); + let should_stop = self.prealloc_exit.clone(); + + let handler = thread::Builder::new() + .name("PreallocThread".to_string()) + .spawn(move || { + info!("PreallocThread start start_npage: {:?}, end_npage: {:?}, per_addr: {:?}, thread_number: {:?}", + start_npage, end_npage, per_addr, touch_thread ); + for _ in start_npage..end_npage { + if should_stop.load(Ordering::Acquire) { + info!("PreallocThread stop start_npage: {:?}, end_npage: {:?}, per_addr: {:?}, thread_number: {:?}", + start_npage, end_npage, per_addr, touch_thread); + break; + } + + // Reading from a THP page may be served by the zero page, so only + // write operation could ensure THP memory allocation. So use + // the compare_exchange(old_val, old_val) trick to trigger allocation. + let addr_ptr = per_addr as *mut u8; + let read_byte = unsafe { std::ptr::read_volatile(addr_ptr) }; + let atomic_u8 : &AtomicU8 = unsafe {&*(addr_ptr as *mut AtomicU8)}; + let _ = atomic_u8.compare_exchange(read_byte, read_byte, Ordering::SeqCst, Ordering::SeqCst); + per_addr += PAGE_SIZE; + } + + info!("PreallocThread done start_npage: {:?}, end_npage: {:?}, per_addr: {:?}, thread_number: {:?}", + start_npage, end_npage, per_addr, touch_thread ); + }); + + match handler { + Err(e) => error!( + "Failed to create working thread for async pre-allocation, {:?}. This may affect performance stability at the start of the workload.", + e + ), + Ok(hdl) => self.prealloc_handlers.push(hdl), + } + } + } + + Ok(()) + } + + /// Get the address space object + pub fn get_address_space(&self) -> Option<&AddressSpace> { + self.address_space.as_ref() + } + + /// Get the default guest memory object, which will be used to access virtual machine's default + /// guest memory. + pub fn get_vm_as(&self) -> Option<&GuestAddressSpaceImpl> { + self.vm_as.as_ref() + } + + /// Get the base to slot map + pub fn get_base_to_slot_map(&self) -> Arc>> { + self.base_to_slot.clone() + } + + /// get numa nodes infos from address space manager. + pub fn get_numa_nodes(&self) -> &BTreeMap { + &self.numa_nodes + } + + /// add cpu and memory numa informations to BtreeMap + fn insert_into_numa_nodes( + &mut self, + region: &Arc, + guest_numa_node_id: u32, + vcpu_ids: &[u32], + ) { + let node = self + .numa_nodes + .entry(guest_numa_node_id) + .or_insert(NumaNode::new()); + node.add_info(&NumaNodeInfo { + base: region.start_addr(), + size: region.len(), + }); + node.add_vcpu_ids(vcpu_ids); + } + + /// get address space layout from address space manager. + pub fn get_layout(&self) -> Result { + self.address_space + .as_ref() + .map(|v| v.layout()) + .ok_or(AddressManagerError::GuestMemoryNotInitialized) + } + + /// Wait for the pre-allocation working threads to finish work. + /// + /// Force all working threads to exit if `stop` is true. + pub fn wait_prealloc(&mut self, stop: bool) -> Result<()> { + if stop { + self.prealloc_exit.store(true, Ordering::Release); + } + while let Some(handlers) = self.prealloc_handlers.pop() { + if let Err(e) = handlers.join() { + error!("wait_prealloc join fail {:?}", e); + return Err(AddressManagerError::JoinFail); + } + } + Ok(()) + } +} + +impl Default for AddressSpaceMgr { + /// Create a new empty AddressSpaceMgr + fn default() -> Self { + AddressSpaceMgr { + address_space: None, + vm_as: None, + base_to_slot: Arc::new(Mutex::new(HashMap::new())), + prealloc_handlers: Vec::new(), + prealloc_exit: Arc::new(AtomicBool::new(false)), + numa_nodes: BTreeMap::new(), + } + } +} + +#[cfg(test)] +mod tests { + use dbs_boot::layout::GUEST_MEM_START; + use std::ops::Deref; + + use vm_memory::{Bytes, GuestAddressSpace, GuestMemory, GuestMemoryRegion}; + use vmm_sys_util::tempfile::TempFile; + + use super::*; + + #[test] + fn test_create_address_space() { + let res_mgr = ResourceManager::new(None); + let mem_size = 128 << 20; + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: vec![1, 2], + }]; + let builder = AddressSpaceMgrBuilder::new("shmem", "").unwrap(); + let as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + let vm_as = as_mgr.get_vm_as().unwrap(); + let guard = vm_as.memory(); + let gmem = guard.deref(); + assert_eq!(gmem.num_regions(), 1); + + let reg = gmem + .find_region(GuestAddress(GUEST_MEM_START + mem_size - 1)) + .unwrap(); + assert_eq!(reg.start_addr(), GuestAddress(GUEST_MEM_START)); + assert_eq!(reg.len(), mem_size); + assert!(gmem + .find_region(GuestAddress(GUEST_MEM_START + mem_size)) + .is_none()); + assert!(reg.file_offset().is_some()); + + let buf = [0x1u8, 0x2u8, 0x3u8, 0x4u8, 0x5u8]; + gmem.write_slice(&buf, GuestAddress(GUEST_MEM_START)) + .unwrap(); + + // Update middle of mapped memory region + let mut val = 0xa5u8; + gmem.write_obj(val, GuestAddress(GUEST_MEM_START + 0x1)) + .unwrap(); + val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x1)).unwrap(); + assert_eq!(val, 0xa5); + val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x0)).unwrap(); + assert_eq!(val, 1); + val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x2)).unwrap(); + assert_eq!(val, 3); + val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x5)).unwrap(); + assert_eq!(val, 0); + + // Read ahead of mapped memory region + assert!(gmem + .read_obj::(GuestAddress(GUEST_MEM_START + mem_size)) + .is_err()); + + let res_mgr = ResourceManager::new(None); + let mem_size = dbs_boot::layout::MMIO_LOW_START + (1 << 30); + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: vec![1, 2], + }]; + let builder = AddressSpaceMgrBuilder::new("shmem", "").unwrap(); + let as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + let vm_as = as_mgr.get_vm_as().unwrap(); + let guard = vm_as.memory(); + let gmem = guard.deref(); + #[cfg(target_arch = "x86_64")] + assert_eq!(gmem.num_regions(), 2); + #[cfg(target_arch = "aarch64")] + assert_eq!(gmem.num_regions(), 1); + + // Test dropping GuestMemoryMmap object releases all resources. + for _ in 0..10000 { + let res_mgr = ResourceManager::new(None); + let mem_size = 1 << 20; + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: vec![1, 2], + }]; + let builder = AddressSpaceMgrBuilder::new("shmem", "").unwrap(); + let _as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + } + let file = TempFile::new().unwrap().into_file(); + let fd = file.as_raw_fd(); + // fd should be small enough if there's no leaking of fds. + assert!(fd < 1000); + } + + #[test] + fn test_address_space_mgr_get_boundary() { + let layout = AddressSpaceLayout::new( + *dbs_boot::layout::GUEST_PHYS_END, + dbs_boot::layout::GUEST_MEM_START, + *dbs_boot::layout::GUEST_MEM_END, + ); + let res_mgr = ResourceManager::new(None); + let mem_size = 128 << 20; + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: vec![1, 2], + }]; + let builder = AddressSpaceMgrBuilder::new("shmem", "").unwrap(); + let as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + assert_eq!(as_mgr.get_layout().unwrap(), layout); + } + + #[test] + fn test_address_space_mgr_get_numa_nodes() { + let res_mgr = ResourceManager::new(None); + let mem_size = 128 << 20; + let cpu_vec = vec![1, 2]; + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: cpu_vec.clone(), + }]; + let builder = AddressSpaceMgrBuilder::new("shmem", "").unwrap(); + let as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + let mut numa_node = NumaNode::new(); + numa_node.add_info(&NumaNodeInfo { + base: GuestAddress(GUEST_MEM_START), + size: mem_size, + }); + numa_node.add_vcpu_ids(&cpu_vec); + + assert_eq!(*as_mgr.get_numa_nodes().get(&0).unwrap(), numa_node); + } + + #[test] + fn test_address_space_mgr_async_prealloc() { + let res_mgr = ResourceManager::new(None); + let mem_size = 2 << 20; + let cpu_vec = vec![1, 2]; + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: cpu_vec.clone(), + }]; + let mut builder = AddressSpaceMgrBuilder::new("hugeshmem", "").unwrap(); + builder.toggle_prealloc(true); + let mut as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + as_mgr.wait_prealloc(false).unwrap(); + } + + #[test] + fn test_address_space_mgr_builder() { + let mut builder = AddressSpaceMgrBuilder::new("shmem", "/tmp/shmem").unwrap(); + + assert_eq!(builder.mem_type, "shmem"); + assert_eq!(builder.mem_file, "/tmp/shmem"); + assert_eq!(builder.mem_index, 0); + assert_eq!(builder.mem_suffix, true); + assert_eq!(builder.mem_prealloc, false); + assert_eq!(builder.dirty_page_logging, false); + assert!(builder.vmfd.is_none()); + + assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem0"); + assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem1"); + assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem2"); + assert_eq!(builder.mem_index, 3); + + builder.toggle_file_suffix(false); + assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem"); + assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem"); + assert_eq!(builder.mem_index, 3); + + builder.toggle_prealloc(true); + builder.toggle_dirty_page_logging(true); + assert_eq!(builder.mem_prealloc, true); + assert_eq!(builder.dirty_page_logging, true); + } + + #[test] + fn test_configure_invalid_numa() { + let res_mgr = ResourceManager::new(None); + let mem_size = 128 << 20; + let numa_region_infos = vec![NumaRegionInfo { + size: mem_size >> 20, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids: vec![1, 2], + }]; + let builder = AddressSpaceMgrBuilder::new("shmem", "").unwrap(); + let as_mgr = builder.build(&res_mgr, &numa_region_infos).unwrap(); + let mmap_reg = MmapRegion::new(8).unwrap(); + + assert!(as_mgr.configure_numa(&mmap_reg, u32::MAX).is_err()); + } +} diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index f25b8c13d6b1..727906161750 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -4,5 +4,9 @@ //! Dragonball is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual //! Machine(KVM) which is optimized for container workloads. +/// Address space manager for virtual machines. +pub mod address_space_manager; /// Resource manager for virtual machines. pub mod resource_manager; +/// Virtual machine manager for virtual machines. +pub mod vm; diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs new file mode 100644 index 000000000000..c461b6a28288 --- /dev/null +++ b/src/dragonball/src/vm/mod.rs @@ -0,0 +1,17 @@ +// Copyright (C) 2021 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_derive::{Deserialize, Serialize}; + +/// Configuration information for user defined NUMA nodes. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct NumaRegionInfo { + /// memory size for this region (unit: MiB) + pub size: u64, + /// numa node id on host for this region + pub host_numa_node_id: Option, + /// numa node id on guest for this region + pub guest_numa_node_id: Option, + /// vcpu ids belonging to this region + pub vcpu_ids: Vec, +} From 3c45c0715f71d299856444a3bdf3355a9219af8c Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 5 May 2022 01:00:54 +0800 Subject: [PATCH 0056/1953] dragonball: add console manager. Console manager to manage frontend and backend console devcies. A virtual console are composed up of two parts: frontend in virtual machine and backend in host OS. A frontend may be serial port, virtio-console etc, a backend may be stdio or Unix domain socket. The manager connects the frontend with the backend. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 10 + .../src/device_manager/console_manager.rs | 430 ++++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 20 + src/dragonball/src/lib.rs | 4 + 4 files changed, 464 insertions(+) create mode 100644 src/dragonball/src/device_manager/console_manager.rs create mode 100644 src/dragonball/src/device_manager/mod.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 9b5998d91593..432a2e80814d 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -11,10 +11,13 @@ edition = "2018" [dependencies] arc-swap = "1.5.0" +bytes = "1.1.0" dbs-address-space = "0.1.0" dbs-allocator = "0.1.0" dbs-boot = "0.2.0" dbs-device = "0.1.0" +dbs-legacy-devices = "0.1.0" +dbs-utils = "0.1.0" kvm-bindings = "0.5.0" kvm-ioctls = "0.11.0" libc = "0.2.39" @@ -29,5 +32,12 @@ thiserror = "1" vmm-sys-util = "0.9.0" vm-memory = { version = "0.7.0", features = ["backend-mmap"] } +[dev-dependencies] +slog-term = "2.9.0" +slog-async = "2.7.0" + [features] atomic-guest-memory = [] + +[patch.'crates-io'] +dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } diff --git a/src/dragonball/src/device_manager/console_manager.rs b/src/dragonball/src/device_manager/console_manager.rs new file mode 100644 index 000000000000..617b98b167b5 --- /dev/null +++ b/src/dragonball/src/device_manager/console_manager.rs @@ -0,0 +1,430 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +//! Virtual machine console device manager. +//! +//! A virtual console are composed up of two parts: frontend in virtual machine and backend in +//! host OS. A frontend may be serial port, virtio-console etc, a backend may be stdio or Unix +//! domain socket. The manager connects the frontend with the backend. +use std::io::{self, Read}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use bytes::{BufMut, BytesMut}; +use dbs_legacy_devices::{ConsoleHandler, SerialDevice}; +use dbs_utils::epoll_manager::{ + EpollManager, EventOps, EventSet, Events, MutEventSubscriber, SubscriberId, +}; +use vmm_sys_util::terminal::Terminal; + +use super::{DeviceMgrError, Result}; + +const EPOLL_EVENT_SERIAL: u32 = 0; +const EPOLL_EVENT_SERIAL_DATA: u32 = 1; +const EPOLL_EVENT_STDIN: u32 = 2; +// Maximal backend throughput for every data transaction. +const MAX_BACKEND_THROUGHPUT: usize = 64; + +/// Errors related to Console manager operations. +#[derive(Debug, thiserror::Error)] +pub enum ConsoleManagerError { + /// Cannot create unix domain socket for serial port + #[error("cannot create socket for serial console")] + CreateSerialSock(#[source] std::io::Error), + + /// An operation on the epoll instance failed due to resource exhaustion or bad configuration. + #[error("failure while managing epoll event for console fd")] + EpollMgr(#[source] dbs_utils::epoll_manager::Error), + + /// Cannot set mode for terminal. + #[error("failure while setting attribute for terminal")] + StdinHandle(#[source] vmm_sys_util::errno::Error), +} + +enum Backend { + StdinHandle(std::io::Stdin), + SockPath(String), +} + +/// Console manager to manage frontend and backend console devices. +pub struct ConsoleManager { + epoll_mgr: EpollManager, + logger: slog::Logger, + subscriber_id: Option, + backend: Option, +} + +impl ConsoleManager { + /// Create a console manager instance. + pub fn new(epoll_mgr: EpollManager, logger: &slog::Logger) -> Self { + let logger = logger.new(slog::o!("subsystem" => "console_manager")); + ConsoleManager { + epoll_mgr, + logger, + subscriber_id: Default::default(), + backend: None, + } + } + + /// Create a console backend device by using stdio streams. + pub fn create_stdio_console(&mut self, device: Arc>) -> Result<()> { + let stdin_handle = std::io::stdin(); + stdin_handle + .lock() + .set_raw_mode() + .map_err(|e| DeviceMgrError::ConsoleManager(ConsoleManagerError::StdinHandle(e)))?; + + let handler = ConsoleEpollHandler::new(device, Some(stdin_handle), None, &self.logger); + self.subscriber_id = Some(self.epoll_mgr.add_subscriber(Box::new(handler))); + self.backend = Some(Backend::StdinHandle(std::io::stdin())); + + Ok(()) + } + + /// Create s console backend device by using Unix Domain socket. + pub fn create_socket_console( + &mut self, + device: Arc>, + sock_path: String, + ) -> Result<()> { + let sock_listener = Self::bind_domain_socket(&sock_path).map_err(|e| { + DeviceMgrError::ConsoleManager(ConsoleManagerError::CreateSerialSock(e)) + })?; + let handler = ConsoleEpollHandler::new(device, None, Some(sock_listener), &self.logger); + + self.subscriber_id = Some(self.epoll_mgr.add_subscriber(Box::new(handler))); + self.backend = Some(Backend::SockPath(sock_path)); + + Ok(()) + } + + /// Reset the host side terminal to canonical mode. + pub fn reset_console(&self) -> Result<()> { + if let Some(Backend::StdinHandle(stdin_handle)) = self.backend.as_ref() { + stdin_handle + .lock() + .set_canon_mode() + .map_err(|e| DeviceMgrError::ConsoleManager(ConsoleManagerError::StdinHandle(e)))?; + } + + Ok(()) + } + + fn bind_domain_socket(serial_path: &str) -> std::result::Result { + let path = Path::new(serial_path); + if path.is_file() { + let _ = std::fs::remove_file(serial_path); + } + + UnixListener::bind(path) + } +} + +struct ConsoleEpollHandler { + device: Arc>, + stdin_handle: Option, + sock_listener: Option, + sock_conn: Option, + logger: slog::Logger, +} + +impl ConsoleEpollHandler { + fn new( + device: Arc>, + stdin_handle: Option, + sock_listener: Option, + logger: &slog::Logger, + ) -> Self { + ConsoleEpollHandler { + device, + stdin_handle, + sock_listener, + sock_conn: None, + logger: logger.new(slog::o!("subsystem" => "console_manager")), + } + } + + fn uds_listener_accept(&mut self, ops: &mut EventOps) -> std::io::Result<()> { + if self.sock_conn.is_some() { + slog::warn!(self.logger, + "UDS for serial port 1 already exists, reject the new connection"; + "subsystem" => "console_mgr", + ); + // Do not expected poisoned lock. + let _ = self.sock_listener.as_mut().unwrap().accept(); + } else { + // Safe to unwrap() because self.sock_conn is Some(). + let (conn_sock, _) = self.sock_listener.as_ref().unwrap().accept()?; + let events = Events::with_data(&conn_sock, EPOLL_EVENT_SERIAL_DATA, EventSet::IN); + if let Err(e) = ops.add(events) { + slog::error!(self.logger, + "failed to register epoll event for serial, {:?}", e; + "subsystem" => "console_mgr", + ); + return Err(std::io::Error::last_os_error()); + } + + let conn_sock_copy = conn_sock.try_clone()?; + // Do not expected poisoned lock. + self.device + .lock() + .unwrap() + .set_output_stream(Some(Box::new(conn_sock_copy))); + + self.sock_conn = Some(conn_sock); + } + + Ok(()) + } + + fn uds_read_in(&mut self, ops: &mut EventOps) -> std::io::Result<()> { + let mut should_drop = true; + + if let Some(conn_sock) = self.sock_conn.as_mut() { + let mut out = [0u8; MAX_BACKEND_THROUGHPUT]; + match conn_sock.read(&mut out[..]) { + Ok(0) => { + // Zero-length read means EOF. Remove this conn sock. + self.device + .lock() + .expect("console: poisoned console lock") + .set_output_stream(None); + } + Ok(count) => { + self.device + .lock() + .expect("console: poisoned console lock") + .raw_input(&out[..count])?; + should_drop = false; + } + Err(e) => { + slog::warn!(self.logger, + "error while reading serial conn sock: {:?}", e; + "subsystem" => "console_mgr" + ); + self.device + .lock() + .expect("console: poisoned console lock") + .set_output_stream(None); + } + } + } + + if should_drop { + assert!(self.sock_conn.is_some()); + // Safe to unwrap() because self.sock_conn is Some(). + let sock_conn = self.sock_conn.take().unwrap(); + let events = Events::with_data(&sock_conn, EPOLL_EVENT_SERIAL_DATA, EventSet::IN); + if let Err(e) = ops.remove(events) { + slog::error!(self.logger, + "failed deregister epoll event for UDS, {:?}", e; + "subsystem" => "console_mgr" + ); + } + } + + Ok(()) + } + + fn stdio_read_in(&mut self, ops: &mut EventOps) -> std::io::Result<()> { + let mut should_drop = true; + + if let Some(handle) = self.stdin_handle.as_ref() { + let mut out = [0u8; MAX_BACKEND_THROUGHPUT]; + // Safe to unwrap() because self.stdin_handle is Some(). + let stdin_lock = handle.lock(); + match stdin_lock.read_raw(&mut out[..]) { + Ok(0) => { + // Zero-length read indicates EOF. Remove from pollables. + self.device + .lock() + .expect("console: poisoned console lock") + .set_output_stream(None); + } + Ok(count) => { + self.device + .lock() + .expect("console: poisoned console lock") + .raw_input(&out[..count])?; + should_drop = false; + } + Err(e) => { + slog::warn!(self.logger, + "error while reading stdin: {:?}", e; + "subsystem" => "console_mgr" + ); + self.device + .lock() + .expect("console: poisoned console lock") + .set_output_stream(None); + } + } + } + + if should_drop { + let events = Events::with_data_raw(libc::STDIN_FILENO, EPOLL_EVENT_STDIN, EventSet::IN); + if let Err(e) = ops.remove(events) { + slog::error!(self.logger, + "failed to deregister epoll event for stdin, {:?}", e; + "subsystem" => "console_mgr" + ); + } + } + + Ok(()) + } +} + +impl MutEventSubscriber for ConsoleEpollHandler { + fn process(&mut self, events: Events, ops: &mut EventOps) { + slog::trace!(self.logger, "ConsoleEpollHandler::process()"); + let slot = events.data(); + match slot { + EPOLL_EVENT_SERIAL => { + if let Err(e) = self.uds_listener_accept(ops) { + slog::warn!(self.logger, "failed to accept incoming connection, {:?}", e); + } + } + EPOLL_EVENT_SERIAL_DATA => { + if let Err(e) = self.uds_read_in(ops) { + slog::warn!(self.logger, "failed to read data from UDS, {:?}", e); + } + } + EPOLL_EVENT_STDIN => { + if let Err(e) = self.stdio_read_in(ops) { + slog::warn!(self.logger, "failed to read data from stdin, {:?}", e); + } + } + _ => slog::error!(self.logger, "unknown epoll slot number {}", slot), + } + } + + fn init(&mut self, ops: &mut EventOps) { + slog::trace!(self.logger, "ConsoleEpollHandler::init()"); + + if self.stdin_handle.is_some() { + slog::info!(self.logger, "ConsoleEpollHandler: stdin handler"); + let events = Events::with_data_raw(libc::STDIN_FILENO, EPOLL_EVENT_STDIN, EventSet::IN); + if let Err(e) = ops.add(events) { + slog::error!( + self.logger, + "failed to register epoll event for stdin, {:?}", + e + ); + } + } + if let Some(sock) = self.sock_listener.as_ref() { + slog::info!(self.logger, "ConsoleEpollHandler: sock listener"); + let events = Events::with_data(sock, EPOLL_EVENT_SERIAL, EventSet::IN); + if let Err(e) = ops.add(events) { + slog::error!( + self.logger, + "failed to register epoll event for UDS listener, {:?}", + e + ); + } + } + + if let Some(conn) = self.sock_conn.as_ref() { + slog::info!(self.logger, "ConsoleEpollHandler: sock connection"); + let events = Events::with_data(conn, EPOLL_EVENT_SERIAL_DATA, EventSet::IN); + if let Err(e) = ops.add(events) { + slog::error!( + self.logger, + "failed to register epoll event for UDS connection, {:?}", + e + ); + } + } + } +} + +/// Writer to process guest kernel dmesg. +pub struct DmesgWriter { + buf: BytesMut, + logger: slog::Logger, +} + +impl io::Write for DmesgWriter { + /// 0000000 [ 0 . 0 3 4 9 1 6 ] R + /// 5b 20 20 20 20 30 2e 30 33 34 39 31 36 5d 20 52 + /// 0000020 u n / s b i n / i n i t a s + /// 75 6e 20 2f 73 62 69 6e 2f 69 6e 69 74 20 61 73 + /// 0000040 i n i t p r o c e s s \r \n [ + /// + /// dmesg message end a line with /r/n . When redirect message to logger, we should + /// remove the /r/n . + fn write(&mut self, buf: &[u8]) -> io::Result { + let arr: Vec<&[u8]> = buf.split(|c| *c == b'\n').collect(); + let count = arr.len(); + + for (i, sub) in arr.iter().enumerate() { + if sub.is_empty() { + if !self.buf.is_empty() { + slog::info!( + self.logger, + "{}", + String::from_utf8_lossy(self.buf.as_ref()).trim_end() + ); + self.buf.clear(); + } + } else if sub.len() < buf.len() && i < count - 1 { + slog::info!( + self.logger, + "{}{}", + String::from_utf8_lossy(self.buf.as_ref()).trim_end(), + String::from_utf8_lossy(sub).trim_end(), + ); + self.buf.clear(); + } else { + self.buf.put_slice(sub); + } + } + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use slog::Drain; + use std::io::Write; + + fn create_logger() -> slog::Logger { + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + slog::Logger::root(drain, slog::o!()) + } + + #[test] + fn test_dmesg_writer() { + let mut writer = DmesgWriter { + buf: Default::default(), + logger: create_logger(), + }; + + writer.flush().unwrap(); + writer.write("".as_bytes()).unwrap(); + writer.write("\n".as_bytes()).unwrap(); + writer.write("\n\n".as_bytes()).unwrap(); + writer.write("\n\n\n".as_bytes()).unwrap(); + writer.write("12\n23\n34\n56".as_bytes()).unwrap(); + writer.write("78".as_bytes()).unwrap(); + writer.write("90\n".as_bytes()).unwrap(); + writer.flush().unwrap(); + } + + // TODO: add unit tests for console manager +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs new file mode 100644 index 000000000000..fc4d12b80872 --- /dev/null +++ b/src/dragonball/src/device_manager/mod.rs @@ -0,0 +1,20 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Device manager to manage IO devices for a virtual machine. + +/// Virtual machine console device manager. +pub mod console_manager; +/// Console Manager for virtual machines console device. +pub use self::console_manager::ConsoleManager; + +/// Errors related to device manager operations. +#[derive(Debug, thiserror::Error)] +pub enum DeviceMgrError { + /// Failed to manage console devices. + #[error(transparent)] + ConsoleManager(console_manager::ConsoleManagerError), +} + +/// Specialized version of `std::result::Result` for device manager operations. +pub type Result = ::std::result::Result; diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index 727906161750..a3295e8f8f77 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -4,8 +4,12 @@ //! Dragonball is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual //! Machine(KVM) which is optimized for container workloads. +#![warn(missing_docs)] + /// Address space manager for virtual machines. pub mod address_space_manager; +/// Device manager for virtual machines. +pub mod device_manager; /// Resource manager for virtual machines. pub mod resource_manager; /// Virtual machine manager for virtual machines. From 0bcb422fcb5f618f1a43535f1425312cf8843304 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 5 May 2022 01:10:58 +0800 Subject: [PATCH 0057/1953] dragonball: add legacy devices manager The legacy devices manager is used for managing legacy devices. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/src/device_manager/legacy.rs | 155 ++++++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 3 + 2 files changed, 158 insertions(+) create mode 100644 src/dragonball/src/device_manager/legacy.rs diff --git a/src/dragonball/src/device_manager/legacy.rs b/src/dragonball/src/device_manager/legacy.rs new file mode 100644 index 000000000000..9dbb300d4b5f --- /dev/null +++ b/src/dragonball/src/device_manager/legacy.rs @@ -0,0 +1,155 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +//! Device Manager for Legacy Devices. + +use std::io; +use std::sync::{Arc, Mutex}; + +use dbs_device::device_manager::Error as IoManagerError; +use dbs_legacy_devices::SerialDevice; +use vmm_sys_util::eventfd::EventFd; + +// The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself. +const I8042_DATA_PORT: u16 = 0x60; + +/// Errors generated by legacy device manager. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Cannot add legacy device to Bus. + #[error("bus failure while managing legacy device")] + BusError(#[source] IoManagerError), + + /// Cannot create EventFd. + #[error("failure while reading EventFd file descriptor")] + EventFd(#[source] io::Error), + + /// Failed to register/deregister interrupt. + #[error("failure while managing interrupt for legacy device")] + IrqManager(#[source] vmm_sys_util::errno::Error), +} + +/// The `LegacyDeviceManager` is a wrapper that is used for registering legacy devices +/// on an I/O Bus. +/// +/// It currently manages the uart and i8042 devices. The `LegacyDeviceManger` should be initialized +/// only by using the constructor. +pub struct LegacyDeviceManager { + #[cfg(target_arch = "x86_64")] + i8042_reset_eventfd: EventFd, + pub(crate) com1_device: Arc>, + _com1_eventfd: EventFd, + pub(crate) com2_device: Arc>, + _com2_eventfd: EventFd, +} + +impl LegacyDeviceManager { + /// Get the serial device for com1. + pub fn get_com1_serial(&self) -> Arc> { + self.com1_device.clone() + } + + /// Get the serial device for com2 + pub fn get_com2_serial(&self) -> Arc> { + self.com2_device.clone() + } +} + +#[cfg(target_arch = "x86_64")] +pub(crate) mod x86_64 { + use super::*; + use dbs_device::device_manager::IoManager; + use dbs_device::resources::Resource; + use dbs_legacy_devices::{EventFdTrigger, I8042Device, I8042DeviceMetrics}; + use kvm_ioctls::VmFd; + + pub(crate) const COM1_IRQ: u32 = 4; + pub(crate) const COM1_PORT1: u16 = 0x3f8; + pub(crate) const COM2_IRQ: u32 = 3; + pub(crate) const COM2_PORT1: u16 = 0x2f8; + + type Result = ::std::result::Result; + + impl LegacyDeviceManager { + /// Create a LegacyDeviceManager instance handling legacy devices (uart, i8042). + pub fn create_manager(bus: &mut IoManager, vm_fd: Option>) -> Result { + let (com1_device, com1_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), COM1_IRQ, COM1_PORT1)?; + let (com2_device, com2_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), COM2_IRQ, COM2_PORT1)?; + + let exit_evt = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let i8042_device = Arc::new(Mutex::new(I8042Device::new( + EventFdTrigger::new(exit_evt.try_clone().map_err(Error::EventFd)?), + Arc::new(I8042DeviceMetrics::default()), + ))); + let resources = [Resource::PioAddressRange { + // 0x60 and 0x64 are the io ports that i8042 devices used. + // We register pio address range from 0x60 - 0x64 with base I8042_DATA_PORT for i8042 to use. + base: I8042_DATA_PORT, + size: 0x5, + }]; + bus.register_device_io(i8042_device, &resources) + .map_err(Error::BusError)?; + + Ok(LegacyDeviceManager { + i8042_reset_eventfd: exit_evt, + com1_device, + _com1_eventfd: com1_eventfd, + com2_device, + _com2_eventfd: com2_eventfd, + }) + } + + /// Get the eventfd for exit notification. + pub fn get_reset_eventfd(&self) -> Result { + self.i8042_reset_eventfd.try_clone().map_err(Error::EventFd) + } + + fn create_com_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + irq: u32, + port_base: u16, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(SerialDevice::new( + eventfd.try_clone().map_err(Error::EventFd)?, + ))); + // port_base defines the base port address for the COM devices. + // Since every COM device has 8 data registers so we register the pio address range as size 0x8. + let resources = [Resource::PioAddressRange { + base: port_base, + size: 0x8, + }]; + bus.register_device_io(device.clone(), &resources) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(target_arch = "x86_64")] + use super::*; + + #[test] + #[cfg(target_arch = "x86_64")] + fn test_create_legacy_device_manager() { + let mut bus = dbs_device::device_manager::IoManager::new(); + let mgr = LegacyDeviceManager::create_manager(&mut bus, None).unwrap(); + let _exit_fd = mgr.get_reset_eventfd().unwrap(); + } +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index fc4d12b80872..2dad8b8c1044 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -8,6 +8,9 @@ pub mod console_manager; /// Console Manager for virtual machines console device. pub use self::console_manager::ConsoleManager; +mod legacy; +pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; + /// Errors related to device manager operations. #[derive(Debug, thiserror::Error)] pub enum DeviceMgrError { From 6850ef99aec064c425ac3d64c5287a8fa80dc59e Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 5 May 2022 01:29:13 +0800 Subject: [PATCH 0058/1953] dragonball: add configuration manager. It is used for managing a group of configuration information. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/src/config_manager.rs | 724 +++++++++++++++++++++++++++ src/dragonball/src/lib.rs | 2 + 2 files changed, 726 insertions(+) create mode 100644 src/dragonball/src/config_manager.rs diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs new file mode 100644 index 000000000000..de16ab0fbffa --- /dev/null +++ b/src/dragonball/src/config_manager.rs @@ -0,0 +1,724 @@ +// Copyright (C) 2020-2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::convert::TryInto; +use std::io; +use std::ops::{Index, IndexMut}; +use std::sync::Arc; + +use dbs_device::DeviceIo; +use dbs_utils::rate_limiter::{RateLimiter, TokenBucket}; +use serde_derive::{Deserialize, Serialize}; + +/// Trait for generic configuration information. +pub trait ConfigItem { + /// Related errors. + type Err; + + /// Get the unique identifier of the configuration item. + fn id(&self) -> &str; + + /// Check whether current configuration item conflicts with another one. + fn check_conflicts(&self, other: &Self) -> std::result::Result<(), Self::Err>; +} + +/// Struct to manage a group of configuration items. +#[derive(Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct ConfigInfos +where + T: ConfigItem + Clone, +{ + configs: Vec, +} + +impl ConfigInfos +where + T: ConfigItem + Clone + Default, +{ + /// Constructor + pub fn new() -> Self { + ConfigInfos::default() + } + + /// Insert a configuration item in the group. + pub fn insert(&mut self, config: T) -> std::result::Result<(), T::Err> { + for item in self.configs.iter() { + config.check_conflicts(item)?; + } + self.configs.push(config); + + Ok(()) + } + + /// Update a configuration item in the group. + pub fn update(&mut self, config: T, err: T::Err) -> std::result::Result<(), T::Err> { + match self.get_index_by_id(&config) { + None => Err(err), + Some(index) => { + for (idx, item) in self.configs.iter().enumerate() { + if idx != index { + config.check_conflicts(item)?; + } + } + self.configs[index] = config; + Ok(()) + } + } + } + + /// Insert or update a configuration item in the group. + pub fn insert_or_update(&mut self, config: T) -> std::result::Result<(), T::Err> { + match self.get_index_by_id(&config) { + None => { + for item in self.configs.iter() { + config.check_conflicts(item)?; + } + + self.configs.push(config) + } + Some(index) => { + for (idx, item) in self.configs.iter().enumerate() { + if idx != index { + config.check_conflicts(item)?; + } + } + self.configs[index] = config; + } + } + + Ok(()) + } + + /// Remove the matching configuration entry. + pub fn remove(&mut self, config: &T) -> Option { + if let Some(index) = self.get_index_by_id(config) { + Some(self.configs.remove(index)) + } else { + None + } + } + + /// Returns an immutable iterator over the config items + pub fn iter(&self) -> ::std::slice::Iter { + self.configs.iter() + } + + /// Get the configuration entry with matching ID. + pub fn get_by_id(&self, item: &T) -> Option<&T> { + let id = item.id(); + + self.configs.iter().rfind(|cfg| cfg.id() == id) + } + + fn get_index_by_id(&self, item: &T) -> Option { + let id = item.id(); + self.configs.iter().position(|cfg| cfg.id() == id) + } +} + +impl Clone for ConfigInfos +where + T: ConfigItem + Clone, +{ + fn clone(&self) -> Self { + ConfigInfos { + configs: self.configs.clone(), + } + } +} + +/// Struct to maintain configuration information for a device. +pub struct DeviceConfigInfo +where + T: ConfigItem + Clone, +{ + /// Configuration information for the device object. + pub config: T, + /// The associated device object. + pub device: Option>, +} + +impl DeviceConfigInfo +where + T: ConfigItem + Clone, +{ + /// Create a new instance of ['DeviceInfoGroup']. + pub fn new(config: T) -> Self { + DeviceConfigInfo { + config, + device: None, + } + } + + /// Create a new instance of ['DeviceInfoGroup'] with optional device. + pub fn new_with_device(config: T, device: Option>) -> Self { + DeviceConfigInfo { config, device } + } + + /// Set the device object associated with the configuration. + pub fn set_device(&mut self, device: Arc) { + self.device = Some(device); + } +} + +impl Clone for DeviceConfigInfo +where + T: ConfigItem + Clone, +{ + fn clone(&self) -> Self { + DeviceConfigInfo::new_with_device(self.config.clone(), self.device.clone()) + } +} + +/// Struct to maintain configuration information for a group of devices. +pub struct DeviceConfigInfos +where + T: ConfigItem + Clone, +{ + info_list: Vec>, +} + +impl DeviceConfigInfos +where + T: ConfigItem + Clone, +{ + /// Create a new instance of ['DeviceConfigInfos']. + pub fn new() -> Self { + DeviceConfigInfos { + info_list: Vec::new(), + } + } + + /// Insert or update configuration information for a device. + pub fn insert_or_update(&mut self, config: &T) -> std::result::Result { + let device_info = DeviceConfigInfo::new(config.clone()); + Ok(match self.get_index_by_id(config) { + Some(index) => { + for (idx, info) in self.info_list.iter().enumerate() { + if idx != index { + info.config.check_conflicts(config)?; + } + } + self.info_list[index] = device_info; + index + } + None => { + for info in self.info_list.iter() { + info.config.check_conflicts(config)?; + } + self.info_list.push(device_info); + self.info_list.len() - 1 + } + }) + } + + /// Remove a device configuration information object. + pub fn remove(&mut self, index: usize) -> Option> { + if self.info_list.len() > index { + Some(self.info_list.remove(index)) + } else { + None + } + } + + #[allow(dead_code)] + /// Get number of device configuration information objects. + pub fn len(&self) -> usize { + self.info_list.len() + } + + /// Add a device configuration information object at the tail. + pub fn push(&mut self, info: DeviceConfigInfo) { + self.info_list.push(info); + } + + /// Iterator for configuration information objects. + pub fn iter(&self) -> std::slice::Iter> { + self.info_list.iter() + } + + /// Mutable iterator for configuration information objects. + pub fn iter_mut(&mut self) -> std::slice::IterMut> { + self.info_list.iter_mut() + } + + fn get_index_by_id(&self, config: &T) -> Option { + self.info_list + .iter() + .position(|info| info.config.id().eq(config.id())) + } +} + +impl Index for DeviceConfigInfos +where + T: ConfigItem + Clone, +{ + type Output = DeviceConfigInfo; + fn index(&self, idx: usize) -> &Self::Output { + &self.info_list[idx] + } +} + +impl IndexMut for DeviceConfigInfos +where + T: ConfigItem + Clone, +{ + fn index_mut(&mut self, idx: usize) -> &mut Self::Output { + &mut self.info_list[idx] + } +} + +impl Clone for DeviceConfigInfos +where + T: ConfigItem + Clone, +{ + fn clone(&self) -> Self { + DeviceConfigInfos { + info_list: self.info_list.clone(), + } + } +} + +/// Configuration information for RateLimiter token bucket. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct TokenBucketConfigInfo { + /// The size for the token bucket. A TokenBucket of `size` total capacity will take `refill_time` + /// milliseconds to go from zero tokens to total capacity. + pub size: u64, + /// Number of free initial tokens, that can be consumed at no cost. + pub one_time_burst: u64, + /// Complete refill time in milliseconds. + pub refill_time: u64, +} + +impl TokenBucketConfigInfo { + fn resize(&mut self, n: u64) { + if n != 0 { + self.size /= n; + self.one_time_burst /= n; + } + } +} + +impl From for TokenBucket { + fn from(t: TokenBucketConfigInfo) -> TokenBucket { + (&t).into() + } +} + +impl From<&TokenBucketConfigInfo> for TokenBucket { + fn from(t: &TokenBucketConfigInfo) -> TokenBucket { + TokenBucket::new(t.size, t.one_time_burst, t.refill_time) + } +} + +/// Configuration information for RateLimiter objects. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct RateLimiterConfigInfo { + /// Data used to initialize the RateLimiter::bandwidth bucket. + pub bandwidth: TokenBucketConfigInfo, + /// Data used to initialize the RateLimiter::ops bucket. + pub ops: TokenBucketConfigInfo, +} + +impl RateLimiterConfigInfo { + /// Update the bandwidth budget configuration. + pub fn update_bandwidth(&mut self, new_config: TokenBucketConfigInfo) { + self.bandwidth = new_config; + } + + /// Update the ops budget configuration. + pub fn update_ops(&mut self, new_config: TokenBucketConfigInfo) { + self.ops = new_config; + } + + /// resize the limiter to its 1/n. + pub fn resize(&mut self, n: u64) { + self.bandwidth.resize(n); + self.ops.resize(n); + } +} + +impl TryInto for &RateLimiterConfigInfo { + type Error = io::Error; + + fn try_into(self) -> Result { + RateLimiter::new( + self.bandwidth.size, + self.bandwidth.one_time_burst, + self.bandwidth.refill_time, + self.ops.size, + self.ops.one_time_burst, + self.ops.refill_time, + ) + } +} + +impl TryInto for RateLimiterConfigInfo { + type Error = io::Error; + + fn try_into(self) -> Result { + RateLimiter::new( + self.bandwidth.size, + self.bandwidth.one_time_burst, + self.bandwidth.refill_time, + self.ops.size, + self.ops.one_time_burst, + self.ops.refill_time, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, thiserror::Error)] + pub enum DummyError { + #[error("configuration entry exists")] + Exist, + } + + #[derive(Clone, Debug)] + pub struct DummyConfigInfo { + id: String, + content: String, + } + + impl ConfigItem for DummyConfigInfo { + type Err = DummyError; + + fn id(&self) -> &str { + &self.id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), DummyError> { + if self.id == other.id || self.content == other.content { + Err(DummyError::Exist) + } else { + Ok(()) + } + } + } + + type DummyConfigInfos = ConfigInfos; + + #[test] + fn test_insert_config_info() { + let mut configs = DummyConfigInfos::new(); + + let config1 = DummyConfigInfo { + id: "1".to_owned(), + content: "a".to_owned(), + }; + configs.insert(config1).unwrap(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "a"); + + // Test case: cannot insert new item with the same id. + let config2 = DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }; + configs.insert(config2).unwrap_err(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "a"); + + let config3 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.insert(config3).unwrap(); + assert_eq!(configs.configs.len(), 2); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "a"); + assert_eq!(configs.configs[1].id, "2"); + assert_eq!(configs.configs[1].content, "c"); + + // Test case: cannot insert new item with the same content. + let config4 = DummyConfigInfo { + id: "3".to_owned(), + content: "c".to_owned(), + }; + configs.insert(config4).unwrap_err(); + assert_eq!(configs.configs.len(), 2); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "a"); + assert_eq!(configs.configs[1].id, "2"); + assert_eq!(configs.configs[1].content, "c"); + } + + #[test] + fn test_update_config_info() { + let mut configs = DummyConfigInfos::new(); + + let config1 = DummyConfigInfo { + id: "1".to_owned(), + content: "a".to_owned(), + }; + configs.insert(config1).unwrap(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "a"); + + // Test case: succeed to update an existing entry + let config2 = DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }; + configs.update(config2, DummyError::Exist).unwrap(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "b"); + + // Test case: cannot update a non-existing entry + let config3 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.update(config3, DummyError::Exist).unwrap_err(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "b"); + + // Test case: cannot update an entry with conflicting content + let config4 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.insert(config4).unwrap(); + let config5 = DummyConfigInfo { + id: "1".to_owned(), + content: "c".to_owned(), + }; + configs.update(config5, DummyError::Exist).unwrap_err(); + } + + #[test] + fn test_insert_or_update_config_info() { + let mut configs = DummyConfigInfos::new(); + + let config1 = DummyConfigInfo { + id: "1".to_owned(), + content: "a".to_owned(), + }; + configs.insert_or_update(config1).unwrap(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "a"); + + // Test case: succeed to update an existing entry + let config2 = DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }; + configs.insert_or_update(config2.clone()).unwrap(); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "b"); + + // Add a second entry + let config3 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.insert_or_update(config3.clone()).unwrap(); + assert_eq!(configs.configs.len(), 2); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "b"); + assert_eq!(configs.configs[1].id, "2"); + assert_eq!(configs.configs[1].content, "c"); + + // Lookup the first entry + let config4 = configs + .get_by_id(&DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }) + .unwrap(); + assert_eq!(config4.id, config2.id); + assert_eq!(config4.content, config2.content); + + // Lookup the second entry + let config5 = configs + .get_by_id(&DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }) + .unwrap(); + assert_eq!(config5.id, config3.id); + assert_eq!(config5.content, config3.content); + + // Test case: can't insert an entry with conflicting content + let config6 = DummyConfigInfo { + id: "3".to_owned(), + content: "c".to_owned(), + }; + configs.insert_or_update(config6).unwrap_err(); + assert_eq!(configs.configs.len(), 2); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "b"); + assert_eq!(configs.configs[1].id, "2"); + assert_eq!(configs.configs[1].content, "c"); + } + + #[test] + fn test_remove_config_info() { + let mut configs = DummyConfigInfos::new(); + + let config1 = DummyConfigInfo { + id: "1".to_owned(), + content: "a".to_owned(), + }; + configs.insert_or_update(config1).unwrap(); + let config2 = DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }; + configs.insert_or_update(config2.clone()).unwrap(); + let config3 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.insert_or_update(config3.clone()).unwrap(); + assert_eq!(configs.configs.len(), 2); + assert_eq!(configs.configs[0].id, "1"); + assert_eq!(configs.configs[0].content, "b"); + assert_eq!(configs.configs[1].id, "2"); + assert_eq!(configs.configs[1].content, "c"); + + let config4 = configs + .remove(&DummyConfigInfo { + id: "1".to_owned(), + content: "no value".to_owned(), + }) + .unwrap(); + assert_eq!(config4.id, config2.id); + assert_eq!(config4.content, config2.content); + assert_eq!(configs.configs.len(), 1); + assert_eq!(configs.configs[0].id, "2"); + assert_eq!(configs.configs[0].content, "c"); + + let config5 = configs + .remove(&DummyConfigInfo { + id: "2".to_owned(), + content: "no value".to_owned(), + }) + .unwrap(); + assert_eq!(config5.id, config3.id); + assert_eq!(config5.content, config3.content); + assert_eq!(configs.configs.len(), 0); + } + + type DummyDeviceInfoList = DeviceConfigInfos; + + #[test] + fn test_insert_or_update_device_info() { + let mut configs = DummyDeviceInfoList::new(); + + let config1 = DummyConfigInfo { + id: "1".to_owned(), + content: "a".to_owned(), + }; + configs.insert_or_update(&config1).unwrap(); + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].config.id, "1"); + assert_eq!(configs[0].config.content, "a"); + + // Test case: succeed to update an existing entry + let config2 = DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }; + configs.insert_or_update(&config2 /* */).unwrap(); + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].config.id, "1"); + assert_eq!(configs[0].config.content, "b"); + + // Add a second entry + let config3 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.insert_or_update(&config3).unwrap(); + assert_eq!(configs.len(), 2); + assert_eq!(configs[0].config.id, "1"); + assert_eq!(configs[0].config.content, "b"); + assert_eq!(configs[1].config.id, "2"); + assert_eq!(configs[1].config.content, "c"); + + // Lookup the first entry + let config4_id = configs + .get_index_by_id(&DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }) + .unwrap(); + let config4 = &configs[config4_id].config; + assert_eq!(config4.id, config2.id); + assert_eq!(config4.content, config2.content); + + // Lookup the second entry + let config5_id = configs + .get_index_by_id(&DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }) + .unwrap(); + let config5 = &configs[config5_id].config; + assert_eq!(config5.id, config3.id); + assert_eq!(config5.content, config3.content); + + // Test case: can't insert an entry with conflicting content + let config6 = DummyConfigInfo { + id: "3".to_owned(), + content: "c".to_owned(), + }; + configs.insert_or_update(&config6).unwrap_err(); + assert_eq!(configs.len(), 2); + assert_eq!(configs[0].config.id, "1"); + assert_eq!(configs[0].config.content, "b"); + assert_eq!(configs[1].config.id, "2"); + assert_eq!(configs[1].config.content, "c"); + } + + #[test] + fn test_remove_device_info() { + let mut configs = DummyDeviceInfoList::new(); + + let config1 = DummyConfigInfo { + id: "1".to_owned(), + content: "a".to_owned(), + }; + configs.insert_or_update(&config1).unwrap(); + let config2 = DummyConfigInfo { + id: "1".to_owned(), + content: "b".to_owned(), + }; + configs.insert_or_update(&config2).unwrap(); + let config3 = DummyConfigInfo { + id: "2".to_owned(), + content: "c".to_owned(), + }; + configs.insert_or_update(&config3).unwrap(); + assert_eq!(configs.len(), 2); + assert_eq!(configs[0].config.id, "1"); + assert_eq!(configs[0].config.content, "b"); + assert_eq!(configs[1].config.id, "2"); + assert_eq!(configs[1].config.content, "c"); + + let config4 = configs.remove(0).unwrap().config; + assert_eq!(config4.id, config2.id); + assert_eq!(config4.content, config2.content); + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].config.id, "2"); + assert_eq!(configs[0].config.content, "c"); + + let config5 = configs.remove(0).unwrap().config; + assert_eq!(config5.id, config3.id); + assert_eq!(config5.content, config3.content); + assert_eq!(configs.len(), 0); + } +} diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index a3295e8f8f77..8bff8b0436d0 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -8,6 +8,8 @@ /// Address space manager for virtual machines. pub mod address_space_manager; +/// Structs to maintain configuration information. +pub mod config_manager; /// Device manager for virtual machines. pub mod device_manager; /// Resource manager for virtual machines. From c1c1e5152a57d60999d48ea6b2188304360d3bfa Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 5 May 2022 16:24:45 +0800 Subject: [PATCH 0059/1953] dragonball: add kernel config. It is used for holding guest kernel configuration information. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 1 + src/dragonball/src/vm/kernel_config.rs | 67 ++++++++++++++++++++++++++ src/dragonball/src/vm/mod.rs | 3 ++ 3 files changed, 71 insertions(+) create mode 100644 src/dragonball/src/vm/kernel_config.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 432a2e80814d..5171c3584100 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -21,6 +21,7 @@ dbs-utils = "0.1.0" kvm-bindings = "0.5.0" kvm-ioctls = "0.11.0" libc = "0.2.39" +linux-loader = "0.4.0" log = "0.4.14" nix = "0.23.1" serde = "1.0.27" diff --git a/src/dragonball/src/vm/kernel_config.rs b/src/dragonball/src/vm/kernel_config.rs new file mode 100644 index 000000000000..f266b48c4d52 --- /dev/null +++ b/src/dragonball/src/vm/kernel_config.rs @@ -0,0 +1,67 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs::File; + +/// Structure to hold guest kernel configuration information. +pub struct KernelConfigInfo { + /// The descriptor to the kernel file. + kernel_file: File, + /// The descriptor to the initrd file, if there is one + initrd_file: Option, + /// The commandline for guest kernel. + cmdline: linux_loader::cmdline::Cmdline, +} + +impl KernelConfigInfo { + /// Create a KernelConfigInfo instance. + pub fn new( + kernel_file: File, + initrd_file: Option, + cmdline: linux_loader::cmdline::Cmdline, + ) -> Self { + KernelConfigInfo { + kernel_file, + initrd_file, + cmdline, + } + } + + /// Get a mutable reference to the kernel file. + pub fn kernel_file_mut(&mut self) -> &mut File { + &mut self.kernel_file + } + + /// Get a mutable reference to the initrd file. + pub fn initrd_file_mut(&mut self) -> Option<&mut File> { + self.initrd_file.as_mut() + } + + /// Get a shared reference to the guest kernel boot parameter object. + pub fn kernel_cmdline(&self) -> &linux_loader::cmdline::Cmdline { + &self.cmdline + } + + /// Get a mutable reference to the guest kernel boot parameter object. + pub fn kernel_cmdline_mut(&mut self) -> &mut linux_loader::cmdline::Cmdline { + &mut self.cmdline + } +} + +#[cfg(test)] +mod tests { + use super::*; + use vmm_sys_util::tempfile::TempFile; + + #[test] + fn test_kernel_config_info() { + let kernel = TempFile::new().unwrap(); + let initrd = TempFile::new().unwrap(); + let mut cmdline = linux_loader::cmdline::Cmdline::new(1024); + cmdline.insert_str("ro").unwrap(); + let mut info = KernelConfigInfo::new(kernel.into_file(), Some(initrd.into_file()), cmdline); + + assert_eq!(info.cmdline.as_str(), "ro"); + assert!(info.initrd_file_mut().is_some()); + } +} diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index c461b6a28288..c1510308da22 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -3,6 +3,9 @@ use serde_derive::{Deserialize, Serialize}; +mod kernel_config; +pub use self::kernel_config::KernelConfigInfo; + /// Configuration information for user defined NUMA nodes. #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] pub struct NumaRegionInfo { From 52d42af636b8801d40896f973a445fa59af4df4c Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 5 May 2022 16:28:04 +0800 Subject: [PATCH 0060/1953] dragonball: add device manager. Device manager to manage IO devices for a virtual machine. And added DeviceManagerTx to provide operation transaction for device management, added DeviceManagerContext to operation context for device management. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 4 + src/dragonball/src/device_manager/mod.rs | 404 +++++++++++++++++++++++ src/dragonball/src/error.rs | 26 ++ src/dragonball/src/lib.rs | 2 + 4 files changed, 436 insertions(+) create mode 100644 src/dragonball/src/error.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 5171c3584100..f0fc11dda4af 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -16,8 +16,10 @@ dbs-address-space = "0.1.0" dbs-allocator = "0.1.0" dbs-boot = "0.2.0" dbs-device = "0.1.0" +dbs-interrupt = { version = "0.1.0", features = ["kvm-irq"] } dbs-legacy-devices = "0.1.0" dbs-utils = "0.1.0" +dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] } kvm-bindings = "0.5.0" kvm-ioctls = "0.11.0" libc = "0.2.39" @@ -31,6 +33,7 @@ slog = "2.5.2" slog-scope = "4.4.0" thiserror = "1" vmm-sys-util = "0.9.0" +virtio-queue = { version = "0.1.0", optional = true } vm-memory = { version = "0.7.0", features = ["backend-mmap"] } [dev-dependencies] @@ -42,3 +45,4 @@ atomic-guest-memory = [] [patch.'crates-io'] dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } +dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 2dad8b8c1044..5ffe5ed3ab5f 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -3,6 +3,24 @@ //! Device manager to manage IO devices for a virtual machine. +use std::io; +use std::sync::{Arc, Mutex, MutexGuard}; + +use arc_swap::ArcSwap; +use dbs_address_space::AddressSpace; +use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext}; +use dbs_device::resources::Resource; +use dbs_device::DeviceIo; +use dbs_interrupt::KvmIrqManager; +use dbs_legacy_devices::ConsoleHandler; +use dbs_utils::epoll_manager::EpollManager; +use kvm_ioctls::VmFd; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::error::StartMicrovmError; +use crate::resource_manager::ResourceManager; +use crate::vm::KernelConfigInfo; + /// Virtual machine console device manager. pub mod console_manager; /// Console Manager for virtual machines console device. @@ -14,10 +32,396 @@ pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; /// Errors related to device manager operations. #[derive(Debug, thiserror::Error)] pub enum DeviceMgrError { + /// Invalid operation. + #[error("invalid device manager operation")] + InvalidOperation, + /// Failed to get device resource. + #[error("failed to get device assigned resources")] + GetDeviceResource, + /// Appending to kernel command line failed. + #[error("failed to add kernel command line parameter for device: {0}")] + Cmdline(#[source] linux_loader::cmdline::Error), /// Failed to manage console devices. #[error(transparent)] ConsoleManager(console_manager::ConsoleManagerError), + /// Failed to create the device. + #[error("failed to create virtual device: {0}")] + CreateDevice(#[source] io::Error), + /// Failed to perform an operation on the bus. + #[error(transparent)] + IoManager(IoManagerError), + /// Failure from legacy device manager. + #[error(transparent)] + LegacyManager(legacy::Error), } /// Specialized version of `std::result::Result` for device manager operations. pub type Result = ::std::result::Result; + +/// Type of the dragonball virtio mmio devices. +#[cfg(feature = "dbs-virtio-devices")] +pub type DbsMmioV2Device = + MmioV2Device; + +/// Struct to support transactional operations for device management. +pub struct DeviceManagerTx { + io_manager: IoManager, + _io_lock: Arc>, + _guard: MutexGuard<'static, ()>, +} + +impl DeviceManagerTx { + fn new(mgr_ctx: &DeviceManagerContext) -> Self { + // Do not expect poisoned lock. + let guard = mgr_ctx.io_lock.lock().unwrap(); + + // It's really a heavy burden to carry on a lifetime parameter for MutexGuard. + // So we play a tricky here that we hold a reference to the Arc> and transmute + // the MutexGuard<'a, ()> to MutexGuard<'static, ()>. + // It's safe because we hold a reference to the Mutex lock. + let guard = + unsafe { std::mem::transmute::, MutexGuard<'static, ()>>(guard) }; + + DeviceManagerTx { + io_manager: mgr_ctx.io_manager.load().as_ref().clone(), + _io_lock: mgr_ctx.io_lock.clone(), + _guard: guard, + } + } +} + +/// Operation context for device management. +#[derive(Clone)] +pub struct DeviceManagerContext { + io_manager: Arc>, + io_lock: Arc>, +} + +impl DeviceManagerContext { + /// Create a DeviceManagerContext object. + pub fn new(io_manager: Arc>, io_lock: Arc>) -> Self { + DeviceManagerContext { + io_manager, + io_lock, + } + } +} + +impl IoManagerContext for DeviceManagerContext { + type Context = DeviceManagerTx; + + fn begin_tx(&self) -> Self::Context { + DeviceManagerTx::new(self) + } + + fn commit_tx(&self, context: Self::Context) { + self.io_manager.store(Arc::new(context.io_manager)); + } + + fn cancel_tx(&self, context: Self::Context) { + drop(context); + } + + fn register_device_io( + &self, + ctx: &mut Self::Context, + device: Arc, + resources: &[Resource], + ) -> std::result::Result<(), dbs_device::device_manager::Error> { + ctx.io_manager.register_device_io(device, resources) + } + + fn unregister_device_io( + &self, + ctx: &mut Self::Context, + resources: &[Resource], + ) -> std::result::Result<(), dbs_device::device_manager::Error> { + ctx.io_manager.unregister_device_io(resources) + } +} + +/// Context for device addition/removal operations. +pub struct DeviceOpContext { + epoll_mgr: Option, + io_context: DeviceManagerContext, + irq_manager: Arc, + res_manager: Arc, + vm_fd: Arc, + vm_as: Option, + address_space: Option, + logger: slog::Logger, + is_hotplug: bool, + + #[cfg(feature = "dbs-virtio-devices")] + virtio_devices: Vec>, +} + +impl DeviceOpContext { + pub(crate) fn new( + epoll_mgr: Option, + device_mgr: &DeviceManager, + vm_as: Option, + address_space: Option, + is_hotplug: bool, + ) -> Self { + let irq_manager = device_mgr.irq_manager.clone(); + let res_manager = device_mgr.res_manager.clone(); + + let vm_fd = device_mgr.vm_fd.clone(); + let io_context = DeviceManagerContext { + io_manager: device_mgr.io_manager.clone(), + io_lock: device_mgr.io_lock.clone(), + }; + let logger = device_mgr.logger.new(slog::o!()); + + DeviceOpContext { + epoll_mgr, + io_context, + irq_manager, + res_manager, + vm_fd, + vm_as, + address_space, + logger, + is_hotplug, + #[cfg(feature = "dbs-virtio-devices")] + virtio_devices: Vec::new(), + } + } + + pub(crate) fn get_vm_as(&self) -> Result { + match self.vm_as.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn logger(&self) -> &slog::Logger { + &self.logger + } + + fn generate_kernel_boot_args(&mut self, kernel_config: &mut KernelConfigInfo) -> Result<()> { + if !self.is_hotplug { + return Err(DeviceMgrError::InvalidOperation); + } + + #[cfg(feature = "dbs-virtio-devices")] + let cmdline = kernel_config.kernel_cmdline_mut(); + + #[cfg(feature = "dbs-virtio-devices")] + for device in self.virtio_devices.iter() { + let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_device_info(device)?; + + // as per doc, [virtio_mmio.]device=@: needs to be appended + // to kernel commandline for virtio mmio devices to get recognized + // the size parameter has to be transformed to KiB, so dividing hexadecimal value in + // bytes to 1024; further, the '{}' formatting rust construct will automatically + // transform it to decimal + cmdline + .insert( + "virtio_mmio.device", + &format!("{}K@0x{:08x}:{}", mmio_size / 1024, mmio_base, irq), + ) + .map_err(DeviceMgrError::Cmdline)?; + } + + Ok(()) + } +} + +/// Device manager for virtual machines, which manages all device for a virtual machine. +pub struct DeviceManager { + io_manager: Arc>, + io_lock: Arc>, + irq_manager: Arc, + res_manager: Arc, + vm_fd: Arc, + pub(crate) logger: slog::Logger, + + pub(crate) con_manager: ConsoleManager, + pub(crate) legacy_manager: Option, +} + +impl DeviceManager { + /// Create a new device manager instance. + pub fn new( + vm_fd: Arc, + res_manager: Arc, + epoll_manager: EpollManager, + logger: &slog::Logger, + ) -> Self { + DeviceManager { + io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))), + io_lock: Arc::new(Mutex::new(())), + irq_manager: Arc::new(KvmIrqManager::new(vm_fd.clone())), + res_manager, + vm_fd, + logger: logger.new(slog::o!()), + con_manager: ConsoleManager::new(epoll_manager, logger), + legacy_manager: None, + } + } + + /// Create the underline interrupt manager for the device manager. + pub fn create_interrupt_manager(&mut self) -> Result<()> { + self.irq_manager + .initialize() + .map_err(DeviceMgrError::CreateDevice) + } + + /// Get the underlying logger. + pub fn logger(&self) -> &slog::Logger { + &self.logger + } + + /// Create legacy devices associted virtual machine + pub fn create_legacy_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), StartMicrovmError> { + #[cfg(target_arch = "x86_64")] + { + let mut tx = ctx.io_context.begin_tx(); + let legacy_manager = + LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + + match legacy_manager { + Ok(v) => { + self.legacy_manager = Some(v); + ctx.io_context.commit_tx(tx); + } + Err(e) => { + ctx.io_context.cancel_tx(tx); + return Err(StartMicrovmError::LegacyDevice(e)); + } + } + } + + Ok(()) + } + + /// Init legacy devices with logger stream in associted virtual machine + pub fn init_legacy_devices( + &mut self, + dmesg_fifo: Option>, + com1_sock_path: Option, + _ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), StartMicrovmError> { + // Connect serial ports to the console and dmesg_fifo. + self.set_guest_kernel_log_stream(dmesg_fifo) + .map_err(|_| StartMicrovmError::EventFd)?; + + slog::info!(self.logger, "init console path: {:?}", com1_sock_path); + if let Some(path) = com1_sock_path { + if let Some(legacy_manager) = self.legacy_manager.as_ref() { + let com1 = legacy_manager.get_com1_serial(); + self.con_manager + .create_socket_console(com1, path) + .map_err(StartMicrovmError::DeviceManager)?; + } + } else if let Some(legacy_manager) = self.legacy_manager.as_ref() { + let com1 = legacy_manager.get_com1_serial(); + self.con_manager + .create_stdio_console(com1) + .map_err(StartMicrovmError::DeviceManager)?; + } + + Ok(()) + } + + /// Set the stream for guest kernel log. + /// + /// Note: com2 is used for guest kernel logging. + /// TODO: check whether it works with aarch64. + pub fn set_guest_kernel_log_stream( + &self, + stream: Option>, + ) -> std::result::Result<(), io::Error> { + if let Some(legacy) = self.legacy_manager.as_ref() { + legacy + .get_com2_serial() + .lock() + .unwrap() + .set_output_stream(stream); + } + Ok(()) + } + + /// Restore legacy devices + pub fn restore_legacy_devices( + &mut self, + dmesg_fifo: Option>, + com1_sock_path: Option, + ) -> std::result::Result<(), StartMicrovmError> { + self.set_guest_kernel_log_stream(dmesg_fifo) + .map_err(|_| StartMicrovmError::EventFd)?; + slog::info!(self.logger, "restore console path: {:?}", com1_sock_path); + // TODO: restore console + Ok(()) + } + + /// Reset the console into canonical mode. + pub fn reset_console(&self) -> Result<()> { + self.con_manager.reset_console() + } + + /// Create all registered devices when booting the associated virtual machine. + pub fn create_devices( + &mut self, + vm_as: GuestAddressSpaceImpl, + epoll_mgr: EpollManager, + kernel_config: &mut KernelConfigInfo, + com1_sock_path: Option, + dmesg_fifo: Option>, + address_space: Option<&AddressSpace>, + ) -> std::result::Result<(), StartMicrovmError> { + let mut ctx = DeviceOpContext::new( + Some(epoll_mgr), + self, + Some(vm_as), + address_space.cloned(), + false, + ); + + self.create_legacy_devices(&mut ctx)?; + self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; + + ctx.generate_kernel_boot_args(kernel_config) + .map_err(StartMicrovmError::DeviceManager)?; + + Ok(()) + } + + #[cfg(target_arch = "x86_64")] + /// Get the underlying eventfd for vm exit notification. + pub fn get_reset_eventfd(&self) -> Result { + if let Some(legacy) = self.legacy_manager.as_ref() { + legacy + .get_reset_eventfd() + .map_err(DeviceMgrError::LegacyManager) + } else { + Err(DeviceMgrError::LegacyManager(legacy::Error::EventFd( + io::Error::from_raw_os_error(libc::ENOENT), + ))) + } + } +} + +#[cfg(feature = "dbs-virtio-devices")] +impl DeviceManager { + fn get_virtio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { + let resources = device.get_assigned_resources(); + let irq = resources + .get_legacy_irq() + .ok_or(DeviceMgrError::GetDeviceResource)?; + let mmio_address_range = device.get_trapped_io_resources().get_mmio_address_ranges(); + + // Assume the first MMIO region is virtio configuration region. + // Virtio-fs needs to pay attention to this assumption. + if let Some(range) = mmio_address_range.into_iter().next() { + Ok((range.0, range.1, irq)) + } else { + Err(DeviceMgrError::GetDeviceResource) + } + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs new file mode 100644 index 000000000000..af54810d2416 --- /dev/null +++ b/src/dragonball/src/error.rs @@ -0,0 +1,26 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file + +//! Error codes for the virtual machine monitor subsystem. + +/// Errors associated with starting the instance. +#[derive(Debug, thiserror::Error)] +pub enum StartMicrovmError { + /// The device manager was not configured. + #[error("the device manager failed to manage devices: {0}")] + DeviceManager(#[source] crate::device_manager::DeviceMgrError), + + /// Cannot add devices to the Legacy I/O Bus. + #[error("failure in managing legacy device: {0}")] + LegacyDevice(#[source] crate::device_manager::LegacyDeviceError), + + /// Cannot read from an Event file descriptor. + #[error("failure while reading from EventFd file descriptor")] + EventFd, +} diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index 8bff8b0436d0..41a7178a1398 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -12,6 +12,8 @@ pub mod address_space_manager; pub mod config_manager; /// Device manager for virtual machines. pub mod device_manager; +/// Errors related to Virtual machine manager. +pub mod error; /// Resource manager for virtual machines. pub mod resource_manager; /// Virtual machine manager for virtual machines. From 8619f2b3d6eae2ff3e620e2c9f9023d11144c888 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 5 May 2022 17:36:04 +0800 Subject: [PATCH 0061/1953] dragonball: add virtio vsock device manager. Added VsockDeviceMgr struct to manage all vsock devices. Fixes: #4257 Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 4 + src/dragonball/src/device_manager/mod.rs | 175 +++++++++++ .../src/device_manager/vsock_dev_mgr.rs | 285 ++++++++++++++++++ src/dragonball/src/error.rs | 25 +- 4 files changed, 484 insertions(+), 5 deletions(-) create mode 100644 src/dragonball/src/device_manager/vsock_dev_mgr.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index f0fc11dda4af..e7b1da0c068c 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -42,7 +42,11 @@ slog-async = "2.7.0" [features] atomic-guest-memory = [] +virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] [patch.'crates-io'] +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 5ffe5ed3ab5f..5691690ea8d0 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -16,6 +16,19 @@ use dbs_legacy_devices::ConsoleHandler; use dbs_utils::epoll_manager::EpollManager; use kvm_ioctls::VmFd; +#[cfg(feature = "dbs-virtio-devices")] +use dbs_device::resources::ResourceConstraint; +#[cfg(feature = "dbs-virtio-devices")] +use dbs_virtio_devices as virtio; +#[cfg(feature = "dbs-virtio-devices")] +use dbs_virtio_devices::{ + mmio::{ + MmioV2Device, DRAGONBALL_FEATURE_INTR_USED, DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY, + DRAGONBALL_MMIO_DOORBELL_SIZE, MMIO_DEFAULT_CFG_SIZE, + }, + VirtioDevice, +}; + use crate::address_space_manager::GuestAddressSpaceImpl; use crate::error::StartMicrovmError; use crate::resource_manager::ResourceManager; @@ -29,6 +42,18 @@ pub use self::console_manager::ConsoleManager; mod legacy; pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; +#[cfg(feature = "virtio-vsock")] +/// Device manager for user-space vsock devices. +pub mod vsock_dev_mgr; +#[cfg(feature = "virtio-vsock")] +use self::vsock_dev_mgr::VsockDeviceMgr; + +macro_rules! info( + ($l:expr, $($args:tt)+) => { + slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) + }; +); + /// Errors related to device manager operations. #[derive(Debug, thiserror::Error)] pub enum DeviceMgrError { @@ -53,11 +78,22 @@ pub enum DeviceMgrError { /// Failure from legacy device manager. #[error(transparent)] LegacyManager(legacy::Error), + + #[cfg(feature = "dbs-virtio-devices")] + /// Error from Virtio subsystem. + #[error(transparent)] + Virtio(virtio::Error), } /// Specialized version of `std::result::Result` for device manager operations. pub type Result = ::std::result::Result; +/// Type of the dragonball virtio devices. +#[cfg(feature = "dbs-virtio-devices")] +pub type DbsVirtioDevice = Box< + dyn VirtioDevice, +>; + /// Type of the dragonball virtio mmio devices. #[cfg(feature = "dbs-virtio-devices")] pub type DbsMmioV2Device = @@ -240,6 +276,8 @@ pub struct DeviceManager { pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, + #[cfg(feature = "virtio-vsock")] + pub(crate) vsock_manager: VsockDeviceMgr, } impl DeviceManager { @@ -259,6 +297,8 @@ impl DeviceManager { logger: logger.new(slog::o!()), con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, + #[cfg(feature = "virtio-vsock")] + vsock_manager: VsockDeviceMgr::default(), } } @@ -386,6 +426,9 @@ impl DeviceManager { self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; + #[cfg(feature = "virtio-vsock")] + self.vsock_manager.attach_devices(&mut ctx)?; + ctx.generate_kernel_boot_args(kernel_config) .map_err(StartMicrovmError::DeviceManager)?; @@ -424,4 +467,136 @@ impl DeviceManager { Err(DeviceMgrError::GetDeviceResource) } } + + /// Create an Virtio MMIO transport layer device for the virtio backend device. + pub fn create_mmio_virtio_device( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + let features = DRAGONBALL_FEATURE_INTR_USED | DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY; + DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(features), + use_shared_irq, + use_generic_irq, + ) + } + + /// Create an Virtio MMIO transport layer device for the virtio backend device with specified + /// features. + pub fn create_mmio_virtio_device_with_features( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + features: Option, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + // Every emulated Virtio MMIO device needs a 4K configuration space, + // and another 4K space for per queue notification. + const MMIO_ADDRESS_DEFAULT: ResourceConstraint = ResourceConstraint::MmioAddress { + range: None, + align: 0, + size: MMIO_DEFAULT_CFG_SIZE + DRAGONBALL_MMIO_DOORBELL_SIZE, + }; + let mut requests = vec![MMIO_ADDRESS_DEFAULT]; + device.get_resource_requirements(&mut requests, use_generic_irq); + let resources = ctx + .res_manager + .allocate_device_resources(&requests, use_shared_irq) + .map_err(|_| DeviceMgrError::GetDeviceResource)?; + + let virtio_dev = match MmioV2Device::new( + ctx.vm_fd.clone(), + ctx.get_vm_as()?, + ctx.irq_manager.clone(), + device, + resources, + features, + ) { + Ok(d) => d, + Err(e) => return Err(DeviceMgrError::Virtio(e)), + }; + + Self::register_mmio_virtio_device(Arc::new(virtio_dev), ctx) + } + + /// Teardown the Virtio MMIO transport layer device associated with the virtio backend device. + pub fn destroy_mmio_virtio_device( + device: Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), DeviceMgrError> { + Self::destroy_mmio_device(device.clone(), ctx)?; + + let mmio_dev = device + .as_any() + .downcast_ref::() + .ok_or(DeviceMgrError::InvalidOperation)?; + + mmio_dev.remove(); + + Ok(()) + } + + fn destroy_mmio_device( + device: Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), DeviceMgrError> { + // unregister IoManager + Self::deregister_mmio_virtio_device(&device, ctx)?; + + // unregister Resource manager + let resources = device.get_assigned_resources(); + ctx.res_manager.free_device_resources(&resources); + + Ok(()) + } + + /// Create an Virtio MMIO transport layer device for the virtio backend device. + pub fn register_mmio_virtio_device( + device: Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result, DeviceMgrError> { + let (mmio_base, mmio_size, irq) = Self::get_virtio_device_info(&device)?; + info!( + ctx.logger(), + "create virtio mmio device 0x{:x}@0x{:x}, irq: 0x{:x}", mmio_size, mmio_base, irq + ); + let resources = device.get_trapped_io_resources(); + + let mut tx = ctx.io_context.begin_tx(); + if let Err(e) = ctx + .io_context + .register_device_io(&mut tx, device.clone(), &resources) + { + ctx.io_context.cancel_tx(tx); + Err(DeviceMgrError::IoManager(e)) + } else { + ctx.virtio_devices.push(device.clone()); + ctx.io_context.commit_tx(tx); + Ok(device) + } + } + + /// Deregister a Virtio MMIO device from IoManager + pub fn deregister_mmio_virtio_device( + device: &Arc, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), DeviceMgrError> { + let resources = device.get_trapped_io_resources(); + info!( + ctx.logger(), + "unregister mmio virtio device: {:?}", resources + ); + let mut tx = ctx.io_context.begin_tx(); + if let Err(e) = ctx.io_context.unregister_device_io(&mut tx, &resources) { + ctx.io_context.cancel_tx(tx); + Err(DeviceMgrError::IoManager(e)) + } else { + ctx.io_context.commit_tx(tx); + Ok(()) + } + } } diff --git a/src/dragonball/src/device_manager/vsock_dev_mgr.rs b/src/dragonball/src/device_manager/vsock_dev_mgr.rs new file mode 100644 index 000000000000..cec58d7de105 --- /dev/null +++ b/src/dragonball/src/device_manager/vsock_dev_mgr.rs @@ -0,0 +1,285 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::sync::Arc; + +use dbs_virtio_devices as virtio; +use dbs_virtio_devices::mmio::DRAGONBALL_FEATURE_INTR_USED; +use dbs_virtio_devices::vsock::backend::{ + VsockInnerBackend, VsockInnerConnector, VsockTcpBackend, VsockUnixStreamBackend, +}; +use dbs_virtio_devices::vsock::Vsock; +use dbs_virtio_devices::Error as VirtioError; +use serde_derive::{Deserialize, Serialize}; + +use super::StartMicrovmError; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::{DeviceManager, DeviceOpContext}; + +pub use dbs_virtio_devices::vsock::QUEUE_SIZES; + +const SUBSYSTEM: &str = "vsock_dev_mgr"; +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; + +/// Errors associated with `VsockDeviceConfigInfo`. +#[derive(Debug, thiserror::Error)] +pub enum VsockDeviceError { + /// The virtual machine instance ID is invalid. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + + /// The Context Identifier is already in use. + #[error("the device ID {0} already exists")] + DeviceIDAlreadyExist(String), + + /// The Context Identifier is invalid. + #[error("the guest CID {0} is invalid")] + GuestCIDInvalid(u32), + + /// The Context Identifier is already in use. + #[error("the guest CID {0} is already in use")] + GuestCIDAlreadyInUse(u32), + + /// The Unix Domain Socket path is already in use. + #[error("the Unix Domain Socket path {0} is already in use")] + UDSPathAlreadyInUse(String), + + /// The net address is already in use. + #[error("the net address {0} is already in use")] + NetAddrAlreadyInUse(String), + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The VsockId Already Exists + #[error("vsock id {0} already exists")] + VsockIdAlreadyExists(String), + + /// Inner backend create error + #[error("vsock inner backend create error: {0}")] + CreateInnerBackend(#[source] std::io::Error), +} + +/// Configuration information for a vsock device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct VsockDeviceConfigInfo { + /// ID of the vsock device. + pub id: String, + /// A 32-bit Context Identifier (CID) used to identify the guest. + pub guest_cid: u32, + /// unix domain socket path. + pub uds_path: Option, + /// tcp socket address. + pub tcp_addr: Option, + /// Virtio queue size. + pub queue_size: Vec, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl VsockDeviceConfigInfo { + /// Get number and size of queues supported. + pub fn queue_sizes(&self) -> Vec { + self.queue_size.clone() + } +} + +impl ConfigItem for VsockDeviceConfigInfo { + type Err = VsockDeviceError; + + fn id(&self) -> &str { + &self.id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), VsockDeviceError> { + if self.id == other.id { + return Err(VsockDeviceError::DeviceIDAlreadyExist(self.id.clone())); + } + if self.guest_cid == other.guest_cid { + return Err(VsockDeviceError::GuestCIDAlreadyInUse(self.guest_cid)); + } + if let (Some(self_uds_path), Some(other_uds_path)) = + (self.uds_path.as_ref(), other.uds_path.as_ref()) + { + if self_uds_path == other_uds_path { + return Err(VsockDeviceError::UDSPathAlreadyInUse(self_uds_path.clone())); + } + } + if let (Some(self_net_addr), Some(other_net_addr)) = + (self.tcp_addr.as_ref(), other.tcp_addr.as_ref()) + { + if self_net_addr == other_net_addr { + return Err(VsockDeviceError::NetAddrAlreadyInUse(self_net_addr.clone())); + } + } + + Ok(()) + } +} + +/// Vsock Device Info +pub type VsockDeviceInfo = DeviceConfigInfo; + +/// Device manager to manage all vsock devices. +pub struct VsockDeviceMgr { + pub(crate) info_list: DeviceConfigInfos, + pub(crate) default_inner_backend: Option, + pub(crate) default_inner_connector: Option, + pub(crate) use_shared_irq: bool, +} + +impl VsockDeviceMgr { + /// Insert or update a vsock device into the manager. + pub fn insert_device( + &mut self, + ctx: DeviceOpContext, + config: VsockDeviceConfigInfo, + ) -> std::result::Result<(), VsockDeviceError> { + if ctx.is_hotplug { + slog::error!( + ctx.logger(), + "no support of virtio-vsock device hotplug"; + "subsystem" => SUBSYSTEM, + "id" => &config.id, + "uds_path" => &config.uds_path, + ); + + return Err(VsockDeviceError::UpdateNotAllowedPostBoot); + } + + // VMADDR_CID_ANY (-1U) means any address for binding; + // VMADDR_CID_HYPERVISOR (0) is reserved for services built into the hypervisor; + // VMADDR_CID_RESERVED (1) must not be used; + // VMADDR_CID_HOST (2) is the well-known address of the host. + if config.guest_cid <= 2 { + return Err(VsockDeviceError::GuestCIDInvalid(config.guest_cid)); + } + + slog::info!( + ctx.logger(), + "add virtio-vsock device configuration"; + "subsystem" => SUBSYSTEM, + "id" => &config.id, + "uds_path" => &config.uds_path, + ); + + self.lazy_make_default_connector()?; + + self.info_list.insert_or_update(&config)?; + + Ok(()) + } + + /// Attach all configured vsock device to the virtual machine instance. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), StartMicrovmError> { + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(StartMicrovmError::CreateVsockDevice( + virtio::Error::InvalidInput, + ))?; + + for info in self.info_list.iter_mut() { + slog::info!( + ctx.logger(), + "attach virtio-vsock device"; + "subsystem" => SUBSYSTEM, + "id" => &info.config.id, + "uds_path" => &info.config.uds_path, + ); + + let mut device = Box::new( + Vsock::new( + info.config.guest_cid as u64, + Arc::new(info.config.queue_sizes()), + epoll_mgr.clone(), + ) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?, + ); + if let Some(uds_path) = info.config.uds_path.as_ref() { + let unix_backend = VsockUnixStreamBackend::new(uds_path.clone()) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + device + .add_backend(Box::new(unix_backend), true) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + } + if let Some(tcp_addr) = info.config.tcp_addr.as_ref() { + let tcp_backend = VsockTcpBackend::new(tcp_addr.clone()) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + device + .add_backend(Box::new(tcp_backend), false) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + } + // add inner backend to the the first added vsock device + if let Some(inner_backend) = self.default_inner_backend.take() { + device + .add_backend(Box::new(inner_backend), false) + .map_err(VirtioError::VirtioVsockError) + .map_err(StartMicrovmError::CreateVsockDevice)?; + } + let device = DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(DRAGONBALL_FEATURE_INTR_USED), + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(StartMicrovmError::RegisterVsockDevice)?; + info.device = Some(device); + } + + Ok(()) + } + + // check the default connector is present, or build it. + fn lazy_make_default_connector(&mut self) -> std::result::Result<(), VsockDeviceError> { + if self.default_inner_connector.is_none() { + let inner_backend = + VsockInnerBackend::new().map_err(VsockDeviceError::CreateInnerBackend)?; + self.default_inner_connector = Some(inner_backend.get_connector()); + self.default_inner_backend = Some(inner_backend); + } + Ok(()) + } + + /// Get the default vsock inner connector. + pub fn get_default_connector( + &mut self, + ) -> std::result::Result { + self.lazy_make_default_connector()?; + + // safe to unwrap, because we created the inner connector before + Ok(self.default_inner_connector.clone().unwrap()) + } +} + +impl Default for VsockDeviceMgr { + /// Create a new Vsock device manager. + fn default() -> Self { + VsockDeviceMgr { + info_list: DeviceConfigInfos::new(), + default_inner_backend: None, + default_inner_connector: None, + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index af54810d2416..5a497abb6b02 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -9,18 +9,33 @@ //! Error codes for the virtual machine monitor subsystem. +#[cfg(feature = "dbs-virtio-devices")] +use dbs_virtio_devices::Error as VirtIoError; + +use crate::device_manager; + /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] pub enum StartMicrovmError { + /// Cannot read from an Event file descriptor. + #[error("failure while reading from EventFd file descriptor")] + EventFd, + /// The device manager was not configured. #[error("the device manager failed to manage devices: {0}")] - DeviceManager(#[source] crate::device_manager::DeviceMgrError), + DeviceManager(#[source] device_manager::DeviceMgrError), /// Cannot add devices to the Legacy I/O Bus. #[error("failure in managing legacy device: {0}")] - LegacyDevice(#[source] crate::device_manager::LegacyDeviceError), + LegacyDevice(#[source] device_manager::LegacyDeviceError), - /// Cannot read from an Event file descriptor. - #[error("failure while reading from EventFd file descriptor")] - EventFd, + #[cfg(feature = "virtio-vsock")] + /// Failed to create the vsock device. + #[error("cannot create virtio-vsock device: {0}")] + CreateVsockDevice(#[source] VirtIoError), + + #[cfg(feature = "virtio-vsock")] + /// Cannot initialize a MMIO Vsock Device or add a device to the MMIO Bus. + #[error("failure while registering virtio-vsock device: {0}")] + RegisterVsockDevice(#[source] device_manager::DeviceMgrError), } From a1df6d096924726bc217de634ec2a09705e44523 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Thu, 12 May 2022 20:27:41 +0800 Subject: [PATCH 0062/1953] Doc: Update Dragonball Readme and add document for device Update Dragonball Readme to fix style problem and add github issue for TODOs. Add document for devices in dragonball. This is the document for the current dragonball device status and we'll keep updating it when we introduce more devices in later pull requets. Fixes: #4257 Signed-off-by: Chao Wu --- src/dragonball/README.md | 41 +++++++++++++++++++++-------------- src/dragonball/docs/device.md | 17 +++++++++++++++ 2 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 src/dragonball/docs/device.md diff --git a/src/dragonball/README.md b/src/dragonball/README.md index dcc066424fcd..0e3bcb45a06c 100644 --- a/src/dragonball/README.md +++ b/src/dragonball/README.md @@ -1,28 +1,37 @@ -# Dragonball - -## Introduction -Dragonball Sandbox is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual Machine(KVM), +# Introduction +`Dragonball Sandbox` is a light-weight virtual machine manager (VMM) based on Linux Kernel-based Virtual Machine (KVM), which is optimized for container workloads with: - container image management and acceleration service - flexible and high-performance virtual device drivers -- low cpu & memory overhead -- industry-leading startup time -- high concurrent startup throughput +- low CPU and memory overhead +- minimal startup time +- optimized concurrent startup speed -Dragonball Sandbox aims to provide a turnkey solution for the Kata Containers community. It is integrated into Kata 3.0 +`Dragonball Sandbox` aims to provide a simple solution for the Kata Containers community. It is integrated into Kata 3.0 runtime as a built-in VMM and gives users an out-of-the-box Kata Containers experience without complex environment setup and configuration process. -## Getting Start -[TODO] +# Getting Started +[TODO](https://github.com/kata-containers/kata-containers/issues/4302) + +# Documentation + +Device: [Device Document](docs/device.md) -## Supported Architectures +You could see the [official documentation](docs/) page for more details. + +# Supported Architectures - x86-64 -- AArch64 +- aarch64 -## Supported Kernel -[TODO] +# Supported Kernel +[TODO](https://github.com/kata-containers/kata-containers/issues/4303) + +# Acknowledgement +Part of the code is based on the [Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor) project, [`crosvm`](https://github.com/google/crosvm) project and [Firecracker](https://github.com/firecracker-microvm/firecracker) project. They are all rust written virtual machine managers with advantages on safety and security. + +`Dragonball sandbox` is designed to be a VMM that is customized for Kata Containers and we will focus on optimizing container workloads for Kata ecosystem. The focus on the Kata community is what differentiates us from other rust written virtual machines. -## License +# License -Dragonball is licensed under [Apache License](http://www.apache.org/licenses/LICENSE-2.0), Version 2.0. \ No newline at end of file +`Dragonball` is licensed under [Apache License](http://www.apache.org/licenses/LICENSE-2.0), Version 2.0. \ No newline at end of file diff --git a/src/dragonball/docs/device.md b/src/dragonball/docs/device.md new file mode 100644 index 000000000000..8f3fdbe6ed6a --- /dev/null +++ b/src/dragonball/docs/device.md @@ -0,0 +1,17 @@ +# Device + +## Device Manager + +Currently we have following device manager: +| Name | Description | +| --- | --- | +| [address space manager](../src/address_space_manager.rs) | abstracts virtual machine's physical management and provide mapping for guest virtual memory and MMIO ranges of emulated virtual devices, pass-through devices and vCPU | +| [config manager](../src/config_manager.rs) | provides abstractions for configuration information | +| [console manager](../src/device_manager/console_manager.rs) | provides management for all console devices | +| [resource manager](../src/resource_manager.rs) |provides resource management for `legacy_irq_pool`, `msi_irq_pool`, `pio_pool`, `mmio_pool`, `mem_pool`, `kvm_mem_slot_pool` with builder `ResourceManagerBuilder` | +| [VSOCK device manager](../src/device_manager/vsock_dev_mgr.rs) | provides configuration info for `VIRTIO-VSOCK` and management for all VSOCK devices | + + +## Device supported +`VIRTIO-VSOCK` + From 71f24d827125eb3eadb51468325b51635c9baa5e Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 12 May 2022 20:58:48 +0800 Subject: [PATCH 0063/1953] dragonball: add Makefile. Currently supported: build, clippy, check, format, test, clean Fixes: #4257 Signed-off-by: wllenyj --- src/dragonball/Makefile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/dragonball/Makefile diff --git a/src/dragonball/Makefile b/src/dragonball/Makefile new file mode 100644 index 000000000000..8acd29de57f8 --- /dev/null +++ b/src/dragonball/Makefile @@ -0,0 +1,29 @@ +# Copyright (c) 2019-2022 Alibaba Cloud. All rights reserved. +# Copyright (c) 2019-2022 Ant Group. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +default: build + +build: + # FIXME: This line will be removed when we solve the vm-memory dependency problem in Dragonball Sandbox + cargo update -p vm-memory:0.8.0 --precise 0.7.0 + cargo build --all-features + +check: clippy format + +clippy: + @echo "INFO: cargo clippy..." + cargo clippy --all-targets --all-features \ + -- \ + -D warnings + +format: + @echo "INFO: cargo fmt..." + cargo fmt -- --check + +clean: + cargo clean + +test: + @echo "INFO: testing dragonball for development build" + cargo test --all-features -- --nocapture From 39ff85d610ec4e486e564de5da2fd851544ee087 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 12 May 2022 21:03:05 +0800 Subject: [PATCH 0064/1953] dragonball: green ci Revert this patch, after dragonball-sandbox is ready. And all subsequent implementations are submitted. Fixes: #4257 Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 11 ++++++----- src/dragonball/src/lib.rs | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index e7b1da0c068c..1368c362f0d6 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -45,8 +45,9 @@ atomic-guest-memory = [] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] [patch.'crates-io'] -dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } -dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } -dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } -dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } -dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "8e1181eca897b5c5e8e6ac45e3a5c995461865be" } +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index 41a7178a1398..cf528d067e4f 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -5,6 +5,8 @@ //! Machine(KVM) which is optimized for container workloads. #![warn(missing_docs)] +//TODO: Remove this, after the rest of dragonball has been committed. +#![allow(dead_code)] /// Address space manager for virtual machines. pub mod address_space_manager; From dfe6de7714d96fd0c79995167c35c2fd570b33a1 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Tue, 17 May 2022 17:20:24 +0800 Subject: [PATCH 0065/1953] dragonball: add dragonball into kata README add dragonball description into kata README to help introduce dragonball sandbox. Fixes: #4257 Signed-off-by: Chao Wu --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8fa30996f64..e66917251e05 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,8 @@ The table below lists the core parts of the project: |-|-|-| | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. | | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. | -| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.ht -ml) | +| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) | +| [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads | | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). | | [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) | | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. | From 93c10dfd8662c0e7eaff87132944655f823fffd5 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 30 May 2022 14:09:56 +0800 Subject: [PATCH 0066/1953] runtime-rs: add crosvm license in Dragonball add THIRD-PARTY file to add license for crosvm. Signed-off-by: Chao Wu --- src/dragonball/THIRD-PARTY | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/dragonball/THIRD-PARTY diff --git a/src/dragonball/THIRD-PARTY b/src/dragonball/THIRD-PARTY new file mode 100644 index 000000000000..c3069125a350 --- /dev/null +++ b/src/dragonball/THIRD-PARTY @@ -0,0 +1,27 @@ +// Copyright 2017 The Chromium OS Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file From d5ee3fc8560223ab8aa46664a7f43c3f3cfc05c8 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Sat, 11 Jun 2022 20:51:54 +0800 Subject: [PATCH 0067/1953] safe-path: fix clippy warning fix clippy warnings in safe-path lib to make clippy happy. Signed-off-by: Chao Wu --- src/libs/safe-path/src/pinned_path_buf.rs | 3 ++- src/libs/safe-path/src/scoped_dir_builder.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libs/safe-path/src/pinned_path_buf.rs b/src/libs/safe-path/src/pinned_path_buf.rs index 4310637df57e..d1816f450dcb 100644 --- a/src/libs/safe-path/src/pinned_path_buf.rs +++ b/src/libs/safe-path/src/pinned_path_buf.rs @@ -253,7 +253,7 @@ mod tests { fs::write(rootfs_path.join("endpoint"), "test").unwrap(); // Pin the target and validate the path/content. - let path = PinnedPathBuf::new(rootfs_path.to_path_buf(), "symlink_dir/endpoint").unwrap(); + let path = PinnedPathBuf::new(rootfs_path, "symlink_dir/endpoint").unwrap(); assert!(!path.is_dir()); let path_ref = path.deref(); let target = fs::read_link(path_ref).unwrap(); @@ -344,6 +344,7 @@ mod tests { PinnedPathBuf::new(rootfs_path, "does_not_exist").unwrap_err(); } + #[allow(clippy::zero_prefixed_literal)] #[test] fn test_new_pinned_path_buf_without_read_perm() { let rootfs_dir = tempfile::tempdir().expect("failed to create tmpdir"); diff --git a/src/libs/safe-path/src/scoped_dir_builder.rs b/src/libs/safe-path/src/scoped_dir_builder.rs index 39ceac1076f2..1a4ba189f2e7 100644 --- a/src/libs/safe-path/src/scoped_dir_builder.rs +++ b/src/libs/safe-path/src/scoped_dir_builder.rs @@ -87,7 +87,7 @@ impl ScopedDirBuilder { ) })?; - self.do_mkdir(&stripped_path) + self.do_mkdir(stripped_path) } /// Creates sub-directory with the options configured in this builder. @@ -134,7 +134,7 @@ impl ScopedDirBuilder { if !self.recursive && idx != levels { return Err(Error::new( ErrorKind::NotFound, - format!("parent directory does not exist"), + "parent directory does not exist".to_string(), )); } dir = dir.mkdir(comp, self.mode)?; @@ -146,6 +146,7 @@ impl ScopedDirBuilder { } } +#[allow(clippy::zero_prefixed_literal)] #[cfg(test)] mod tests { use super::*; From e80e0c4645a04be7b865d67ffc2ccf47645e8e86 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sat, 14 May 2022 22:34:16 +0800 Subject: [PATCH 0068/1953] dragonball: add io manager wrapper Wrapper over IoManager to support device hotplug. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/src/io_manager.rs | 60 ++++++++++++++++++++++++++++++++ src/dragonball/src/lib.rs | 4 +++ 2 files changed, 64 insertions(+) create mode 100644 src/dragonball/src/io_manager.rs diff --git a/src/dragonball/src/io_manager.rs b/src/dragonball/src/io_manager.rs new file mode 100644 index 000000000000..410703bc7abf --- /dev/null +++ b/src/dragonball/src/io_manager.rs @@ -0,0 +1,60 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; + +use arc_swap::{ArcSwap, Cache}; +use dbs_device::device_manager::Error; +use dbs_device::device_manager::IoManager; + +/// A specialized version of [`std::result::Result`] for IO manager related operations. +pub type Result = std::result::Result; + +/// Wrapper over IoManager to support device hotplug with [`ArcSwap`] and [`Cache`]. +#[derive(Clone)] +pub struct IoManagerCached(pub(crate) Cache>, Arc>); + +impl IoManagerCached { + /// Create a new instance of [`IoManagerCached`]. + pub fn new(io_manager: Arc>) -> Self { + IoManagerCached(Cache::new(io_manager)) + } + + #[cfg(target_arch = "x86_64")] + #[inline] + /// Read data from IO ports. + pub fn pio_read(&mut self, addr: u16, data: &mut [u8]) -> Result<()> { + self.0.load().pio_read(addr, data) + } + + #[cfg(target_arch = "x86_64")] + #[inline] + /// Write data to IO ports. + pub fn pio_write(&mut self, addr: u16, data: &[u8]) -> Result<()> { + self.0.load().pio_write(addr, data) + } + + #[inline] + /// Read data to MMIO address. + pub fn mmio_read(&mut self, addr: u64, data: &mut [u8]) -> Result<()> { + self.0.load().mmio_read(addr, data) + } + + #[inline] + /// Write data to MMIO address. + pub fn mmio_write(&mut self, addr: u64, data: &[u8]) -> Result<()> { + self.0.load().mmio_write(addr, data) + } + + #[inline] + /// Revalidate the inner cache + pub fn revalidate_cache(&mut self) { + let _ = self.0.load(); + } + + #[inline] + /// Get immutable reference to underlying [`IoManager`]. + pub fn load(&mut self) -> &IoManager { + self.0.load() + } +} diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index cf528d067e4f..2e65d8de6804 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -5,6 +5,7 @@ //! Machine(KVM) which is optimized for container workloads. #![warn(missing_docs)] + //TODO: Remove this, after the rest of dragonball has been committed. #![allow(dead_code)] @@ -20,3 +21,6 @@ pub mod error; pub mod resource_manager; /// Virtual machine manager for virtual machines. pub mod vm; + +mod io_manager; +pub use self::io_manager::IoManagerCached; From b6cb2c4ae38021fd6b76e9d5aaf9e8c0c7cc95b2 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sat, 14 May 2022 22:53:37 +0800 Subject: [PATCH 0069/1953] dragonball: add metrics system metrics system is added for collecting Dragonball metrics to analyze the system. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 1 + src/dragonball/src/lib.rs | 2 ++ src/dragonball/src/metric.rs | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 src/dragonball/src/metric.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 1368c362f0d6..e932da1ab2b8 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -22,6 +22,7 @@ dbs-utils = "0.1.0" dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] } kvm-bindings = "0.5.0" kvm-ioctls = "0.11.0" +lazy_static = "1.2" libc = "0.2.39" linux-loader = "0.4.0" log = "0.4.14" diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index 2e65d8de6804..e5f90cfc7afc 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -17,6 +17,8 @@ pub mod config_manager; pub mod device_manager; /// Errors related to Virtual machine manager. pub mod error; +/// Metrics system. +pub mod metric; /// Resource manager for virtual machines. pub mod resource_manager; /// Virtual machine manager for virtual machines. diff --git a/src/dragonball/src/metric.rs b/src/dragonball/src/metric.rs new file mode 100644 index 000000000000..696fcd6d63a0 --- /dev/null +++ b/src/dragonball/src/metric.rs @@ -0,0 +1,39 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dbs_utils::metric::SharedIncMetric; +use lazy_static::lazy_static; +use serde::Serialize; + +pub use dbs_utils::metric::IncMetric; + +lazy_static! { + /// Static instance used for handling metrics. + pub static ref METRICS: DragonballMetrics = DragonballMetrics::default(); +} + +/// Metrics for the seccomp filtering. +#[derive(Default, Serialize)] +pub struct SeccompMetrics { + /// Number of errors inside the seccomp filtering. + pub num_faults: SharedIncMetric, +} + +/// Metrics related to signals. +#[derive(Default, Serialize)] +pub struct SignalMetrics { + /// Number of times that SIGBUS was handled. + pub sigbus: SharedIncMetric, + /// Number of times that SIGSEGV was handled. + pub sigsegv: SharedIncMetric, +} + +/// Structure storing all metrics while enforcing serialization support on them. +#[derive(Default, Serialize)] +pub struct DragonballMetrics { + /// Metrics related to seccomp filtering. + pub seccomp: SeccompMetrics, + /// Metrics related to signals. + pub signals: SignalMetrics, +} From e89e6507a4ca9666bd3400a15708ade857129820 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sat, 14 May 2022 23:17:19 +0800 Subject: [PATCH 0070/1953] dragonball: add signal handler Used to register dragonball's signal handler. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 1 + src/dragonball/src/lib.rs | 22 +++ src/dragonball/src/signal_handler.rs | 219 +++++++++++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 src/dragonball/src/signal_handler.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index e932da1ab2b8..0eefff8e70c9 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -27,6 +27,7 @@ libc = "0.2.39" linux-loader = "0.4.0" log = "0.4.14" nix = "0.23.1" +seccompiler = "0.2.0" serde = "1.0.27" serde_derive = "1.0.27" serde_json = "1.0.9" diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index e5f90cfc7afc..a36fc42d7355 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -1,4 +1,5 @@ // Copyright (C) 2018-2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 //! Dragonball is a light-weight virtual machine manager(VMM) based on Linux Kernel-based Virtual @@ -17,6 +18,8 @@ pub mod config_manager; pub mod device_manager; /// Errors related to Virtual machine manager. pub mod error; +/// Signal handler for virtual machines. +pub mod signal_handler; /// Metrics system. pub mod metric; /// Resource manager for virtual machines. @@ -26,3 +29,22 @@ pub mod vm; mod io_manager; pub use self::io_manager::IoManagerCached; + +/// Success exit code. +pub const EXIT_CODE_OK: u8 = 0; +/// Generic error exit code. +pub const EXIT_CODE_GENERIC_ERROR: u8 = 1; +/// Generic exit code for an error considered not possible to occur if the program logic is sound. +pub const EXIT_CODE_UNEXPECTED_ERROR: u8 = 2; +/// Dragonball was shut down after intercepting a restricted system call. +pub const EXIT_CODE_BAD_SYSCALL: u8 = 148; +/// Dragonball was shut down after intercepting `SIGBUS`. +pub const EXIT_CODE_SIGBUS: u8 = 149; +/// Dragonball was shut down after intercepting `SIGSEGV`. +pub const EXIT_CODE_SIGSEGV: u8 = 150; +/// Invalid json passed to the Dragonball process for configuring microvm. +pub const EXIT_CODE_INVALID_JSON: u8 = 151; +/// Bad configuration for microvm's resources, when using a single json. +pub const EXIT_CODE_BAD_CONFIGURATION: u8 = 152; +/// Command line arguments parsing error. +pub const EXIT_CODE_ARG_PARSING: u8 = 153; diff --git a/src/dragonball/src/signal_handler.rs b/src/dragonball/src/signal_handler.rs new file mode 100644 index 000000000000..23e9ff39767f --- /dev/null +++ b/src/dragonball/src/signal_handler.rs @@ -0,0 +1,219 @@ +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use libc::{_exit, c_int, c_void, siginfo_t, SIGBUS, SIGSEGV, SIGSYS}; +use log::error; +use vmm_sys_util::signal::register_signal_handler; + +use crate::metric::{IncMetric, METRICS}; + +// The offset of `si_syscall` (offending syscall identifier) within the siginfo structure +// expressed as an `(u)int*`. +// Offset `6` for an `i32` field means that the needed information is located at `6 * sizeof(i32)`. +// See /usr/include/linux/signal.h for the C struct definition. +// See https://github.com/rust-lang/libc/issues/716 for why the offset is different in Rust. +const SI_OFF_SYSCALL: isize = 6; + +const SYS_SECCOMP_CODE: i32 = 1; + +extern "C" { + fn __libc_current_sigrtmin() -> c_int; + fn __libc_current_sigrtmax() -> c_int; +} + +/// Gets current sigrtmin +pub fn sigrtmin() -> c_int { + unsafe { __libc_current_sigrtmin() } +} + +/// Gets current sigrtmax +pub fn sigrtmax() -> c_int { + unsafe { __libc_current_sigrtmax() } +} + +/// Signal handler for `SIGSYS`. +/// +/// Increments the `seccomp.num_faults` metric, logs an error message and terminates the process +/// with a specific exit code. +extern "C" fn sigsys_handler(num: c_int, info: *mut siginfo_t, _unused: *mut c_void) { + // Safe because we're just reading some fields from a supposedly valid argument. + let si_signo = unsafe { (*info).si_signo }; + let si_code = unsafe { (*info).si_code }; + + // Sanity check. The condition should never be true. + if num != si_signo || num != SIGSYS || si_code != SYS_SECCOMP_CODE as i32 { + // Safe because we're terminating the process anyway. + unsafe { _exit(i32::from(super::EXIT_CODE_UNEXPECTED_ERROR)) }; + } + + // Other signals which might do async unsafe things incompatible with the rest of this + // function are blocked due to the sa_mask used when registering the signal handler. + let syscall = unsafe { *(info as *const i32).offset(SI_OFF_SYSCALL) as usize }; + // SIGSYS is triggered when bad syscalls are detected. num_faults is only added when SIGSYS is detected + // so it actually only collects the count for bad syscalls. + METRICS.seccomp.num_faults.inc(); + error!( + "Shutting down VM after intercepting a bad syscall ({}).", + syscall + ); + + // Safe because we're terminating the process anyway. We don't actually do anything when + // running unit tests. + #[cfg(not(test))] + unsafe { + _exit(i32::from(super::EXIT_CODE_BAD_SYSCALL)) + }; +} + +/// Signal handler for `SIGBUS` and `SIGSEGV`. +/// +/// Logs an error message and terminates the process with a specific exit code. +extern "C" fn sigbus_sigsegv_handler(num: c_int, info: *mut siginfo_t, _unused: *mut c_void) { + // Safe because we're just reading some fields from a supposedly valid argument. + let si_signo = unsafe { (*info).si_signo }; + let si_code = unsafe { (*info).si_code }; + + // Sanity check. The condition should never be true. + if num != si_signo || (num != SIGBUS && num != SIGSEGV) { + // Safe because we're terminating the process anyway. + unsafe { _exit(i32::from(super::EXIT_CODE_UNEXPECTED_ERROR)) }; + } + + // Other signals which might do async unsafe things incompatible with the rest of this + // function are blocked due to the sa_mask used when registering the signal handler. + match si_signo { + SIGBUS => METRICS.signals.sigbus.inc(), + SIGSEGV => METRICS.signals.sigsegv.inc(), + _ => (), + } + + error!( + "Shutting down VM after intercepting signal {}, code {}.", + si_signo, si_code + ); + + // Safe because we're terminating the process anyway. We don't actually do anything when + // running unit tests. + #[cfg(not(test))] + unsafe { + _exit(i32::from(match si_signo { + SIGBUS => super::EXIT_CODE_SIGBUS, + SIGSEGV => super::EXIT_CODE_SIGSEGV, + _ => super::EXIT_CODE_UNEXPECTED_ERROR, + })) + }; +} + +/// Registers all the required signal handlers. +/// +/// Custom handlers are installed for: `SIGBUS`, `SIGSEGV`, `SIGSYS`. +pub fn register_signal_handlers() -> vmm_sys_util::errno::Result<()> { + // Call to unsafe register_signal_handler which is considered unsafe because it will + // register a signal handler which will be called in the current thread and will interrupt + // whatever work is done on the current thread, so we have to keep in mind that the registered + // signal handler must only do async-signal-safe operations. + register_signal_handler(SIGSYS, sigsys_handler)?; + register_signal_handler(SIGBUS, sigbus_sigsegv_handler)?; + register_signal_handler(SIGSEGV, sigbus_sigsegv_handler)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use libc::{cpu_set_t, syscall}; + use std::convert::TryInto; + use std::{mem, process, thread}; + + use seccompiler::{apply_filter, BpfProgram, SeccompAction, SeccompFilter}; + + // This function is used when running unit tests, so all the unsafes are safe. + fn cpu_count() -> usize { + let mut cpuset: cpu_set_t = unsafe { mem::zeroed() }; + unsafe { + libc::CPU_ZERO(&mut cpuset); + } + let ret = unsafe { + libc::sched_getaffinity( + 0, + mem::size_of::(), + &mut cpuset as *mut cpu_set_t, + ) + }; + assert_eq!(ret, 0); + + let mut num = 0; + for i in 0..libc::CPU_SETSIZE as usize { + if unsafe { libc::CPU_ISSET(i, &cpuset) } { + num += 1; + } + } + num + } + + #[test] + fn test_signal_handler() { + let child = thread::spawn(move || { + assert!(register_signal_handlers().is_ok()); + + let filter = SeccompFilter::new( + vec![ + (libc::SYS_brk, vec![]), + (libc::SYS_exit, vec![]), + (libc::SYS_futex, vec![]), + (libc::SYS_getpid, vec![]), + (libc::SYS_munmap, vec![]), + (libc::SYS_kill, vec![]), + (libc::SYS_rt_sigprocmask, vec![]), + (libc::SYS_rt_sigreturn, vec![]), + (libc::SYS_sched_getaffinity, vec![]), + (libc::SYS_set_tid_address, vec![]), + (libc::SYS_sigaltstack, vec![]), + (libc::SYS_write, vec![]), + ] + .into_iter() + .collect(), + SeccompAction::Trap, + SeccompAction::Allow, + std::env::consts::ARCH.try_into().unwrap(), + ) + .unwrap(); + + assert!(apply_filter(&TryInto::::try_into(filter).unwrap()).is_ok()); + assert_eq!(METRICS.seccomp.num_faults.count(), 0); + + // Call the blacklisted `SYS_mkdirat`. + unsafe { syscall(libc::SYS_mkdirat, "/foo/bar\0") }; + + // Call SIGBUS signal handler. + assert_eq!(METRICS.signals.sigbus.count(), 0); + unsafe { + syscall(libc::SYS_kill, process::id(), SIGBUS); + } + + // Call SIGSEGV signal handler. + assert_eq!(METRICS.signals.sigsegv.count(), 0); + unsafe { + syscall(libc::SYS_kill, process::id(), SIGSEGV); + } + }); + assert!(child.join().is_ok()); + + // Sanity check. + assert!(cpu_count() > 0); + // Kcov somehow messes with our handler getting the SIGSYS signal when a bad syscall + // is caught, so the following assertion no longer holds. Ideally, we'd have a surefire + // way of either preventing this behaviour, or detecting for certain whether this test is + // run by kcov or not. The best we could do so far is to look at the perceived number of + // available CPUs. Kcov seems to make a single CPU available to the process running the + // tests, so we use this as an heuristic to decide if we check the assertion. + if cpu_count() > 1 { + // The signal handler should let the program continue during unit tests. + assert!(METRICS.seccomp.num_faults.count() >= 1); + } + assert!(METRICS.signals.sigbus.count() >= 1); + assert!(METRICS.signals.sigsegv.count() >= 1); + } +} From 468c73b3cb3b50ab1996b57d89b7c038d928dc61 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sat, 14 May 2022 23:49:52 +0800 Subject: [PATCH 0071/1953] dragonball: add kvm context KVM operation context for virtual machines. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/src/error.rs | 31 ++++ src/dragonball/src/kvm_context.rs | 251 ++++++++++++++++++++++++++++++ src/dragonball/src/lib.rs | 7 +- 3 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 src/dragonball/src/kvm_context.rs diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 5a497abb6b02..2858103a4869 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -14,6 +14,37 @@ use dbs_virtio_devices::Error as VirtIoError; use crate::device_manager; +/// Shorthand result type for internal VMM commands. +pub type Result = std::result::Result; + +/// Errors associated with the VMM internal logic. +/// +/// These errors cannot be generated by direct user input, but can result from bad configuration +/// of the host (for example if Dragonball doesn't have permissions to open the KVM fd). +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Failure occurs in issuing KVM ioctls and errors will be returned from kvm_ioctls lib. + #[error("failure in issuing KVM ioctl command")] + Kvm(#[source] kvm_ioctls::Error), + + /// The host kernel reports an unsupported KVM API version. + #[error("unsupported KVM version {0}")] + KvmApiVersion(i32), + + /// Cannot initialize the KVM context due to missing capabilities. + #[error("missing KVM capability")] + KvmCap(kvm_ioctls::Cap), + + #[cfg(target_arch = "x86_64")] + #[error("failed to configure MSRs")] + /// Cannot configure MSRs + GuestMSRs(dbs_arch::msr::Error), + + /// MSR inner error + #[error("MSR inner error")] + Msr(vmm_sys_util::fam::Error), +} + /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] pub enum StartMicrovmError { diff --git a/src/dragonball/src/kvm_context.rs b/src/dragonball/src/kvm_context.rs new file mode 100644 index 000000000000..f160b264b80e --- /dev/null +++ b/src/dragonball/src/kvm_context.rs @@ -0,0 +1,251 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. +#![allow(dead_code)] +use kvm_bindings::KVM_API_VERSION; +use kvm_ioctls::{Cap, Kvm, VmFd}; +use std::os::unix::io::{FromRawFd, RawFd}; + +use crate::error::{Error, Result}; + +/// Describes a KVM context that gets attached to the micro VM instance. +/// It gives access to the functionality of the KVM wrapper as long as every required +/// KVM capability is present on the host. +pub struct KvmContext { + kvm: Kvm, + max_memslots: usize, + #[cfg(target_arch = "x86_64")] + supported_msrs: kvm_bindings::MsrList, +} + +impl KvmContext { + /// Create a new KVM context object, using the provided `kvm_fd` if one is presented. + pub fn new(kvm_fd: Option) -> Result { + let kvm = if let Some(fd) = kvm_fd { + // Safe because we expect kvm_fd to contain a valid fd number when is_some() == true. + unsafe { Kvm::from_raw_fd(fd) } + } else { + Kvm::new().map_err(Error::Kvm)? + }; + + if kvm.get_api_version() != KVM_API_VERSION as i32 { + return Err(Error::KvmApiVersion(kvm.get_api_version())); + } + + Self::check_cap(&kvm, Cap::Irqchip)?; + Self::check_cap(&kvm, Cap::Irqfd)?; + Self::check_cap(&kvm, Cap::Ioeventfd)?; + Self::check_cap(&kvm, Cap::UserMemory)?; + #[cfg(target_arch = "x86_64")] + Self::check_cap(&kvm, Cap::SetTssAddr)?; + + #[cfg(target_arch = "x86_64")] + let supported_msrs = dbs_arch::msr::supported_guest_msrs(&kvm).map_err(Error::GuestMSRs)?; + let max_memslots = kvm.get_nr_memslots(); + + Ok(KvmContext { + kvm, + max_memslots, + #[cfg(target_arch = "x86_64")] + supported_msrs, + }) + } + + /// Get underlying KVM object to access kvm-ioctls interfaces. + pub fn kvm(&self) -> &Kvm { + &self.kvm + } + + /// Get the maximum number of memory slots reported by this KVM context. + pub fn max_memslots(&self) -> usize { + self.max_memslots + } + + /// Create a virtual machine object. + pub fn create_vm(&self) -> Result { + self.kvm.create_vm().map_err(Error::Kvm) + } + + /// Get the max vcpu count supported by kvm + pub fn get_max_vcpus(&self) -> usize { + self.kvm.get_max_vcpus() + } + + fn check_cap(kvm: &Kvm, cap: Cap) -> std::result::Result<(), Error> { + if !kvm.check_extension(cap) { + return Err(Error::KvmCap(cap)); + } + Ok(()) + } +} + +#[cfg(target_arch = "x86_64")] +mod x86_64 { + use super::*; + use dbs_arch::msr::*; + use kvm_bindings::{kvm_msr_entry, CpuId, MsrList, Msrs}; + use std::collections::HashSet; + + impl KvmContext { + /// Get information about supported CPUID of x86 processor. + pub fn supported_cpuid( + &self, + max_entries_count: usize, + ) -> std::result::Result { + self.kvm.get_supported_cpuid(max_entries_count) + } + + /// Get information about supported MSRs of x86 processor. + pub fn supported_msrs( + &self, + _max_entries_count: usize, + ) -> std::result::Result { + Ok(self.supported_msrs.clone()) + } + + // It's very sensible to manipulate MSRs, so please be careful to change code below. + fn build_msrs_list(kvm: &Kvm) -> Result { + let mut mset: HashSet = HashSet::new(); + let supported_msr_list = kvm.get_msr_index_list().map_err(super::Error::Kvm)?; + for msr in supported_msr_list.as_slice() { + mset.insert(*msr); + } + + let mut msrs = vec![ + MSR_IA32_APICBASE, + MSR_IA32_SYSENTER_CS, + MSR_IA32_SYSENTER_ESP, + MSR_IA32_SYSENTER_EIP, + MSR_IA32_CR_PAT, + ]; + + let filters_list = vec![ + MSR_STAR, + MSR_VM_HSAVE_PA, + MSR_TSC_AUX, + MSR_IA32_TSC_ADJUST, + MSR_IA32_TSCDEADLINE, + MSR_IA32_MISC_ENABLE, + MSR_IA32_BNDCFGS, + MSR_IA32_SPEC_CTRL, + ]; + for msr in filters_list { + if mset.contains(&msr) { + msrs.push(msr); + } + } + + // TODO: several msrs are optional. + + // TODO: Since our guests don't support nested-vmx, LMCE nor SGX for now. + // msrs.push(MSR_IA32_FEATURE_CONTROL); + + msrs.push(MSR_CSTAR); + msrs.push(MSR_KERNEL_GS_BASE); + msrs.push(MSR_SYSCALL_MASK); + msrs.push(MSR_LSTAR); + msrs.push(MSR_IA32_TSC); + + msrs.push(MSR_KVM_SYSTEM_TIME_NEW); + msrs.push(MSR_KVM_WALL_CLOCK_NEW); + + // FIXME: check if it's supported. + msrs.push(MSR_KVM_ASYNC_PF_EN); + msrs.push(MSR_KVM_PV_EOI_EN); + msrs.push(MSR_KVM_STEAL_TIME); + + msrs.push(MSR_CORE_PERF_FIXED_CTR_CTRL); + msrs.push(MSR_CORE_PERF_GLOBAL_CTRL); + msrs.push(MSR_CORE_PERF_GLOBAL_STATUS); + msrs.push(MSR_CORE_PERF_GLOBAL_OVF_CTRL); + + const MAX_FIXED_COUNTERS: u32 = 3; + for i in 0..MAX_FIXED_COUNTERS { + msrs.push(MSR_CORE_PERF_FIXED_CTR0 + i); + } + + // FIXME: skip MCE for now. + + let mtrr_msrs = vec![ + MSR_MTRRdefType, + MSR_MTRRfix64K_00000, + MSR_MTRRfix16K_80000, + MSR_MTRRfix16K_A0000, + MSR_MTRRfix4K_C0000, + MSR_MTRRfix4K_C8000, + MSR_MTRRfix4K_D0000, + MSR_MTRRfix4K_D8000, + MSR_MTRRfix4K_E0000, + MSR_MTRRfix4K_E8000, + MSR_MTRRfix4K_F0000, + MSR_MTRRfix4K_F8000, + ]; + for mtrr in mtrr_msrs { + msrs.push(mtrr); + } + + const MSR_MTRRCAP_VCNT: u32 = 8; + for i in 0..MSR_MTRRCAP_VCNT { + msrs.push(0x200 + 2 * i); + msrs.push(0x200 + 2 * i + 1); + } + + let msrs: Vec = msrs + .iter() + .map(|reg| kvm_msr_entry { + index: *reg, + reserved: 0, + data: 0, + }) + .collect(); + + Msrs::from_entries(&msrs).map_err(super::Error::Msr) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kvm_ioctls::Kvm; + use std::fs::File; + use std::os::unix::fs::MetadataExt; + use std::os::unix::io::{AsRawFd, FromRawFd}; + + #[test] + fn test_create_kvm_context() { + let c = KvmContext::new(None).unwrap(); + + assert!(c.max_memslots >= 32); + + let kvm = Kvm::new().unwrap(); + let f = unsafe { File::from_raw_fd(kvm.as_raw_fd()) }; + let m1 = f.metadata().unwrap(); + let m2 = File::open("/dev/kvm").unwrap().metadata().unwrap(); + + assert_eq!(m1.dev(), m2.dev()); + assert_eq!(m1.ino(), m2.ino()); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_get_supported_cpu_id() { + let c = KvmContext::new(None).unwrap(); + + let _ = c + .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES) + .expect("failed to get supported CPUID"); + assert!(c.supported_cpuid(0).is_err()); + } + + #[test] + fn test_create_vm() { + let c = KvmContext::new(None).unwrap(); + + let _ = c.create_vm().unwrap(); + } +} diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index a36fc42d7355..79e27cc61709 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -6,7 +6,6 @@ //! Machine(KVM) which is optimized for container workloads. #![warn(missing_docs)] - //TODO: Remove this, after the rest of dragonball has been committed. #![allow(dead_code)] @@ -18,12 +17,14 @@ pub mod config_manager; pub mod device_manager; /// Errors related to Virtual machine manager. pub mod error; -/// Signal handler for virtual machines. -pub mod signal_handler; +/// KVM operation context for virtual machines. +pub mod kvm_context; /// Metrics system. pub mod metric; /// Resource manager for virtual machines. pub mod resource_manager; +/// Signal handler for virtual machines. +pub mod signal_handler; /// Virtual machine manager for virtual machines. pub mod vm; From 7d1953b52e9015a45c1fc0101841183cb3e07cd0 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 00:03:45 +0800 Subject: [PATCH 0072/1953] dragonball: add vcpu Virtual CPU manager for virtual machines. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 1 + src/dragonball/src/lib.rs | 2 + src/dragonball/src/metric.rs | 19 + src/dragonball/src/vcpu/aarch64.rs | 94 +++ src/dragonball/src/vcpu/mod.rs | 31 + src/dragonball/src/vcpu/sm.rs | 149 ++++ src/dragonball/src/vcpu/vcpu_impl.rs | 955 ++++++++++++++++++++++++ src/dragonball/src/vcpu/vcpu_manager.rs | 5 + src/dragonball/src/vcpu/x86_64.rs | 149 ++++ 9 files changed, 1405 insertions(+) create mode 100644 src/dragonball/src/vcpu/aarch64.rs create mode 100644 src/dragonball/src/vcpu/mod.rs create mode 100644 src/dragonball/src/vcpu/sm.rs create mode 100644 src/dragonball/src/vcpu/vcpu_impl.rs create mode 100644 src/dragonball/src/vcpu/vcpu_manager.rs create mode 100644 src/dragonball/src/vcpu/x86_64.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 0eefff8e70c9..3440b4cff7de 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -14,6 +14,7 @@ arc-swap = "1.5.0" bytes = "1.1.0" dbs-address-space = "0.1.0" dbs-allocator = "0.1.0" +dbs-arch = "0.1.0" dbs-boot = "0.2.0" dbs-device = "0.1.0" dbs-interrupt = { version = "0.1.0", features = ["kvm-irq"] } diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index 79e27cc61709..e9330fc95346 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -25,6 +25,8 @@ pub mod metric; pub mod resource_manager; /// Signal handler for virtual machines. pub mod signal_handler; +/// Virtual CPU manager for virtual machines. +pub mod vcpu; /// Virtual machine manager for virtual machines. pub mod vm; diff --git a/src/dragonball/src/metric.rs b/src/dragonball/src/metric.rs index 696fcd6d63a0..716e9e04406e 100644 --- a/src/dragonball/src/metric.rs +++ b/src/dragonball/src/metric.rs @@ -13,6 +13,23 @@ lazy_static! { pub static ref METRICS: DragonballMetrics = DragonballMetrics::default(); } +/// Metrics specific to VCPUs' mode of functioning. +#[derive(Default, Serialize)] +pub struct VcpuMetrics { + /// Number of KVM exits for handling input IO. + pub exit_io_in: SharedIncMetric, + /// Number of KVM exits for handling output IO. + pub exit_io_out: SharedIncMetric, + /// Number of KVM exits for handling MMIO reads. + pub exit_mmio_read: SharedIncMetric, + /// Number of KVM exits for handling MMIO writes. + pub exit_mmio_write: SharedIncMetric, + /// Number of errors during this VCPU's run. + pub failures: SharedIncMetric, + /// Failures in configuring the CPUID. + pub filter_cpuid: SharedIncMetric, +} + /// Metrics for the seccomp filtering. #[derive(Default, Serialize)] pub struct SeccompMetrics { @@ -32,6 +49,8 @@ pub struct SignalMetrics { /// Structure storing all metrics while enforcing serialization support on them. #[derive(Default, Serialize)] pub struct DragonballMetrics { + /// Metrics related to a vcpu's functioning. + pub vcpu: VcpuMetrics, /// Metrics related to seccomp filtering. pub seccomp: SeccompMetrics, /// Metrics related to signals. diff --git a/src/dragonball/src/vcpu/aarch64.rs b/src/dragonball/src/vcpu/aarch64.rs new file mode 100644 index 000000000000..8e78efdc10ac --- /dev/null +++ b/src/dragonball/src/vcpu/aarch64.rs @@ -0,0 +1,94 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::sync::mpsc::{channel, Sender}; +use std::sync::Arc; + +use crate::IoManagerCached; +use dbs_utils::time::TimestampUs; +use kvm_ioctls::{VcpuFd, VmFd}; +use vm_memory::GuestAddress; +use vmm_sys_util::eventfd::EventFd; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuStateEvent}; +use crate::vcpu::VcpuConfig; + +#[allow(unused)] +impl Vcpu { + /// Constructs a new VCPU for `vm`. + /// + /// # Arguments + /// + /// * `id` - Represents the CPU number between [0, max vcpus). + /// * `vcpu_fd` - The kvm `VcpuFd` for the vcpu. + /// * `io_mgr` - The io-manager used to access port-io and mmio devices. + /// * `exit_evt` - An `EventFd` that will be written into when this vcpu + /// exits. + /// * `vcpu_state_event` - The eventfd which can notify vmm state of some + /// vcpu should change. + /// * `vcpu_state_sender` - The channel to send state change message from + /// vcpu thread to vmm thread. + /// * `create_ts` - A timestamp used by the vcpu to calculate its lifetime. + /// * `support_immediate_exit` - whether kvm uses supports immediate_exit flag. + pub fn new_aarch64( + id: u8, + vcpu_fd: Arc, + io_mgr: IoManagerCached, + exit_evt: EventFd, + vcpu_state_event: EventFd, + vcpu_state_sender: Sender, + create_ts: TimestampUs, + support_immediate_exit: bool, + ) -> Result { + let (event_sender, event_receiver) = channel(); + let (response_sender, response_receiver) = channel(); + + Ok(Vcpu { + fd: vcpu_fd, + id, + io_mgr, + create_ts, + event_receiver, + event_sender: Some(event_sender), + response_receiver: Some(response_receiver), + response_sender, + vcpu_state_event, + vcpu_state_sender, + support_immediate_exit, + mpidr: 0, + exit_evt, + }) + } + + /// Configures an aarch64 specific vcpu. + /// + /// # Arguments + /// + /// * `vcpu_config` - vCPU config for this vCPU status + /// * `vm_fd` - The kvm `VmFd` for this microvm. + /// * `vm_as` - The guest memory address space used by this microvm. + /// * `kernel_load_addr` - Offset from `guest_mem` at which the kernel is loaded. + /// * `_pgtable_addr` - pgtable address for ap vcpu (not used in aarch64) + pub fn configure( + &mut self, + _vcpu_config: &VcpuConfig, + vm_fd: &VmFd, + vm_as: &GuestAddressSpaceImpl, + kernel_load_addr: Option, + _pgtable_addr: Option, + ) -> Result<()> { + // TODO: add arm vcpu configure() function. issue: #4445 + Ok(()) + } + + /// Gets the MPIDR register value. + pub fn get_mpidr(&self) -> u64 { + self.mpidr + } +} diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs new file mode 100644 index 000000000000..82b58e9ea953 --- /dev/null +++ b/src/dragonball/src/vcpu/mod.rs @@ -0,0 +1,31 @@ +// Copyright (C) 2022 Alibaba Cloud Computing. All rights reserved. +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +mod sm; +pub mod vcpu_impl; + +#[cfg(target_arch = "x86_64")] +use dbs_arch::cpuid::VpmuFeatureLevel; + +/// vcpu config collection +pub struct VcpuConfig { + /// initial vcpu count + pub boot_vcpu_count: u8, + /// max vcpu count for hotplug + pub max_vcpu_count: u8, + /// threads per core for cpu topology information + pub threads_per_core: u8, + /// cores per die for cpu topology information + pub cores_per_die: u8, + /// dies per socket for cpu topology information + pub dies_per_socket: u8, + /// socket number for cpu topology information + pub sockets: u8, + /// if vpmu feature is Disabled, it means vpmu feature is off (by default) + /// if vpmu feature is LimitedlyEnabled, it means minimal vpmu counters are supported (cycles and instructions) + /// if vpmu feature is FullyEnabled, it means all vpmu counters are supported + #[cfg(target_arch = "x86_64")] + pub vpmu_feature: VpmuFeatureLevel, +} diff --git a/src/dragonball/src/vcpu/sm.rs b/src/dragonball/src/vcpu/sm.rs new file mode 100644 index 000000000000..2a51d64083a4 --- /dev/null +++ b/src/dragonball/src/vcpu/sm.rs @@ -0,0 +1,149 @@ +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ops::Deref; + +/// Simple abstraction of a state machine. +/// +/// `StateMachine` is a wrapper over `T` that also encodes state information for `T`. +/// +/// Each state for `T` is represented by a `StateFn` which is a function that acts as +/// the state handler for that particular state of `T`. +/// +/// `StateFn` returns exactly one other `StateMachine` thus each state gets clearly +/// defined transitions to other states. +pub struct StateMachine { + function: StateFn, + end_state: bool, +} + +/// Type representing a state handler of a `StateMachine` machine. Each state handler +/// is a function from `T` that handles a specific state of `T`. +type StateFn = fn(&mut T) -> StateMachine; + +impl StateMachine { + /// Creates a new state wrapper. + /// + /// # Arguments + /// + /// `function` - the state handler for this state. + /// `end_state` - whether this state is final. + pub fn new(function: StateFn, end_state: bool) -> StateMachine { + StateMachine { + function, + end_state, + } + } + + /// Creates a new state wrapper that has further possible transitions. + /// + /// # Arguments + /// + /// `function` - the state handler for this state. + pub fn next(function: StateFn) -> StateMachine { + StateMachine::new(function, false) + } + + /// Creates a new state wrapper that has no further transitions. The state machine + /// will finish after running this handler. + /// + /// # Arguments + /// + /// `function` - the state handler for this last state. + pub fn finish(function: StateFn) -> StateMachine { + StateMachine::new(function, true) + } + + /// Runs a state machine for `T` starting from the provided state. + /// + /// # Arguments + /// + /// `machine` - a mutable reference to the object running through the various states. + /// `starting_state_fn` - a `fn(&mut T) -> StateMachine` that should be the handler for + /// the initial state. + pub fn run(machine: &mut T, starting_state_fn: StateFn) { + // Start off in the `starting_state` state. + let mut sf = StateMachine::new(starting_state_fn, false); + // While current state is not a final/end state, keep churning. + while !sf.end_state { + // Run the current state handler, and get the next one. + sf = sf(machine); + } + } +} + +// Implement Deref of `StateMachine` so that we can directly call its underlying state handler. +impl Deref for StateMachine { + type Target = StateFn; + fn deref(&self) -> &Self::Target { + &self.function + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // DummyMachine with states `s1`, `s2` and `s3`. + struct DummyMachine { + private_data_s1: bool, + private_data_s2: bool, + private_data_s3: bool, + } + + impl DummyMachine { + fn new() -> Self { + DummyMachine { + private_data_s1: false, + private_data_s2: false, + private_data_s3: false, + } + } + + // DummyMachine functions here. + + // Simple state-machine: start->s1->s2->s3->done. + fn run(&mut self) { + // Verify the machine has not run yet. + assert!(!self.private_data_s1); + assert!(!self.private_data_s2); + assert!(!self.private_data_s3); + + // Run the state-machine. + StateMachine::run(self, Self::s1); + + // Verify the machine went through all states. + assert!(self.private_data_s1); + assert!(self.private_data_s2); + assert!(self.private_data_s3); + } + + fn s1(&mut self) -> StateMachine { + // Verify private data mutates along with the states. + assert!(!self.private_data_s1); + self.private_data_s1 = true; + StateMachine::next(Self::s2) + } + + fn s2(&mut self) -> StateMachine { + // Verify private data mutates along with the states. + assert!(!self.private_data_s2); + self.private_data_s2 = true; + StateMachine::next(Self::s3) + } + + fn s3(&mut self) -> StateMachine { + // Verify private data mutates along with the states. + assert!(!self.private_data_s3); + self.private_data_s3 = true; + // The machine ends here, adding `s1` as next state to validate this. + StateMachine::finish(Self::s1) + } + } + + #[test] + fn test_sm() { + let mut machine = DummyMachine::new(); + machine.run(); + } +} diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs new file mode 100644 index 000000000000..cb7e83af12b6 --- /dev/null +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -0,0 +1,955 @@ +// Copyright (C) 2019-2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +//! The implementation for per vcpu + +use std::cell::Cell; +use std::result; +use std::sync::atomic::{fence, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError}; +use std::sync::{Arc, Barrier}; +use std::thread; + +use dbs_utils::time::TimestampUs; +use kvm_bindings::{KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN}; +use kvm_ioctls::{VcpuExit, VcpuFd}; +use libc::{c_int, c_void, siginfo_t}; +use log::{error, info, warn}; +use seccompiler::{apply_filter, BpfProgram, Error as SecError}; +use vmm_sys_util::eventfd::EventFd; +use vmm_sys_util::signal::{register_signal_handler, Killable}; + +use super::sm::StateMachine; +use crate::metric::{IncMetric, METRICS}; +use crate::signal_handler::sigrtmin; +use crate::IoManagerCached; + +#[cfg(target_arch = "x86_64")] +#[path = "x86_64.rs"] +mod x86_64; + +#[cfg(target_arch = "aarch64")] +#[path = "aarch64.rs"] +mod aarch64; + +#[cfg(target_arch = "x86_64")] +const MAGIC_IOPORT_BASE: u16 = 0xdbdb; +#[cfg(target_arch = "x86_64")] +const MAGIC_IOPORT_DEBUG_INFO: u16 = MAGIC_IOPORT_BASE; + +/// Signal number (SIGRTMIN) used to kick Vcpus. +pub const VCPU_RTSIG_OFFSET: i32 = 0; + +#[cfg(target_arch = "x86_64")] +/// Errors associated with the wrappers over KVM ioctls. +#[derive(Debug, thiserror::Error)] +pub enum VcpuError { + /// Failed to signal Vcpu. + #[error("cannot signal the vCPU thread")] + SignalVcpu(#[source] vmm_sys_util::errno::Error), + + /// Cannot open the vCPU file descriptor. + #[error("cannot open the vCPU file descriptor")] + VcpuFd(#[source] kvm_ioctls::Error), + + /// Cannot spawn a new vCPU thread. + #[error("cannot spawn vCPU thread")] + VcpuSpawn(#[source] std::io::Error), + + /// Cannot cleanly initialize vCPU TLS. + #[error("cannot cleanly initialize TLS fro vCPU")] + VcpuTlsInit, + + /// Vcpu not present in TLS. + #[error("vCPU not present in the TLS")] + VcpuTlsNotPresent, + + /// Unexpected KVM_RUN exit reason + #[error("Unexpected KVM_RUN exit reason")] + VcpuUnhandledKvmExit, + + /// Pause vcpu failed + #[error("failed to pause vcpus")] + PauseFailed, + + /// Kvm Ioctl Error + #[error("failure in issuing KVM ioctl command")] + Kvm(#[source] kvm_ioctls::Error), + + /// Msr error + #[error("failure to deal with MSRs")] + Msr(vmm_sys_util::fam::Error), + + /// A call to cpuid instruction failed on x86_64. + #[error("failure while configuring CPUID for virtual CPU on x86_64")] + CpuId(dbs_arch::cpuid::Error), + + /// Error configuring the floating point related registers on x86_64. + #[error("failure while configuring the floating point related registers on x86_64")] + FPUConfiguration(dbs_arch::regs::Error), + + /// Cannot set the local interruption due to bad configuration on x86_64. + #[error("cannot set the local interruption due to bad configuration on x86_64")] + LocalIntConfiguration(dbs_arch::interrupts::Error), + + /// Error configuring the MSR registers on x86_64. + #[error("failure while configuring the MSR registers on x86_64")] + MSRSConfiguration(dbs_arch::regs::Error), + + /// Error configuring the general purpose registers on x86_64. + #[error("failure while configuring the general purpose registers on x86_64")] + REGSConfiguration(dbs_arch::regs::Error), + + /// Error configuring the special registers on x86_64. + #[error("failure while configuring the special registers on x86_64")] + SREGSConfiguration(dbs_arch::regs::Error), + + /// Error configuring the page table on x86_64. + #[error("failure while configuring the page table on x86_64")] + PageTable(dbs_boot::Error), + + /// The call to KVM_SET_CPUID2 failed on x86_64. + #[error("failure while calling KVM_SET_CPUID2 on x86_64")] + SetSupportedCpusFailed(#[source] kvm_ioctls::Error), +} + +#[cfg(target_arch = "aarch64")] +/// Errors associated with the wrappers over KVM ioctls. +#[derive(Debug, thiserror::Error)] +pub enum VcpuError { + /// Failed to signal Vcpu. + #[error("cannot signal the vCPU thread")] + SignalVcpu(#[source] vmm_sys_util::errno::Error), + + /// Cannot open the vCPU file descriptor. + #[error("cannot open the vCPU file descriptor")] + VcpuFd(#[source] kvm_ioctls::Error), + + /// Cannot spawn a new vCPU thread. + #[error("cannot spawn vCPU thread")] + VcpuSpawn(#[source] std::io::Error), + + /// Cannot cleanly initialize vCPU TLS. + #[error("cannot cleanly initialize TLS fro vCPU")] + VcpuTlsInit, + + /// Vcpu not present in TLS. + #[error("vCPU not present in the TLS")] + VcpuTlsNotPresent, + + /// Unexpected KVM_RUN exit reason + #[error("Unexpected KVM_RUN exit reason")] + VcpuUnhandledKvmExit, + + /// Pause vcpu failed + #[error("failed to pause vcpus")] + PauseFailed, + + /// Kvm Ioctl Error + #[error("failure in issuing KVM ioctl command")] + Kvm(#[source] kvm_ioctls::Error), + + /// Msr error + #[error("failure to deal with MSRs")] + Msr(vmm_sys_util::fam::Error), + + #[cfg(target_arch = "aarch64")] + /// Error configuring the general purpose aarch64 registers on aarch64. + #[error("failure while configuring the general purpose registers on aarch64")] + REGSConfiguration(dbs_arch::regs::Error), + + #[cfg(target_arch = "aarch64")] + /// Error setting up the global interrupt controller on aarch64. + #[error("failure while setting up the global interrupt controller on aarch64")] + SetupGIC(dbs_arch::gic::Error), + + #[cfg(target_arch = "aarch64")] + /// Error getting the Vcpu preferred target on aarch64. + #[error("failure while getting the vCPU preferred target on aarch64")] + VcpuArmPreferredTarget(kvm_ioctls::Error), + + #[cfg(target_arch = "aarch64")] + /// Error doing vCPU Init on aarch64. + #[error("failure while doing vCPU init on aarch64")] + VcpuArmInit(kvm_ioctls::Error), +} + +/// Result for Vcpu related operations. +pub type Result = result::Result; + +/// List of events that the Vcpu can receive. +#[derive(Debug)] +pub enum VcpuEvent { + /// Kill the Vcpu. + Exit, + /// Pause the Vcpu. + Pause, + /// Event that should resume the Vcpu. + Resume, + /// Get vcpu thread tid + Gettid, + + /// Event to revalidate vcpu IoManager cache + RevalidateCache, +} + +/// List of responses that the Vcpu reports. +pub enum VcpuResponse { + /// Vcpu is paused. + Paused, + /// Vcpu is resumed. + Resumed, + /// Vcpu index and thread tid. + Tid(u8, u32), + /// Requested Vcpu operation is not allowed. + NotAllowed, + /// Requestion action encountered an error + Error(VcpuError), + /// Vcpu IoManager cache is revalidated + CacheRevalidated, +} + +/// List of events that the vcpu_state_sender can send. +pub enum VcpuStateEvent { + /// For Hotplug + Hotplug((bool, u32)), +} + +/// Wrapper over vCPU that hides the underlying interactions with the vCPU thread. +pub struct VcpuHandle { + event_sender: Sender, + response_receiver: Receiver, + vcpu_thread: thread::JoinHandle<()>, +} + +impl VcpuHandle { + /// Send event to vCPU thread + pub fn send_event(&self, event: VcpuEvent) -> Result<()> { + // Use expect() to crash if the other thread closed this channel. + self.event_sender + .send(event) + .expect("event sender channel closed on vcpu end."); + // Kick the vCPU so it picks up the message. + self.vcpu_thread + .kill(sigrtmin() + VCPU_RTSIG_OFFSET) + .map_err(VcpuError::SignalVcpu)?; + Ok(()) + } + + /// Receive response from vcpu thread + pub fn response_receiver(&self) -> &Receiver { + &self.response_receiver + } + + #[allow(dead_code)] + /// Join the vcpu thread + pub fn join_vcpu_thread(self) -> thread::Result<()> { + self.vcpu_thread.join() + } +} + +#[derive(PartialEq)] +enum VcpuEmulation { + Handled, + Interrupted, + Stopped, +} + +/// A wrapper around creating and using a kvm-based VCPU. +pub struct Vcpu { + // vCPU fd used by the vCPU + fd: Arc, + // vCPU id info + id: u8, + // Io manager Cached for facilitating IO operations + io_mgr: IoManagerCached, + // Records vCPU create time stamp + create_ts: TimestampUs, + + // The receiving end of events channel owned by the vcpu side. + event_receiver: Receiver, + // The transmitting end of the events channel which will be given to the handler. + event_sender: Option>, + // The receiving end of the responses channel which will be given to the handler. + response_receiver: Option>, + // The transmitting end of the responses channel owned by the vcpu side. + response_sender: Sender, + // Event notifier for CPU hotplug. + // After arm adapts to hotplug vcpu, the dead code macro needs to be removed + #[cfg_attr(target_arch = "aarch64", allow(dead_code))] + vcpu_state_event: EventFd, + // CPU hotplug events. + // After arm adapts to hotplug vcpu, the dead code macro needs to be removed + #[cfg_attr(target_arch = "aarch64", allow(dead_code))] + vcpu_state_sender: Sender, + + // An `EventFd` that will be written into when this vcpu exits. + exit_evt: EventFd, + // Whether kvm used supports immediate_exit flag. + support_immediate_exit: bool, + + // CPUID information for the x86_64 CPU + #[cfg(target_arch = "x86_64")] + cpuid: kvm_bindings::CpuId, + + /// Multiprocessor affinity register recorded for aarch64 + #[cfg(target_arch = "aarch64")] + pub(crate) mpidr: u64, +} + +// Using this for easier explicit type-casting to help IDEs interpret the code. +type VcpuCell = Cell>; + +impl Vcpu { + thread_local!(static TLS_VCPU_PTR: VcpuCell = Cell::new(None)); + + /// Associates `self` with the current thread. + /// + /// It is a prerequisite to successfully run `init_thread_local_data()` before using + /// `run_on_thread_local()` on the current thread. + /// This function will return an error if there already is a `Vcpu` present in the TLS. + fn init_thread_local_data(&mut self) -> Result<()> { + Self::TLS_VCPU_PTR.with(|cell: &VcpuCell| { + if cell.get().is_some() { + return Err(VcpuError::VcpuTlsInit); + } + cell.set(Some(self as *const Vcpu)); + Ok(()) + }) + } + + /// Deassociates `self` from the current thread. + /// + /// Should be called if the current `self` had called `init_thread_local_data()` and + /// now needs to move to a different thread. + /// + /// Fails if `self` was not previously associated with the current thread. + fn reset_thread_local_data(&mut self) -> Result<()> { + // Best-effort to clean up TLS. If the `Vcpu` was moved to another thread + // _before_ running this, then there is nothing we can do. + Self::TLS_VCPU_PTR.with(|cell: &VcpuCell| { + if let Some(vcpu_ptr) = cell.get() { + if vcpu_ptr == self as *const Vcpu { + Self::TLS_VCPU_PTR.with(|cell: &VcpuCell| cell.take()); + return Ok(()); + } + } + Err(VcpuError::VcpuTlsNotPresent) + }) + } + + /// Runs `func` for the `Vcpu` associated with the current thread. + /// + /// It requires that `init_thread_local_data()` was run on this thread. + /// + /// Fails if there is no `Vcpu` associated with the current thread. + /// + /// # Safety + /// + /// This is marked unsafe as it allows temporary aliasing through + /// dereferencing from pointer an already borrowed `Vcpu`. + unsafe fn run_on_thread_local(func: F) -> Result<()> + where + F: FnOnce(&Vcpu), + { + Self::TLS_VCPU_PTR.with(|cell: &VcpuCell| { + if let Some(vcpu_ptr) = cell.get() { + // Dereferencing here is safe since `TLS_VCPU_PTR` is populated/non-empty, + // and it is being cleared on `Vcpu::drop` so there is no dangling pointer. + let vcpu_ref: &Vcpu = &*vcpu_ptr; + func(vcpu_ref); + Ok(()) + } else { + Err(VcpuError::VcpuTlsNotPresent) + } + }) + } + + /// Registers a signal handler which makes use of TLS and kvm immediate exit to + /// kick the vcpu running on the current thread, if there is one. + pub fn register_kick_signal_handler() { + extern "C" fn handle_signal(_: c_int, _: *mut siginfo_t, _: *mut c_void) { + // This is safe because it's temporarily aliasing the `Vcpu` object, but we are + // only reading `vcpu.fd` which does not change for the lifetime of the `Vcpu`. + unsafe { + let _ = Vcpu::run_on_thread_local(|vcpu| { + vcpu.fd.set_kvm_immediate_exit(1); + fence(Ordering::Release); + }); + } + } + + register_signal_handler(sigrtmin() + VCPU_RTSIG_OFFSET, handle_signal) + .expect("Failed to register vcpu signal handler"); + } + + /// Returns the cpu index as seen by the guest OS. + pub fn cpu_index(&self) -> u8 { + self.id + } + + /// Moves the vcpu to its own thread and constructs a VcpuHandle. + /// The handle can be used to control the remote vcpu. + pub fn start_threaded( + mut self, + seccomp_filter: BpfProgram, + barrier: Arc, + ) -> Result { + let event_sender = self.event_sender.take().unwrap(); + let response_receiver = self.response_receiver.take().unwrap(); + + let vcpu_thread = thread::Builder::new() + .name(format!("db_vcpu{}", self.cpu_index())) + .spawn(move || { + self.init_thread_local_data() + .expect("Cannot cleanly initialize vcpu TLS."); + barrier.wait(); + self.run(seccomp_filter); + }) + .map_err(VcpuError::VcpuSpawn)?; + + Ok(VcpuHandle { + event_sender, + response_receiver, + vcpu_thread, + }) + } + + /// Extract the vcpu running logic for test mocking. + #[cfg(not(test))] + pub fn emulate(fd: &VcpuFd) -> std::result::Result, kvm_ioctls::Error> { + fd.run() + } + + /// Runs the vCPU in KVM context and handles the kvm exit reason. + /// + /// Returns error or enum specifying whether emulation was handled or interrupted. + fn run_emulation(&mut self) -> Result { + match Vcpu::emulate(&self.fd) { + Ok(run) => match run { + #[cfg(target_arch = "x86_64")] + VcpuExit::IoIn(addr, data) => { + let _ = self.io_mgr.pio_read(addr, data); + METRICS.vcpu.exit_io_in.inc(); + Ok(VcpuEmulation::Handled) + } + #[cfg(target_arch = "x86_64")] + VcpuExit::IoOut(addr, data) => { + if !self.check_io_port_info(addr, data)? { + let _ = self.io_mgr.pio_write(addr, data); + } + METRICS.vcpu.exit_io_out.inc(); + Ok(VcpuEmulation::Handled) + } + VcpuExit::MmioRead(addr, data) => { + let _ = self.io_mgr.mmio_read(addr, data); + METRICS.vcpu.exit_mmio_read.inc(); + Ok(VcpuEmulation::Handled) + } + VcpuExit::MmioWrite(addr, data) => { + #[cfg(target_arch = "aarch64")] + self.check_boot_complete_signal(addr, data); + + let _ = self.io_mgr.mmio_write(addr, data); + METRICS.vcpu.exit_mmio_write.inc(); + Ok(VcpuEmulation::Handled) + } + VcpuExit::Hlt => { + info!("Received KVM_EXIT_HLT signal"); + Err(VcpuError::VcpuUnhandledKvmExit) + } + VcpuExit::Shutdown => { + info!("Received KVM_EXIT_SHUTDOWN signal"); + Err(VcpuError::VcpuUnhandledKvmExit) + } + // Documentation specifies that below kvm exits are considered errors. + VcpuExit::FailEntry => { + METRICS.vcpu.failures.inc(); + error!("Received KVM_EXIT_FAIL_ENTRY signal"); + Err(VcpuError::VcpuUnhandledKvmExit) + } + VcpuExit::InternalError => { + METRICS.vcpu.failures.inc(); + error!("Received KVM_EXIT_INTERNAL_ERROR signal"); + Err(VcpuError::VcpuUnhandledKvmExit) + } + VcpuExit::SystemEvent(event_type, event_flags) => match event_type { + KVM_SYSTEM_EVENT_RESET | KVM_SYSTEM_EVENT_SHUTDOWN => { + info!( + "Received KVM_SYSTEM_EVENT: type: {}, event: {}", + event_type, event_flags + ); + Ok(VcpuEmulation::Stopped) + } + _ => { + METRICS.vcpu.failures.inc(); + error!( + "Received KVM_SYSTEM_EVENT signal type: {}, flag: {}", + event_type, event_flags + ); + Err(VcpuError::VcpuUnhandledKvmExit) + } + }, + r => { + METRICS.vcpu.failures.inc(); + // TODO: Are we sure we want to finish running a vcpu upon + // receiving a vm exit that is not necessarily an error? + error!("Unexpected exit reason on vcpu run: {:?}", r); + Err(VcpuError::VcpuUnhandledKvmExit) + } + }, + // The unwrap on raw_os_error can only fail if we have a logic + // error in our code in which case it is better to panic. + Err(ref e) => { + match e.errno() { + libc::EAGAIN => Ok(VcpuEmulation::Handled), + libc::EINTR => { + self.fd.set_kvm_immediate_exit(0); + // Notify that this KVM_RUN was interrupted. + Ok(VcpuEmulation::Interrupted) + } + _ => { + METRICS.vcpu.failures.inc(); + error!("Failure during vcpu run: {}", e); + #[cfg(target_arch = "x86_64")] + { + error!( + "dump regs: {:?}, dump sregs: {:?}", + self.fd.get_regs(), + self.fd.get_sregs() + ); + } + Err(VcpuError::VcpuUnhandledKvmExit) + } + } + } + } + } + + #[cfg(target_arch = "x86_64")] + // checkout the io port that dragonball used only + fn check_io_port_info(&self, addr: u16, data: &[u8]) -> Result { + let mut checked = false; + + match addr { + // debug info signal + MAGIC_IOPORT_DEBUG_INFO => { + if data.len() == 4 { + let data = unsafe { std::ptr::read(data.as_ptr() as *const u32) }; + warn!("KDBG: guest kernel debug info: 0x{:x}", data); + checked = true; + } + } + _ => {} + }; + + Ok(checked) + } + + fn gettid() -> u32 { + nix::unistd::gettid().as_raw() as u32 + } + + fn revalidate_cache(&mut self) -> Result<()> { + self.io_mgr.revalidate_cache(); + + Ok(()) + } + + /// Main loop of the vCPU thread. + /// + /// Runs the vCPU in KVM context in a loop. Handles KVM_EXITs then goes back in. + /// Note that the state of the VCPU and associated VM must be setup first for this to do + /// anything useful. + pub fn run(&mut self, seccomp_filter: BpfProgram) { + // Load seccomp filters for this vCPU thread. + // Execution panics if filters cannot be loaded, use --seccomp-level=0 if skipping filters + // altogether is the desired behaviour. + if let Err(e) = apply_filter(&seccomp_filter) { + if matches!(e, SecError::EmptyFilter) { + info!("vCPU thread {} use empty seccomp filters.", self.id); + } else { + panic!( + "Failed to set the requested seccomp filters on vCPU {}: Error: {}", + self.id, e + ); + } + } + + info!("vcpu {} is running", self.cpu_index()); + + // Start running the machine state in the `Paused` state. + StateMachine::run(self, Self::paused); + } + + // This is the main loop of the `Running` state. + fn running(&mut self) -> StateMachine { + // This loop is here just for optimizing the emulation path. + // No point in ticking the state machine if there are no external events. + loop { + match self.run_emulation() { + // Emulation ran successfully, continue. + Ok(VcpuEmulation::Handled) => { + // We need to break here if kvm doesn't support + // immediate_exit flag. Because the signal sent from vmm + // thread may occurs when handling the vcpu exit events, and + // in this case the external vcpu events may not be handled + // correctly, so we need to check the event_receiver channel + // after handle vcpu exit events to decrease the window that + // doesn't handle the vcpu external events. + if !self.support_immediate_exit { + break; + } + } + // Emulation was interrupted, check external events. + Ok(VcpuEmulation::Interrupted) => break, + // Emulation was stopped due to reset or shutdown. + Ok(VcpuEmulation::Stopped) => return StateMachine::next(Self::waiting_exit), + // Emulation errors lead to vCPU exit. + Err(e) => { + error!("vcpu: {}, run_emulation failed: {:?}", self.id, e); + return StateMachine::next(Self::waiting_exit); + } + } + } + + // By default don't change state. + let mut state = StateMachine::next(Self::running); + + // Break this emulation loop on any transition request/external event. + match self.event_receiver.try_recv() { + // Running ---- Exit ----> Exited + Ok(VcpuEvent::Exit) => { + // Move to 'exited' state. + state = StateMachine::next(Self::exited); + } + // Running ---- Pause ----> Paused + Ok(VcpuEvent::Pause) => { + // Nothing special to do. + self.response_sender + .send(VcpuResponse::Paused) + .expect("failed to send pause status"); + + // TODO: we should call `KVM_KVMCLOCK_CTRL` here to make sure + // TODO continued: the guest soft lockup watchdog does not panic on Resume. + //let _ = self.fd.kvmclock_ctrl(); + + // Move to 'paused' state. + state = StateMachine::next(Self::paused); + } + Ok(VcpuEvent::Resume) => { + self.response_sender + .send(VcpuResponse::Resumed) + .expect("failed to send resume status"); + } + Ok(VcpuEvent::Gettid) => { + self.response_sender + .send(VcpuResponse::Tid(self.cpu_index(), Vcpu::gettid())) + .expect("failed to send vcpu thread tid"); + } + Ok(VcpuEvent::RevalidateCache) => { + self.revalidate_cache() + .map(|()| { + self.response_sender + .send(VcpuResponse::CacheRevalidated) + .expect("failed to revalidate vcpu IoManager cache"); + }) + .map_err(|e| self.response_sender.send(VcpuResponse::Error(e))) + .expect("failed to revalidate vcpu IoManager cache"); + } + // Unhandled exit of the other end. + Err(TryRecvError::Disconnected) => { + // Move to 'exited' state. + state = StateMachine::next(Self::exited); + } + // All other events or lack thereof have no effect on current 'running' state. + Err(TryRecvError::Empty) => (), + } + + state + } + + // This is the main loop of the `Paused` state. + fn paused(&mut self) -> StateMachine { + match self.event_receiver.recv() { + // Paused ---- Exit ----> Exited + Ok(VcpuEvent::Exit) => { + // Move to 'exited' state. + StateMachine::next(Self::exited) + } + // Paused ---- Resume ----> Running + Ok(VcpuEvent::Resume) => { + self.response_sender + .send(VcpuResponse::Resumed) + .expect("failed to send resume status"); + // Move to 'running' state. + StateMachine::next(Self::running) + } + Ok(VcpuEvent::Pause) => { + self.response_sender + .send(VcpuResponse::Paused) + .expect("failed to send pause status"); + // continue 'pause' state. + StateMachine::next(Self::paused) + } + Ok(VcpuEvent::Gettid) => { + self.response_sender + .send(VcpuResponse::Tid(self.cpu_index(), Vcpu::gettid())) + .expect("failed to send vcpu thread tid"); + StateMachine::next(Self::paused) + } + Ok(VcpuEvent::RevalidateCache) => { + self.revalidate_cache() + .map(|()| { + self.response_sender + .send(VcpuResponse::CacheRevalidated) + .expect("failed to revalidate vcpu IoManager cache"); + }) + .map_err(|e| self.response_sender.send(VcpuResponse::Error(e))) + .expect("failed to revalidate vcpu IoManager cache"); + + StateMachine::next(Self::paused) + } + // Unhandled exit of the other end. + Err(_) => { + // Move to 'exited' state. + StateMachine::next(Self::exited) + } + } + } + + // This is the main loop of the `WaitingExit` state. + fn waiting_exit(&mut self) -> StateMachine { + // trigger vmm to stop machine + if let Err(e) = self.exit_evt.write(1) { + METRICS.vcpu.failures.inc(); + error!("Failed signaling vcpu exit event: {}", e); + } + + let mut state = StateMachine::next(Self::waiting_exit); + + match self.event_receiver.recv() { + Ok(VcpuEvent::Exit) => state = StateMachine::next(Self::exited), + Ok(_) => error!( + "wrong state received in waiting exit state on vcpu {}", + self.id + ), + Err(_) => { + error!( + "vcpu channel closed in waiting exit state on vcpu {}", + self.id + ); + state = StateMachine::next(Self::exited); + } + } + + state + } + + // This is the main loop of the `Exited` state. + fn exited(&mut self) -> StateMachine { + // State machine reached its end. + StateMachine::finish(Self::exited) + } +} + +impl Drop for Vcpu { + fn drop(&mut self) { + let _ = self.reset_thread_local_data(); + } +} + +#[cfg(test)] +pub mod tests { + use std::os::unix::io::AsRawFd; + use std::sync::mpsc::{channel, Receiver}; + use std::sync::Mutex; + + use arc_swap::ArcSwap; + use dbs_device::device_manager::IoManager; + use kvm_ioctls::Kvm; + use lazy_static::lazy_static; + + use super::*; + use crate::kvm_context::KvmContext; + + pub enum EmulationCase { + IoIn, + IoOut, + MmioRead, + MmioWrite, + Hlt, + Shutdown, + FailEntry, + InternalError, + Unknown, + SystemEvent(u32, u64), + Error(i32), + } + + lazy_static! { + pub static ref EMULATE_RES: Mutex = Mutex::new(EmulationCase::Unknown); + } + + impl Vcpu { + pub fn emulate(_fd: &VcpuFd) -> std::result::Result, kvm_ioctls::Error> { + let res = &*EMULATE_RES.lock().unwrap(); + match res { + EmulationCase::IoIn => Ok(VcpuExit::IoIn(0, &mut [])), + EmulationCase::IoOut => Ok(VcpuExit::IoOut(0, &[])), + EmulationCase::MmioRead => Ok(VcpuExit::MmioRead(0, &mut [])), + EmulationCase::MmioWrite => Ok(VcpuExit::MmioWrite(0, &[])), + EmulationCase::Hlt => Ok(VcpuExit::Hlt), + EmulationCase::Shutdown => Ok(VcpuExit::Shutdown), + EmulationCase::FailEntry => Ok(VcpuExit::FailEntry), + EmulationCase::InternalError => Ok(VcpuExit::InternalError), + EmulationCase::Unknown => Ok(VcpuExit::Unknown), + EmulationCase::SystemEvent(event_type, event_flags) => { + Ok(VcpuExit::SystemEvent(*event_type, *event_flags)) + } + EmulationCase::Error(e) => Err(kvm_ioctls::Error::new(*e)), + } + } + } + + #[cfg(target_arch = "x86_64")] + fn create_vcpu() -> (Vcpu, Receiver) { + // Call for kvm too frequently would cause error in some host kernel. + std::thread::sleep(std::time::Duration::from_millis(5)); + + let kvm = Kvm::new().unwrap(); + let vm = Arc::new(kvm.create_vm().unwrap()); + let kvm_context = KvmContext::new(Some(kvm.as_raw_fd())).unwrap(); + let vcpu_fd = Arc::new(vm.create_vcpu(0).unwrap()); + let io_manager = IoManagerCached::new(Arc::new(ArcSwap::new(Arc::new(IoManager::new())))); + let supported_cpuid = kvm_context + .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES) + .unwrap(); + let reset_event_fd = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let vcpu_state_event = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let (tx, rx) = channel(); + let time_stamp = TimestampUs::default(); + + let vcpu = Vcpu::new_x86_64( + 0, + vcpu_fd, + io_manager, + supported_cpuid, + reset_event_fd, + vcpu_state_event, + tx, + time_stamp, + false, + ) + .unwrap(); + + (vcpu, rx) + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_vcpu_run_emulation() { + let (mut vcpu, _) = create_vcpu(); + + // Io in + *(EMULATE_RES.lock().unwrap()) = EmulationCase::IoIn; + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + + // Io out + *(EMULATE_RES.lock().unwrap()) = EmulationCase::IoOut; + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + + // Mmio read + *(EMULATE_RES.lock().unwrap()) = EmulationCase::MmioRead; + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + + // Mmio write + *(EMULATE_RES.lock().unwrap()) = EmulationCase::MmioWrite; + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + + // KVM_EXIT_HLT signal + *(EMULATE_RES.lock().unwrap()) = EmulationCase::Hlt; + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + + // KVM_EXIT_SHUTDOWN signal + *(EMULATE_RES.lock().unwrap()) = EmulationCase::Shutdown; + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + + // KVM_EXIT_FAIL_ENTRY signal + *(EMULATE_RES.lock().unwrap()) = EmulationCase::FailEntry; + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + + // KVM_EXIT_INTERNAL_ERROR signal + *(EMULATE_RES.lock().unwrap()) = EmulationCase::InternalError; + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + + // KVM_SYSTEM_EVENT_RESET + *(EMULATE_RES.lock().unwrap()) = EmulationCase::SystemEvent(KVM_SYSTEM_EVENT_RESET, 0); + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Stopped))); + + // KVM_SYSTEM_EVENT_SHUTDOWN + *(EMULATE_RES.lock().unwrap()) = EmulationCase::SystemEvent(KVM_SYSTEM_EVENT_SHUTDOWN, 0); + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Stopped))); + + // Other system event + *(EMULATE_RES.lock().unwrap()) = EmulationCase::SystemEvent(0, 0); + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + + // Unknown exit reason + *(EMULATE_RES.lock().unwrap()) = EmulationCase::Unknown; + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + + // Error: EAGAIN + *(EMULATE_RES.lock().unwrap()) = EmulationCase::Error(libc::EAGAIN); + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + + // Error: EINTR + *(EMULATE_RES.lock().unwrap()) = EmulationCase::Error(libc::EINTR); + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Interrupted))); + + // other error + *(EMULATE_RES.lock().unwrap()) = EmulationCase::Error(libc::EINVAL); + let res = vcpu.run_emulation(); + assert!(matches!(res, Err(VcpuError::VcpuUnhandledKvmExit))); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_vcpu_check_io_port_info() { + let (vcpu, receiver) = create_vcpu(); + + // boot complete signal + let res = vcpu + .check_io_port_info( + MAGIC_IOPORT_SIGNAL_GUEST_BOOT_COMPLETE, + &[MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE], + ) + .unwrap(); + assert!(res); + + // debug info signal + let res = vcpu + .check_io_port_info(MAGIC_IOPORT_DEBUG_INFO, &[0, 0, 0, 0]) + .unwrap(); + assert!(res); + } +} diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs new file mode 100644 index 000000000000..618b282c96ce --- /dev/null +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -0,0 +1,5 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! The implementation of vcpu manager diff --git a/src/dragonball/src/vcpu/x86_64.rs b/src/dragonball/src/vcpu/x86_64.rs new file mode 100644 index 000000000000..738d574bba9b --- /dev/null +++ b/src/dragonball/src/vcpu/x86_64.rs @@ -0,0 +1,149 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::sync::mpsc::{channel, Sender}; +use std::sync::Arc; + +use dbs_arch::cpuid::{process_cpuid, VmSpec}; +use dbs_arch::gdt::gdt_entry; +use dbs_utils::time::TimestampUs; +use kvm_bindings::CpuId; +use kvm_ioctls::{VcpuFd, VmFd}; +use log::error; +use vm_memory::{Address, GuestAddress, GuestAddressSpace}; +use vmm_sys_util::eventfd::EventFd; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::metric::{IncMetric, METRICS}; +use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuError, VcpuStateEvent}; +use crate::vcpu::VcpuConfig; +use crate::IoManagerCached; + +impl Vcpu { + /// Constructs a new VCPU for `vm`. + /// + /// # Arguments + /// + /// * `id` - Represents the CPU number between [0, max vcpus). + /// * `vcpu_fd` - The kvm `VcpuFd` for the vcpu. + /// * `io_mgr` - The io-manager used to access port-io and mmio devices. + /// * `cpuid` - The `CpuId` listing the supported capabilities of this vcpu. + /// * `exit_evt` - An `EventFd` that will be written into when this vcpu + /// exits. + /// * `vcpu_state_event` - The eventfd which can notify vmm state of some + /// vcpu should change. + /// * `vcpu_state_sender` - The channel to send state change message from + /// vcpu thread to vmm thread. + /// * `create_ts` - A timestamp used by the vcpu to calculate its lifetime. + /// * `support_immediate_exit` - whether kvm used supports immediate_exit flag. + #[allow(clippy::too_many_arguments)] + pub fn new_x86_64( + id: u8, + vcpu_fd: Arc, + io_mgr: IoManagerCached, + cpuid: CpuId, + exit_evt: EventFd, + vcpu_state_event: EventFd, + vcpu_state_sender: Sender, + create_ts: TimestampUs, + support_immediate_exit: bool, + ) -> Result { + let (event_sender, event_receiver) = channel(); + let (response_sender, response_receiver) = channel(); + // Initially the cpuid per vCPU is the one supported by this VM. + Ok(Vcpu { + fd: vcpu_fd, + id, + io_mgr, + create_ts, + event_receiver, + event_sender: Some(event_sender), + response_receiver: Some(response_receiver), + response_sender, + vcpu_state_event, + vcpu_state_sender, + exit_evt, + support_immediate_exit, + cpuid, + }) + } + + /// Configures a x86_64 specific vcpu and should be called once per vcpu. + /// + /// # Arguments + /// + /// * `vm_config` - The machine configuration of this microvm needed for the CPUID configuration. + /// * `vm_fd` - The kvm `VmFd` for the virtual machine this vcpu will get attached to. + /// * `vm_memory` - The guest memory used by this microvm. + /// * `kernel_start_addr` - Offset from `guest_mem` at which the kernel starts. + /// * `pgtable_addr` - pgtable address for ap vcpu + pub fn configure( + &mut self, + vcpu_config: &VcpuConfig, + _vm_fd: &VmFd, + vm_as: &GuestAddressSpaceImpl, + kernel_start_addr: Option, + _pgtable_addr: Option, + ) -> Result<()> { + self.set_cpuid(vcpu_config)?; + + dbs_arch::regs::setup_msrs(&self.fd).map_err(VcpuError::MSRSConfiguration)?; + if let Some(start_addr) = kernel_start_addr { + dbs_arch::regs::setup_regs( + &self.fd, + start_addr.raw_value() as u64, + dbs_boot::layout::BOOT_STACK_POINTER, + dbs_boot::layout::BOOT_STACK_POINTER, + dbs_boot::layout::ZERO_PAGE_START, + ) + .map_err(VcpuError::REGSConfiguration)?; + dbs_arch::regs::setup_fpu(&self.fd).map_err(VcpuError::FPUConfiguration)?; + let gdt_table: [u64; dbs_boot::layout::BOOT_GDT_MAX as usize] = [ + gdt_entry(0, 0, 0), // NULL + gdt_entry(0xa09b, 0, 0xfffff), // CODE + gdt_entry(0xc093, 0, 0xfffff), // DATA + gdt_entry(0x808b, 0, 0xfffff), // TSS + ]; + let pgtable_addr = + dbs_boot::setup_identity_mapping(&*vm_as.memory()).map_err(VcpuError::PageTable)?; + dbs_arch::regs::setup_sregs( + &*vm_as.memory(), + &self.fd, + pgtable_addr, + &gdt_table, + dbs_boot::layout::BOOT_GDT_OFFSET, + dbs_boot::layout::BOOT_IDT_OFFSET, + ) + .map_err(VcpuError::SREGSConfiguration)?; + } + dbs_arch::interrupts::set_lint(&self.fd).map_err(VcpuError::LocalIntConfiguration)?; + + Ok(()) + } + + fn set_cpuid(&mut self, vcpu_config: &VcpuConfig) -> Result<()> { + let cpuid_vm_spec = VmSpec::new( + self.id, + vcpu_config.max_vcpu_count as u8, + vcpu_config.threads_per_core, + vcpu_config.cores_per_die, + vcpu_config.dies_per_socket, + vcpu_config.vpmu_feature, + ) + .map_err(VcpuError::CpuId)?; + process_cpuid(&mut self.cpuid, &cpuid_vm_spec).map_err(|e| { + METRICS.vcpu.filter_cpuid.inc(); + error!("Failure in configuring CPUID for vcpu {}: {:?}", self.id, e); + VcpuError::CpuId(e) + })?; + + self.fd + .set_cpuid2(&self.cpuid) + .map_err(VcpuError::SetSupportedCpusFailed) + } +} From 78c97187529b0bda4815bfb7b042a9000108238f Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 00:42:42 +0800 Subject: [PATCH 0073/1953] dragonball: add upcall support Upcall is a direct communication tool between VMM and guest developed upon vsock. It is used to implement device hotplug. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj Signed-off-by: Zizheng Bian --- src/dragonball/Cargo.toml | 14 ++-- src/dragonball/src/device_manager/mod.rs | 95 ++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 3440b4cff7de..3fa2dc5064c2 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -19,6 +19,7 @@ dbs-boot = "0.2.0" dbs-device = "0.1.0" dbs-interrupt = { version = "0.1.0", features = ["kvm-irq"] } dbs-legacy-devices = "0.1.0" +dbs-upcall = { version = "0.1.0", optional = true } dbs-utils = "0.1.0" dbs-virtio-devices = { version = "0.1.0", optional = true, features = ["virtio-mmio"] } kvm-bindings = "0.5.0" @@ -45,12 +46,13 @@ slog-async = "2.7.0" [features] atomic-guest-memory = [] +hotplug = ["dbs-upcall", "virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] [patch.'crates-io'] -dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } +dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } +dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } +dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 5691690ea8d0..a33e6e2a1247 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -29,6 +29,12 @@ use dbs_virtio_devices::{ VirtioDevice, }; +#[cfg(feature = "hotplug")] +use dbs_upcall::{ + MmioDevRequest, DevMgrRequest, DevMgrService, UpcallClient, UpcallClientError, + UpcallClientRequest, UpcallClientResponse +}; + use crate::address_space_manager::GuestAddressSpaceImpl; use crate::error::StartMicrovmError; use crate::resource_manager::ResourceManager; @@ -83,6 +89,11 @@ pub enum DeviceMgrError { /// Error from Virtio subsystem. #[error(transparent)] Virtio(virtio::Error), + + #[cfg(feature = "hotplug")] + /// Failed to hotplug the device. + #[error("failed to hotplug virtual device")] + HotplugDevice(#[source] UpcallClientError), } /// Specialized version of `std::result::Result` for device manager operations. @@ -188,6 +199,8 @@ pub struct DeviceOpContext { logger: slog::Logger, is_hotplug: bool, + #[cfg(feature = "hotplug")] + upcall_client: Option>>, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec>, } @@ -220,6 +233,8 @@ impl DeviceOpContext { address_space, logger, is_hotplug, + #[cfg(feature = "hotplug")] + upcall_client: None, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec::new(), } @@ -265,6 +280,86 @@ impl DeviceOpContext { } } +#[cfg(not(feature = "hotplug"))] +impl DeviceOpContext { + pub(crate) fn insert_hotplug_mmio_device( + &self, + _dev: &Arc, + _callback: Option<()>, + ) -> Result<()> { + Err(DeviceMgrError::InvalidOperation) + } + + pub(crate) fn remove_hotplug_mmio_device( + &self, + _dev: &Arc, + _callback: Option<()>, + ) -> Result<()> { + Err(DeviceMgrError::InvalidOperation) + } +} + +#[cfg(feature = "hotplug")] +impl DeviceOpContext { + fn call_hotplug_device( + &self, + req: DevMgrRequest, + callback: Option>, + ) -> Result<()> { + if let Some(upcall_client) = self.upcall_client.as_ref() { + if let Some(cb) = callback { + upcall_client + .send_request(UpcallClientRequest::DevMgr(req), cb) + .map_err(DeviceMgrError::HotplugDevice)?; + } else { + upcall_client + .send_request_without_result(UpcallClientRequest::DevMgr(req)) + .map_err(DeviceMgrError::HotplugDevice)?; + } + Ok(()) + } else { + Err(DeviceMgrError::InvalidOperation) + } + } + + pub(crate) fn insert_hotplug_mmio_device( + &self, + dev: &Arc, + callback: Option>, + ) -> Result<()> { + if !self.is_hotplug { + return Err(DeviceMgrError::InvalidOperation); + } + + let (mmio_base, mmio_size, mmio_irq) = DeviceManager::get_virtio_device_info(dev)?; + let req = DevMgrRequest::AddMmioDev(MmioDevRequest { + mmio_base, + mmio_size, + mmio_irq, + }); + + self.call_hotplug_device(req, callback) + } + + pub(crate) fn remove_hotplug_mmio_device( + &self, + dev: &Arc, + callback: Option>, + ) -> Result<()> { + if !self.is_hotplug { + return Err(DeviceMgrError::InvalidOperation); + } + let (mmio_base, mmio_size, mmio_irq) = DeviceManager::get_virtio_device_info(dev)?; + let req = DevMgrRequest::DelMmioDev(MmioDevRequest { + mmio_base, + mmio_size, + mmio_irq, + }); + + self.call_hotplug_device(req, callback) + } +} + /// Device manager for virtual machines, which manages all device for a virtual machine. pub struct DeviceManager { io_manager: Arc>, From 07f44c3e0aeb73818c036729f72edd641b3696e8 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 01:51:11 +0800 Subject: [PATCH 0074/1953] dragonball: add vcpu manager Manage vcpu related operations. Signed-off-by: Liu Jiang Signed-off-by: jingshan Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/src/device_manager/mod.rs | 15 +- src/dragonball/src/vcpu/mod.rs | 1 + src/dragonball/src/vcpu/vcpu_impl.rs | 8 +- src/dragonball/src/vcpu/vcpu_manager.rs | 1035 +++++++++++++++++++++- src/dragonball/src/vm/mod.rs | 76 ++ 5 files changed, 1116 insertions(+), 19 deletions(-) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index a33e6e2a1247..4a1953113652 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -446,7 +446,7 @@ impl DeviceManager { self.set_guest_kernel_log_stream(dmesg_fifo) .map_err(|_| StartMicrovmError::EventFd)?; - slog::info!(self.logger, "init console path: {:?}", com1_sock_path); + info!(self.logger, "init console path: {:?}", com1_sock_path); if let Some(path) = com1_sock_path { if let Some(legacy_manager) = self.legacy_manager.as_ref() { let com1 = legacy_manager.get_com1_serial(); @@ -482,19 +482,6 @@ impl DeviceManager { Ok(()) } - /// Restore legacy devices - pub fn restore_legacy_devices( - &mut self, - dmesg_fifo: Option>, - com1_sock_path: Option, - ) -> std::result::Result<(), StartMicrovmError> { - self.set_guest_kernel_log_stream(dmesg_fifo) - .map_err(|_| StartMicrovmError::EventFd)?; - slog::info!(self.logger, "restore console path: {:?}", com1_sock_path); - // TODO: restore console - Ok(()) - } - /// Reset the console into canonical mode. pub fn reset_console(&self) -> Result<()> { self.con_manager.reset_console() diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs index 82b58e9ea953..d1075e734d38 100644 --- a/src/dragonball/src/vcpu/mod.rs +++ b/src/dragonball/src/vcpu/mod.rs @@ -5,6 +5,7 @@ mod sm; pub mod vcpu_impl; +pub mod vcpu_manager; #[cfg(target_arch = "x86_64")] use dbs_arch::cpuid::VpmuFeatureLevel; diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index cb7e83af12b6..7c39ca280573 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -19,7 +19,7 @@ use dbs_utils::time::TimestampUs; use kvm_bindings::{KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN}; use kvm_ioctls::{VcpuExit, VcpuFd}; use libc::{c_int, c_void, siginfo_t}; -use log::{error, info, warn}; +use log::{error, info}; use seccompiler::{apply_filter, BpfProgram, Error as SecError}; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::{register_signal_handler, Killable}; @@ -216,8 +216,8 @@ pub enum VcpuResponse { /// List of events that the vcpu_state_sender can send. pub enum VcpuStateEvent { - /// For Hotplug - Hotplug((bool, u32)), + /// (result, response) for hotplug, result 0 means failure, 1 means success. + Hotplug((i32, u32)), } /// Wrapper over vCPU that hides the underlying interactions with the vCPU thread. @@ -541,7 +541,7 @@ impl Vcpu { MAGIC_IOPORT_DEBUG_INFO => { if data.len() == 4 { let data = unsafe { std::ptr::read(data.as_ptr() as *const u32) }; - warn!("KDBG: guest kernel debug info: 0x{:x}", data); + log::warn!("KDBG: guest kernel debug info: 0x{:x}", data); checked = true; } } diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 618b282c96ce..f548035b2c8e 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -1,5 +1,1038 @@ // Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. // +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. +// +// Copyright © 2019 Intel Corporation // SPDX-License-Identifier: Apache-2.0 -//! The implementation of vcpu manager +//! vCPU manager to enable bootstrap and CPU hotplug. +use std::io; +use std::os::unix::io::AsRawFd; +use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender}; +use std::sync::{Arc, Barrier, Mutex, RwLock}; +use std::time::Duration; + +#[cfg(feature = "hotplug")] +use dbs_upcall::{DevMgrService, UpcallClient}; +use dbs_utils::epoll_manager::{EpollManager, EventOps, EventSet, Events, MutEventSubscriber}; +use dbs_utils::time::TimestampUs; +use kvm_ioctls::{Cap, VcpuFd, VmFd}; +use log::{debug, error, info}; +use seccompiler::{apply_filter, BpfProgram, Error as SecError}; +use vm_memory::GuestAddress; +use vmm_sys_util::eventfd::EventFd; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::api::v1::InstanceInfo; +use crate::kvm_context::KvmContext; +use crate::vcpu::vcpu_impl::{ + Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResponse, VcpuStateEvent, +}; +use crate::vcpu::VcpuConfig; +use crate::vm::VmConfigInfo; +use crate::IoManagerCached; + +/// the timeout for communication with vcpu threads +const CPU_RECV_TIMEOUT_MS: u64 = 1000; + +/// vCPU manager error +#[derive(Debug, thiserror::Error)] +pub enum VcpuManagerError { + /// IO errors in vCPU manager + #[error("IO errors in vCPU manager {0}")] + VcpuIO(#[source] io::Error), + + /// vCPU manager is not initialized + #[error("vcpu manager is not initialized")] + VcpuManagerNotInitialized, + + /// Expected vcpu exceed max count + #[error("expected vcpu exceed max count")] + ExpectedVcpuExceedMax, + + /// vCPU not found + #[error("vcpu not found {0}")] + VcpuNotFound(u8), + + /// Cannot recv vCPU thread tid + #[error("cannot get vCPU thread id")] + VcpuGettid, + + /// vCPU pause failed. + #[error("failure while pausing vCPU thread")] + VcpuPause, + + /// vCPU resume failed. + #[error("failure while resuming vCPU thread")] + VcpuResume, + + /// vCPU save failed. + #[error("failure while save vCPU state")] + VcpuSave, + + /// Vcpu is in unexpected state. + #[error("Vcpu is in unexpected state")] + UnexpectedVcpuResponse, + + /// Vcpu not create + #[error("Vcpu is not create")] + VcpuNotCreate, + + /// The number of max_vcpu reached kvm's limitation + #[error("specified vcpu count {0} is greater than max allowed count {1} by kvm")] + MaxVcpuLimitation(u8, usize), + + /// Revalidate vcpu IoManager cache failed. + #[error("failure while revalidating vcpu IoManager cache")] + VcpuRevalidateCache, + + /// Event fd is already set so there could be some problem in the VMM if we try to reset it. + #[error("Event fd is already set for the vcpu")] + EventAlreadyExist, + + /// Response channel error + #[error("Response channel error: {0}")] + VcpuResponseChannel(RecvError), + + /// Vcpu response timeout + #[error("Vcpu response timeout: {0}")] + VcpuResponseTimeout(RecvTimeoutError), + + /// Cannot build seccomp filters. + #[error("failure while configuring seccomp filters: {0}")] + SeccompFilters(#[source] seccompiler::Error), + + /// Cannot send event to vCPU. + #[error("failure while sending message to vCPU thread: {0}")] + VcpuEvent(#[source] VcpuError), + + /// vCPU Error + #[error("vcpu internal error: {0}")] + Vcpu(#[source] VcpuError), + + #[cfg(feature = "hotplug")] + /// vCPU resize error + #[error("resize vcpu error: {0}")] + VcpuResize(#[source] VcpuResizeError), + + /// Kvm Ioctl Error + #[error("failure in issuing KVM ioctl command: {0}")] + Kvm(#[source] kvm_ioctls::Error), +} + +#[cfg(feature = "hotplug")] +/// Errror associated with resize instance +#[derive(Debug, thiserror::Error)] +pub enum VcpuResizeError { + /// vcpu is in hotplug process + #[error("vcpu is in hotplug process")] + VcpuIsHotplugging, + + /// Cannot update the configuration of the microvm pre boot. + #[error("resize vcpu operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// Expected vcpu exceed max count + #[error("expected vcpu exceed max count")] + ExpectedVcpuExceedMax, + + /// vcpu 0 can't be removed + #[error("vcpu 0 can't be removed")] + Vcpu0CanNotBeRemoved, + + /// Lack removable vcpu + #[error("Removable vcpu not enough, removable vcpu num: {0}, number to remove: {1}, present vcpu count {2}")] + LackRemovableVcpus(u16, u16, u16), + + /// Cannot update the configuration by upcall channel. + #[error("cannot update the configuration by upcall channel: {0}")] + Upcall(#[source] dbs_upcall::UpcallClientError), +} + +/// Result for vCPU manager operations +pub type Result = std::result::Result; + +#[derive(Debug, PartialEq, Copy, Clone)] +enum VcpuAction { + None, + Hotplug, + Hotunplug, +} + +/// Infos related to per vcpu +#[derive(Default)] +pub(crate) struct VcpuInfo { + pub(crate) vcpu: Option, + vcpu_fd: Option>, + handle: Option, + tid: u32, +} + +impl std::fmt::Debug for VcpuInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VcpuInfo") + .field("vcpu", &self.vcpu.is_some()) + .field("vcpu_fd", &self.vcpu_fd.is_some()) + .field("handle", &self.handle.is_some()) + .field("tid", &self.tid) + .finish() + } +} + +/// Manage all vcpu related actions +pub struct VcpuManager { + pub(crate) vcpu_infos: Vec, + vcpu_config: VcpuConfig, + vcpu_seccomp_filter: BpfProgram, + vcpu_state_event: EventFd, + vcpu_state_sender: Sender, + support_immediate_exit: bool, + + // The purpose of putting a reference of IoManager here is to simplify the + // design of the API when creating vcpus, and the IoManager has numerous OS + // resources that need to be released when vmm exits. However, since + // VcpuManager is referenced by VcpuEpollHandler and VcpuEpollHandler will + // not be released when vmm is closed, we need to release io manager + // manually when we exit all vcpus. + io_manager: Option, + shared_info: Arc>, + vm_as: GuestAddressSpaceImpl, + pub(crate) vm_fd: Arc, + + action_sycn_tx: Option>, + vcpus_in_action: (VcpuAction, Vec), + pub(crate) reset_event_fd: Option, + + #[cfg(feature = "hotplug")] + upcall_channel: Option>>, + + // X86 specific fields. + #[cfg(target_arch = "x86_64")] + pub(crate) supported_cpuid: kvm_bindings::CpuId, +} + +#[allow(clippy::too_many_arguments)] +impl VcpuManager { + /// Get a new VcpuManager instance + pub fn new( + vm_fd: Arc, + kvm_context: &KvmContext, + vm_config_info: &VmConfigInfo, + vm_as: GuestAddressSpaceImpl, + vcpu_seccomp_filter: BpfProgram, + shared_info: Arc>, + io_manager: IoManagerCached, + epoll_manager: EpollManager, + ) -> Result>> { + let support_immediate_exit = kvm_context.kvm().check_extension(Cap::ImmediateExit); + let max_vcpu_count = vm_config_info.max_vcpu_count; + let kvm_max_vcpu_count = kvm_context.get_max_vcpus(); + + // check the max vcpu count in kvm. max_vcpu_count is u8 and kvm_context.get_max_vcpus() + // returns usize, so convert max_vcpu_count to usize instead of converting kvm max vcpu to + // u8, to avoid wraping usize. Otherwise if kvm_max_vcpu_count is greater than 255, it'll + // be casted into a smaller number. + if max_vcpu_count as usize > kvm_max_vcpu_count { + error!( + "vcpu_manager: specified vcpu count {} is greater than max allowed count {} by kvm", + max_vcpu_count, kvm_max_vcpu_count + ); + return Err(VcpuManagerError::MaxVcpuLimitation( + max_vcpu_count, + kvm_max_vcpu_count, + )); + } + + let mut vcpu_infos = Vec::with_capacity(max_vcpu_count.into()); + vcpu_infos.resize_with(max_vcpu_count.into(), Default::default); + + let (tx, rx) = channel(); + let vcpu_state_event = + EventFd::new(libc::EFD_NONBLOCK).map_err(VcpuManagerError::VcpuIO)?; + let vcpu_state_event2 = vcpu_state_event + .try_clone() + .map_err(VcpuManagerError::VcpuIO)?; + + #[cfg(target_arch = "x86_64")] + let supported_cpuid = kvm_context + .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES) + .map_err(VcpuManagerError::Kvm)?; + #[cfg(target_arch = "x86_64")] + let vpmu_feature_level = match vm_config_info.vpmu_feature { + 1 => dbs_arch::cpuid::VpmuFeatureLevel::LimitedlyEnabled, + 2 => dbs_arch::cpuid::VpmuFeatureLevel::FullyEnabled, + _ => dbs_arch::cpuid::VpmuFeatureLevel::Disabled, + }; + + let vcpu_manager = Arc::new(Mutex::new(VcpuManager { + vcpu_infos, + vcpu_config: VcpuConfig { + boot_vcpu_count: vm_config_info.vcpu_count, + max_vcpu_count, + threads_per_core: vm_config_info.cpu_topology.threads_per_core, + cores_per_die: vm_config_info.cpu_topology.cores_per_die, + dies_per_socket: vm_config_info.cpu_topology.dies_per_socket, + sockets: vm_config_info.cpu_topology.sockets, + #[cfg(target_arch = "x86_64")] + vpmu_feature: vpmu_feature_level, + }, + vcpu_seccomp_filter, + vcpu_state_event, + vcpu_state_sender: tx, + support_immediate_exit, + io_manager: Some(io_manager), + shared_info, + vm_as, + vm_fd, + action_sycn_tx: None, + vcpus_in_action: (VcpuAction::None, Vec::new()), + reset_event_fd: None, + #[cfg(feature = "hotplug")] + upcall_channel: None, + #[cfg(target_arch = "x86_64")] + supported_cpuid, + })); + + let handler = Box::new(VcpuEpollHandler { + vcpu_manager: vcpu_manager.clone(), + eventfd: vcpu_state_event2, + rx, + }); + epoll_manager.add_subscriber(handler); + + Ok(vcpu_manager) + } + + /// get vcpu instances in vcpu manager + pub fn vcpus(&self) -> Vec<&Vcpu> { + let mut vcpus = Vec::new(); + for vcpu_info in &self.vcpu_infos { + if let Some(vcpu) = &vcpu_info.vcpu { + vcpus.push(vcpu); + } + } + vcpus + } + + /// get vcpu instances in vcpu manager as mut + pub fn vcpus_mut(&mut self) -> Vec<&mut Vcpu> { + let mut vcpus = Vec::new(); + for vcpu_info in &mut self.vcpu_infos { + if let Some(vcpu) = &mut vcpu_info.vcpu { + vcpus.push(vcpu); + } + } + vcpus + } + + /// add reset event fd for each vcpu, if the reset_event_fd is already set, error will be returned. + pub fn set_reset_event_fd(&mut self, reset_event_fd: EventFd) -> Result<()> { + if self.reset_event_fd.is_some() { + return Err(VcpuManagerError::EventAlreadyExist); + } + self.reset_event_fd = Some(reset_event_fd); + Ok(()) + } + + /// create default num of vcpus for bootup + pub fn create_boot_vcpus( + &mut self, + request_ts: TimestampUs, + entry_addr: GuestAddress, + ) -> Result<()> { + info!("create boot vcpus"); + self.create_vcpus( + self.vcpu_config.boot_vcpu_count, + Some(request_ts), + Some(entry_addr), + )?; + + Ok(()) + } + + /// start the boot vcpus + pub fn start_boot_vcpus(&mut self, vmm_seccomp_filter: BpfProgram) -> Result<()> { + info!("start boot vcpus"); + self.start_vcpus(self.vcpu_config.boot_vcpu_count, vmm_seccomp_filter, true)?; + + Ok(()) + } + + /// create a specified num of vcpu + /// note: we can't create vcpus again until the previously created vcpus are + /// started + pub fn create_vcpus( + &mut self, + vcpu_count: u8, + request_ts: Option, + entry_addr: Option, + ) -> Result> { + info!("create vcpus"); + if vcpu_count > self.vcpu_config.max_vcpu_count { + return Err(VcpuManagerError::ExpectedVcpuExceedMax); + } + + let request_ts = request_ts.unwrap_or_default(); + let mut created_cpus = Vec::new(); + for cpu_id in self.calculate_available_vcpus(vcpu_count) { + self.create_vcpu(cpu_id, request_ts.clone(), entry_addr)?; + created_cpus.push(cpu_id); + } + + Ok(created_cpus) + } + + /// start a specified num of vcpu + pub fn start_vcpus( + &mut self, + vcpu_count: u8, + vmm_seccomp_filter: BpfProgram, + need_resume: bool, + ) -> Result<()> { + info!("start vcpus"); + Vcpu::register_kick_signal_handler(); + self.activate_vcpus(vcpu_count, need_resume)?; + + // Load seccomp filters for the VMM thread. + // Execution panics if filters cannot be loaded, use --seccomp-level=0 if skipping filters + // altogether is the desired behaviour. + if let Err(e) = apply_filter(&vmm_seccomp_filter) { + if !matches!(e, SecError::EmptyFilter) { + return Err(VcpuManagerError::SeccompFilters(e)); + } + } + + Ok(()) + } + + /// pause all vcpus + pub fn pause_all_vcpus(&mut self) -> Result<()> { + self.pause_vcpus(&self.present_vcpus()) + } + + /// resume all vcpus + pub fn resume_all_vcpus(&mut self) -> Result<()> { + self.resume_vcpus(&self.present_vcpus()) + } + + /// exit all vcpus, and never restart again + pub fn exit_all_vcpus(&mut self) -> Result<()> { + self.exit_vcpus(&self.present_vcpus())?; + // clear all vcpu infos + self.vcpu_infos.clear(); + // release io manager's reference manually + self.io_manager.take(); + + Ok(()) + } + + /// revalidate IoManager cache of all vcpus + pub fn revalidate_all_vcpus_cache(&mut self) -> Result<()> { + self.revalidate_vcpus_cache(&self.present_vcpus()) + } + + /// return all present vcpus + pub fn present_vcpus(&self) -> Vec { + self.vcpu_infos + .iter() + .enumerate() + .filter(|(_i, info)| info.handle.is_some()) + .map(|(i, _info)| i as u8) + .collect() + } + + /// Get available vcpus to create with target vcpu_count + /// Argument: + /// * vcpu_count: target vcpu_count online in VcpuManager. + /// Return: + /// * return available vcpu ids to create vcpu . + fn calculate_available_vcpus(&self, vcpu_count: u8) -> Vec { + let present_vcpus_count = self.present_vcpus_count(); + let mut available_vcpus = Vec::new(); + + if present_vcpus_count < vcpu_count { + let mut size = vcpu_count - present_vcpus_count; + for cpu_id in 0..self.vcpu_config.max_vcpu_count { + let info = &self.vcpu_infos[cpu_id as usize]; + if info.handle.is_none() { + available_vcpus.push(cpu_id); + size -= 1; + if size == 0 { + break; + } + } + } + } + + available_vcpus + } + + /// Present vcpus count + fn present_vcpus_count(&self) -> u8 { + self.vcpu_infos + .iter() + .fold(0, |sum, info| sum + info.handle.is_some() as u8) + } + + /// Configure single vcpu + fn configure_single_vcpu( + &mut self, + entry_addr: Option, + vcpu: &mut Vcpu, + ) -> std::result::Result<(), VcpuError> { + vcpu.configure( + &self.vcpu_config, + &self.vm_fd, + &self.vm_as, + entry_addr, + None, + ) + } + + fn create_vcpu( + &mut self, + cpu_index: u8, + request_ts: TimestampUs, + entry_addr: Option, + ) -> Result<()> { + info!("creating vcpu {}", cpu_index); + if self.vcpu_infos.get(cpu_index as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(cpu_index)); + } + // We will reuse the kvm's vcpufd after first creation, for we can't + // create vcpufd with same id in one kvm instance. + let kvm_vcpu = match &self.vcpu_infos[cpu_index as usize].vcpu_fd { + Some(vcpu_fd) => vcpu_fd.clone(), + None => { + let vcpu_fd = Arc::new( + self.vm_fd + .create_vcpu(cpu_index as u64) + .map_err(VcpuError::VcpuFd) + .map_err(VcpuManagerError::Vcpu)?, + ); + self.vcpu_infos[cpu_index as usize].vcpu_fd = Some(vcpu_fd.clone()); + vcpu_fd + } + }; + + let mut vcpu = self.create_vcpu_arch(cpu_index, kvm_vcpu, request_ts)?; + self.configure_single_vcpu(entry_addr, &mut vcpu) + .map_err(VcpuManagerError::Vcpu)?; + self.vcpu_infos[cpu_index as usize].vcpu = Some(vcpu); + + Ok(()) + } + + fn start_vcpu(&mut self, cpu_index: u8, barrier: Arc) -> Result<()> { + info!("starting vcpu {}", cpu_index); + if self.vcpu_infos.get(cpu_index as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(cpu_index)); + } + if let Some(vcpu) = self.vcpu_infos[cpu_index as usize].vcpu.take() { + let handle = vcpu + .start_threaded(self.vcpu_seccomp_filter.clone(), barrier) + .map_err(VcpuManagerError::Vcpu)?; + self.vcpu_infos[cpu_index as usize].handle = Some(handle); + Ok(()) + } else { + Err(VcpuManagerError::VcpuNotCreate) + } + } + + fn get_vcpus_tid(&mut self, cpu_indexes: &[u8]) -> Result<()> { + for cpu_id in cpu_indexes { + if self.vcpu_infos.get(*cpu_id as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + if let Some(handle) = &self.vcpu_infos[*cpu_id as usize].handle { + handle + .send_event(VcpuEvent::Gettid) + .map_err(VcpuManagerError::VcpuEvent)?; + } else { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + } + + for cpu_id in cpu_indexes { + if let Some(handle) = &self.vcpu_infos[*cpu_id as usize].handle { + match handle + .response_receiver() + .recv_timeout(Duration::from_millis(CPU_RECV_TIMEOUT_MS)) + { + Ok(VcpuResponse::Tid(_, id)) => self.vcpu_infos[*cpu_id as usize].tid = id, + Err(e) => { + error!("vCPU get tid error! {:?}", e); + return Err(VcpuManagerError::VcpuGettid); + } + _ => { + error!("vCPU get tid error!"); + return Err(VcpuManagerError::VcpuGettid); + } + } + } else { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + } + + // Save all vCPU thread ID to self.shared_info + let tids: Vec<(u8, u32)> = cpu_indexes + .iter() + .map(|cpu_id| (*cpu_id, self.vcpu_infos[*cpu_id as usize].tid)) + .collect(); + + // Append the new started vcpu thread IDs into self.shared_info + self.shared_info + .write() + .unwrap() + .tids + .extend_from_slice(&tids[..]); + + Ok(()) + } + + fn revalidate_vcpus_cache(&mut self, cpu_indexes: &[u8]) -> Result<()> { + for cpu_id in cpu_indexes { + if self.vcpu_infos.get(*cpu_id as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + if let Some(handle) = &self.vcpu_infos[*cpu_id as usize].handle { + handle + .send_event(VcpuEvent::RevalidateCache) + .map_err(VcpuManagerError::VcpuEvent)?; + } else { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + } + + Ok(()) + } + + fn pause_vcpus(&mut self, cpu_indexes: &[u8]) -> Result<()> { + for cpu_id in cpu_indexes { + if self.vcpu_infos.get(*cpu_id as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + if let Some(handle) = &self.vcpu_infos[*cpu_id as usize].handle { + handle + .send_event(VcpuEvent::Pause) + .map_err(VcpuManagerError::VcpuEvent)?; + } else { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + } + + Ok(()) + } + + fn resume_vcpus(&mut self, cpu_indexes: &[u8]) -> Result<()> { + for cpu_id in cpu_indexes { + if self.vcpu_infos.get(*cpu_id as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + if let Some(handle) = &self.vcpu_infos[*cpu_id as usize].handle { + handle + .send_event(VcpuEvent::Resume) + .map_err(VcpuManagerError::VcpuEvent)?; + } else { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + } + + Ok(()) + } + + // exit vcpus and notify the vmm exit event + fn exit_vcpus(&mut self, cpu_indexes: &[u8]) -> Result<()> { + info!("exiting vcpus {:?}", cpu_indexes); + for cpu_id in cpu_indexes { + if self.vcpu_infos.get(*cpu_id as usize).is_none() { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + if let Some(handle) = &self.vcpu_infos[*cpu_id as usize].handle { + handle + .send_event(VcpuEvent::Exit) + .map_err(VcpuManagerError::VcpuEvent)?; + } else { + return Err(VcpuManagerError::VcpuNotFound(*cpu_id)); + } + } + + for cpu_id in cpu_indexes { + let handle = self.vcpu_infos[*cpu_id as usize].handle.take().unwrap(); + handle + .join_vcpu_thread() + .map_err(|e| error!("vcpu exit error! {:?}", e)) + .ok(); + } + + let tids: &mut Vec<(u8, u32)> = &mut self + .shared_info + .write() + .expect( + "Failed to stop vcpus because shared info couldn't be written due to poisoned lock", + ) + .tids; + + // Here's a trick: since we always stop the vcpus started latest, + // thus it's ok here to remove the stopped vcpus from end to head. + tids.truncate(tids.len() - cpu_indexes.len()); + + Ok(()) + } + + fn stop_vcpus_in_action(&mut self) -> Result<()> { + let vcpus_in_action = self.vcpus_in_action.1.clone(); + self.exit_vcpus(&vcpus_in_action) + } + + fn activate_vcpus(&mut self, vcpu_count: u8, need_resume: bool) -> Result> { + let present_vcpus_count = self.present_vcpus_count(); + if vcpu_count > self.vcpu_config.max_vcpu_count { + return Err(VcpuManagerError::ExpectedVcpuExceedMax); + } else if vcpu_count < present_vcpus_count { + return Ok(Vec::new()); + } + + let available_vcpus = self.calculate_available_vcpus(vcpu_count); + let barrier = Arc::new(Barrier::new(available_vcpus.len() + 1_usize)); + for cpu_id in available_vcpus.iter() { + self.start_vcpu(*cpu_id, barrier.clone())?; + } + barrier.wait(); + + self.get_vcpus_tid(&available_vcpus)?; + if need_resume { + self.resume_vcpus(&available_vcpus)?; + } + + Ok(available_vcpus) + } + + fn sync_action_finish(&mut self, got_error: bool) { + if let Some(tx) = self.action_sycn_tx.take() { + if let Err(e) = tx.send(got_error) { + debug!("cpu sync action send to closed channel {}", e); + } + } + } + + fn set_vcpus_action(&mut self, action: VcpuAction, vcpus: Vec) { + self.vcpus_in_action = (action, vcpus); + } + + fn get_vcpus_action(&self) -> VcpuAction { + self.vcpus_in_action.0 + } +} + +#[cfg(target_arch = "x86_64")] +impl VcpuManager { + fn create_vcpu_arch( + &self, + cpu_index: u8, + vcpu_fd: Arc, + request_ts: TimestampUs, + ) -> Result { + // It's safe to unwrap because guest_kernel always exist until vcpu manager done + Vcpu::new_x86_64( + cpu_index, + vcpu_fd, + // safe to unwrap + self.io_manager.as_ref().unwrap().clone(), + self.supported_cpuid.clone(), + self.reset_event_fd.as_ref().unwrap().try_clone().unwrap(), + self.vcpu_state_event.try_clone().unwrap(), + self.vcpu_state_sender.clone(), + request_ts, + self.support_immediate_exit, + ) + .map_err(VcpuManagerError::Vcpu) + } +} + +#[cfg(target_arch = "aarch64")] +impl VcpuManager { + // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before + // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP + // was already initialized. + // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. + fn create_vcpu_arch( + &self, + cpu_index: u8, + vcpu_fd: Arc, + request_ts: TimestampUs, + ) -> Result { + Vcpu::new_aarch64( + cpu_index, + vcpu_fd, + // safe to unwrap + self.io_manager.as_ref().unwrap().clone(), + self.reset_event_fd.as_ref().unwrap().try_clone().unwrap(), + self.vcpu_state_event.try_clone().unwrap(), + self.vcpu_state_sender.clone(), + request_ts.clone(), + self.support_immediate_exit, + ) + .map_err(VcpuManagerError::Vcpu) + } +} + +#[cfg(feature = "hotplug")] +mod hotplug { + use std::cmp::Ordering; + + use super::*; + #[cfg(not(test))] + use dbs_upcall::CpuDevRequest; + use dbs_upcall::{DevMgrRequest, DevMgrResponse, UpcallClientRequest, UpcallClientResponse}; + + #[cfg(all(target_arch = "x86_64", not(test)))] + use dbs_boot::mptable::APIC_VERSION; + #[cfg(all(target_arch = "aarch64", not(test)))] + const APIC_VERSION: u8 = 0; + + impl VcpuManager { + /// add upcall channel for vcpu manager + pub fn set_upcall_channel( + &mut self, + upcall_channel: Option>>, + ) { + self.upcall_channel = upcall_channel; + } + + /// resize the count of vcpu in runtime + pub fn resize_vcpu( + &mut self, + vcpu_count: u8, + sync_tx: Option>, + ) -> std::result::Result<(), VcpuManagerError> { + if self.get_vcpus_action() != VcpuAction::None { + return Err(VcpuManagerError::VcpuResize( + VcpuResizeError::VcpuIsHotplugging, + )); + } + self.action_sycn_tx = sync_tx; + + if let Some(upcall) = self.upcall_channel.clone() { + let now_vcpu = self.present_vcpus_count(); + info!("resize vcpu: now: {}, desire: {}", now_vcpu, vcpu_count); + match vcpu_count.cmp(&now_vcpu) { + Ordering::Equal => { + info!("resize vcpu: no need to resize"); + self.sync_action_finish(false); + Ok(()) + } + Ordering::Greater => self.do_add_vcpu(vcpu_count, upcall), + Ordering::Less => self.do_del_vcpu(vcpu_count, upcall), + } + } else { + Err(VcpuManagerError::VcpuResize( + VcpuResizeError::UpdateNotAllowedPostBoot, + )) + } + } + + fn do_add_vcpu( + &mut self, + vcpu_count: u8, + upcall_client: Arc>, + ) -> std::result::Result<(), VcpuManagerError> { + info!("resize vcpu: add"); + if vcpu_count > self.vcpu_config.max_vcpu_count { + return Err(VcpuManagerError::VcpuResize( + VcpuResizeError::ExpectedVcpuExceedMax, + )); + } + + let created_vcpus = self.create_vcpus(vcpu_count, None, None)?; + let cpu_ids = self.activate_vcpus(vcpu_count, true).map_err(|e| { + // we need to rollback when activate vcpu error + error!("activate vcpu error, rollback! {:?}", e); + let activated_vcpus: Vec = created_vcpus + .iter() + .filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some()) + .copied() + .collect(); + if let Err(e) = self.exit_vcpus(&activated_vcpus) { + error!("try to rollback error, stop_vcpu: {:?}", e); + } + e + })?; + + let mut cpu_ids_array = [0u8; (u8::MAX as usize) + 1]; + cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]); + let req = DevMgrRequest::AddVcpu(CpuDevRequest { + count: cpu_ids.len() as u8, + apic_ids: cpu_ids_array, + apic_ver: APIC_VERSION, + }); + self.send_upcall_action(upcall_client, req)?; + + self.set_vcpus_action(VcpuAction::Hotplug, cpu_ids); + + Ok(()) + } + + fn do_del_vcpu( + &mut self, + vcpu_count: u8, + upcall_client: Arc>, + ) -> std::result::Result<(), VcpuManagerError> { + info!("resize vcpu: delete"); + if vcpu_count == 0 { + return Err(VcpuManagerError::VcpuResize( + VcpuResizeError::Vcpu0CanNotBeRemoved, + )); + } + + let mut cpu_ids = self.calculate_removable_vcpus(); + let cpu_num_to_be_del = (self.present_vcpus_count() - vcpu_count) as usize; + if cpu_num_to_be_del >= cpu_ids.len() { + return Err(VcpuManagerError::VcpuResize( + VcpuResizeError::LackRemovableVcpus( + cpu_ids.len() as u16, + cpu_num_to_be_del as u16, + self.present_vcpus_count() as u16, + ), + )); + } + + cpu_ids.reverse(); + cpu_ids.truncate(cpu_num_to_be_del); + + let mut cpu_ids_array = [0u8; 256]; + cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]); + let req = DevMgrRequest::DelVcpu(CpuDevRequest { + count: cpu_num_to_be_del as u8, + apic_ids: cpu_ids_array, + apic_ver: APIC_VERSION, + }); + self.send_upcall_action(upcall_client, req)?; + + self.set_vcpus_action(VcpuAction::Hotunplug, cpu_ids); + + Ok(()) + } + + #[cfg(test)] + fn send_upcall_action( + &self, + _upcall_client: Arc>, + _request: DevMgrRequest, + ) -> std::result::Result<(), VcpuManagerError> { + Ok(()) + } + + #[cfg(not(test))] + fn send_upcall_action( + &self, + upcall_client: Arc>, + request: DevMgrRequest, + ) -> std::result::Result<(), VcpuManagerError> { + let vcpu_state_event = self.vcpu_state_event.try_clone().unwrap(); + let vcpu_state_sender = self.vcpu_state_sender.clone(); + + upcall_client + .send_request( + UpcallClientRequest::DevMgr(request), + Box::new(move |result| match result { + UpcallClientResponse::DevMgr(response) => { + if let DevMgrResponse::CpuDev(resp) = response { + vcpu_state_sender + .send(VcpuStateEvent::Hotplug(( + resp.result, + resp.info.apic_id_index, + ))) + .unwrap(); + vcpu_state_event.write(1).unwrap(); + } + } + UpcallClientResponse::UpcallReset => { + vcpu_state_sender + .send(VcpuStateEvent::Hotplug((0, 0))) + .unwrap(); + vcpu_state_event.write(1).unwrap(); + } + #[cfg(test)] + UpcallClientResponse::FakeResponse => { + panic!("shouldn't happen"); + } + }), + ) + .map_err(VcpuResizeError::Upcall) + .map_err(VcpuManagerError::VcpuResize) + } + + /// Get removable vcpus. + /// Return: + /// * return removable vcpu_id with cascade order. + fn calculate_removable_vcpus(&self) -> Vec { + self.present_vcpus() + } + } +} + +struct VcpuEpollHandler { + vcpu_manager: Arc>, + eventfd: EventFd, + rx: Receiver, +} + +impl VcpuEpollHandler { + fn process_cpu_state_event(&mut self, _ops: &mut EventOps) { + // It's level triggered, so it's safe to ignore the result. + let _ = self.eventfd.read(); + while let Ok(event) = self.rx.try_recv() { + match event { + VcpuStateEvent::Hotplug((success, cpu_count)) => { + info!("get vcpu event, cpu_index {}", cpu_count); + self.process_cpu_action(success != 0, cpu_count); + } + } + } + } + + fn process_cpu_action(&self, success: bool, _cpu_index: u32) { + let mut vcpu_manager = self.vcpu_manager.lock().unwrap(); + if success { + match vcpu_manager.get_vcpus_action() { + VcpuAction::Hotplug => { + // Notify hotplug success + vcpu_manager.sync_action_finish(false); + } + VcpuAction::Hotunplug => { + if let Err(e) = vcpu_manager.stop_vcpus_in_action() { + error!("stop vcpus in action error: {:?}", e); + } + // notify hotunplug success + vcpu_manager.sync_action_finish(false); + } + VcpuAction::None => { + error!("cannot be here"); + } + }; + vcpu_manager.set_vcpus_action(VcpuAction::None, Vec::new()); + + vcpu_manager.sync_action_finish(true); + // TODO(sicun): rollback + } + } +} + +impl MutEventSubscriber for VcpuEpollHandler { + fn process(&mut self, events: Events, ops: &mut EventOps) { + let vcpu_state_eventfd = self.eventfd.as_raw_fd(); + + match events.fd() { + fd if fd == vcpu_state_eventfd => self.process_cpu_state_event(ops), + _ => error!("vcpu manager epoll handler: unknown event"), + } + } + + fn init(&mut self, ops: &mut EventOps) { + ops.add(Events::new(&self.eventfd, EventSet::IN)).unwrap(); + } +} diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index c1510308da22..e09ec316bb41 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -18,3 +18,79 @@ pub struct NumaRegionInfo { /// vcpu ids belonging to this region pub vcpu_ids: Vec, } + +/// Information for cpu topology to guide guest init +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CpuTopology { + /// threads per core to indicate hyperthreading is enabled or not + pub threads_per_core: u8, + /// cores per die to guide guest cpu topology init + pub cores_per_die: u8, + /// dies per socket to guide guest cpu topology + pub dies_per_socket: u8, + /// number of sockets + pub sockets: u8, +} + +impl Default for CpuTopology { + fn default() -> Self { + CpuTopology { + threads_per_core: 1, + cores_per_die: 1, + dies_per_socket: 1, + sockets: 1, + } + } +} + +/// Configuration information for virtual machine instance. +#[derive(Clone, Debug, PartialEq)] +pub struct VmConfigInfo { + /// Number of vcpu to start. + pub vcpu_count: u8, + /// Max number of vcpu can be added + pub max_vcpu_count: u8, + /// Enable or disable hyperthreading. + pub ht_enabled: bool, + /// cpu power management. + pub cpu_pm: String, + /// cpu topology information + pub cpu_topology: CpuTopology, + /// vpmu support level + pub vpmu_feature: u8, + + /// Memory type that can be either hugetlbfs or shmem, default is shmem + pub mem_type: String, + /// Memory file path + pub mem_file_path: String, + /// The memory size in MiB. + pub mem_size_mib: usize, + /// reserve memory bytes + pub reserve_memory_bytes: u64, + + /// sock path + pub serial_path: Option, +} + +impl Default for VmConfigInfo { + fn default() -> Self { + VmConfigInfo { + vcpu_count: 1, + max_vcpu_count: 1, + ht_enabled: false, + cpu_pm: String::from("on"), + cpu_topology: CpuTopology { + threads_per_core: 1, + cores_per_die: 1, + dies_per_socket: 1, + sockets: 1, + }, + vpmu_feature: 0, + mem_type: String::from("shmem"), + mem_file_path: String::from(""), + mem_size_mib: 128, + reserve_memory_bytes: 0, + serial_path: None, + } + } +} From bec22ad01fc56c359fb3463d62fc10924e608f23 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 01:49:56 +0800 Subject: [PATCH 0075/1953] dragonball: add api module It is used to define the vmm communication interface. Signed-off-by: Chao Wu Signed-off-by: wllenyj --- src/dragonball/src/api/mod.rs | 6 ++ src/dragonball/src/api/v1/instance_info.rs | 84 ++++++++++++++++++++++ src/dragonball/src/api/v1/mod.rs | 7 ++ src/dragonball/src/lib.rs | 2 + 4 files changed, 99 insertions(+) create mode 100644 src/dragonball/src/api/mod.rs create mode 100644 src/dragonball/src/api/v1/instance_info.rs create mode 100644 src/dragonball/src/api/v1/mod.rs diff --git a/src/dragonball/src/api/mod.rs b/src/dragonball/src/api/mod.rs new file mode 100644 index 000000000000..75ca6af690a0 --- /dev/null +++ b/src/dragonball/src/api/mod.rs @@ -0,0 +1,6 @@ +// Copyright (C) 2019-2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! API related data structures to configure the vmm. + +pub mod v1; diff --git a/src/dragonball/src/api/v1/instance_info.rs b/src/dragonball/src/api/v1/instance_info.rs new file mode 100644 index 000000000000..d457b6124ba3 --- /dev/null +++ b/src/dragonball/src/api/v1/instance_info.rs @@ -0,0 +1,84 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use serde_derive::{Deserialize, Serialize}; + +/// The microvm state. +/// +/// When Dragonball starts, the instance state is Uninitialized. Once start_microvm method is +/// called, the state goes from Uninitialized to Starting. The state is changed to Running until +/// the start_microvm method ends. Halting and Halted are currently unsupported. +#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum InstanceState { + /// Microvm is not initialized. + Uninitialized, + /// Microvm is starting. + Starting, + /// Microvm is running. + Running, + /// Microvm is Paused. + Paused, + /// Microvm received a halt instruction. + Halting, + /// Microvm is halted. + Halted, + /// Microvm exit instead of process exit. + Exited(i32), +} + +/// The state of async actions +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] +pub enum AsyncState { + /// Uninitialized + Uninitialized, + /// Success + Success, + /// Failure + Failure, +} + +/// The strongly typed that contains general information about the microVM. +#[derive(Debug, Deserialize, Serialize)] +pub struct InstanceInfo { + /// The ID of the microVM. + pub id: String, + /// The state of the microVM. + pub state: InstanceState, + /// The version of the VMM that runs the microVM. + pub vmm_version: String, + /// The pid of the current VMM process. + pub pid: u32, + /// The state of async actions. + pub async_state: AsyncState, + /// List of tids of vcpu threads (vcpu index, tid) + pub tids: Vec<(u8, u32)>, +} + +impl InstanceInfo { + /// create instance info object with given id, version, and platform type + pub fn new(id: String, vmm_version: String) -> Self { + InstanceInfo { + id, + state: InstanceState::Uninitialized, + vmm_version, + pid: std::process::id(), + async_state: AsyncState::Uninitialized, + tids: Vec::new(), + } + } +} + +impl Default for InstanceInfo { + fn default() -> Self { + InstanceInfo { + id: String::from(""), + state: InstanceState::Uninitialized, + vmm_version: env!("CARGO_PKG_VERSION").to_string(), + pid: std::process::id(), + async_state: AsyncState::Uninitialized, + tids: Vec::new(), + } + } +} diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs new file mode 100644 index 000000000000..f25fb8436423 --- /dev/null +++ b/src/dragonball/src/api/v1/mod.rs @@ -0,0 +1,7 @@ +// Copyright (C) 2019-2022 Alibaba Cloud. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! API Version 1 related data structures to configure the vmm. + +mod instance_info; +pub use self::instance_info::{InstanceInfo, InstanceState}; diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index e9330fc95346..6bf0b9298a1b 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -11,6 +11,8 @@ /// Address space manager for virtual machines. pub mod address_space_manager; +/// API to handle vmm requests. +pub mod api; /// Structs to maintain configuration information. pub mod config_manager; /// Device manager for virtual machines. From 2aedd4d12aad52b3dc69a6cd9b0d2a0c84a2d5c0 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Sun, 15 May 2022 20:09:29 +0800 Subject: [PATCH 0076/1953] doc: add document for vCPU, api and device Create the document for vCPU and api. Add some detail in the device document. Fixes: #4257 Signed-off-by: Chao Wu --- src/dragonball/README.md | 3 +++ src/dragonball/docs/api.md | 7 ++++++ src/dragonball/docs/device.md | 3 +++ src/dragonball/docs/vcpu.md | 42 +++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 src/dragonball/docs/api.md create mode 100644 src/dragonball/docs/vcpu.md diff --git a/src/dragonball/README.md b/src/dragonball/README.md index 0e3bcb45a06c..c9d7e5119c3d 100644 --- a/src/dragonball/README.md +++ b/src/dragonball/README.md @@ -17,7 +17,10 @@ and configuration process. # Documentation Device: [Device Document](docs/device.md) +vCPU: [vCPU Document](docs/vcpu.md) +API: [API Document](docs/api.md) +Currently, the documents are still actively adding. You could see the [official documentation](docs/) page for more details. # Supported Architectures diff --git a/src/dragonball/docs/api.md b/src/dragonball/docs/api.md new file mode 100644 index 000000000000..cd2bc2db8e7f --- /dev/null +++ b/src/dragonball/docs/api.md @@ -0,0 +1,7 @@ +# API + +We provide plenty API for Kata runtime to interact with `Dragonball` virtual machine manager. +This document provides the introduction for each of them. + +TODO: Details will be added in the Part III PR for `Dragonball` + diff --git a/src/dragonball/docs/device.md b/src/dragonball/docs/device.md index 8f3fdbe6ed6a..ab2e078e7b85 100644 --- a/src/dragonball/docs/device.md +++ b/src/dragonball/docs/device.md @@ -14,4 +14,7 @@ Currently we have following device manager: ## Device supported `VIRTIO-VSOCK` +`i8042` +`COM1` +`COM2` diff --git a/src/dragonball/docs/vcpu.md b/src/dragonball/docs/vcpu.md new file mode 100644 index 000000000000..e2be8037b60f --- /dev/null +++ b/src/dragonball/docs/vcpu.md @@ -0,0 +1,42 @@ +# vCPU + +## vCPU Manager +The vCPU manager is to manage all vCPU related actions, we will dive into some of the important structure members in this doc. + +For now, aarch64 vCPU support is still under development, we'll introduce it when we merge `runtime-rs` to the master branch. (issue: #4445) + +### vCPU config +`VcpuConfig` is used to configure guest overall CPU info. + +`boot_vcpu_count` is used to define the initial vCPU number. + +`max_vcpu_count` is used to define the maximum vCPU number and it's used for the upper boundary for CPU hotplug feature + +`thread_per_core`, `cores_per_die`, `dies_per_socket` and `socket` are used to define CPU topology. + +`vpmu_feature` is used to define `vPMU` feature level. +If `vPMU` feature is `Disabled`, it means `vPMU` feature is off (by default). +If `vPMU` feature is `LimitedlyEnabled`, it means minimal `vPMU` counters are supported (cycles and instructions). +If `vPMU` feature is `FullyEnabled`, it means all `vPMU` counters are supported + +## vCPU State + +There are four states for vCPU state machine: `running`, `paused`, `waiting_exit`, `exited`. There is a state machine to maintain the task flow. + +When the vCPU is created, it'll turn to `paused` state. After vCPU resource is ready at VMM, it'll send a `Resume` event to the vCPU thread, and then vCPU state will change to `running`. + +During the `running` state, VMM will catch vCPU exit and execute different logic according to the exit reason. + +If the VMM catch some exit reasons that it cannot handle, the state will change to `waiting_exit` and VMM will stop the virtual machine. +When the state switches to `waiting_exit`, an exit event will be sent to vCPU `exit_evt`, event manager will detect the change in `exit_evt` and set VMM `exit_evt_flag` as 1. A thread serving for VMM event loop will check `exit_evt_flag` and if the flag is 1, it'll stop the VMM. + +When the VMM is stopped / destroyed, the state will change to `exited`. + +## vCPU Hot plug +Since `Dragonball Sandbox` doesn't support virtualization of ACPI system, we use [`upcall`](https://github.com/openanolis/dragonball-sandbox/tree/main/crates/dbs-upcall) to establish a direct communication channel between `Dragonball` and Guest in order to trigger vCPU hotplug. + +To use `upcall`, kernel patches are needed, you can get the patches from [`upcall`](https://github.com/openanolis/dragonball-sandbox/tree/main/crates/dbs-upcall) page, and we'll provide a ready-to-use guest kernel binary for you to try. + +vCPU hot plug / hot unplug range is [1, `max_vcpu_count`]. Operations not in this range will be invalid. + + From 8bb00a3dc8bc8c947e88c0652a2b835354ebe26a Mon Sep 17 00:00:00 2001 From: Zizheng Bian Date: Tue, 14 Jun 2022 12:00:39 +0800 Subject: [PATCH 0077/1953] dragonball: fix a bug when generating kernel boot args We should refuse to generate boot args when hotplugging, not cold starting. Signed-off-by: Zizheng Bian --- src/dragonball/src/device_manager/mod.rs | 38 +++++++++++++----------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 4a1953113652..0c485963ea2b 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -251,29 +251,31 @@ impl DeviceOpContext { &self.logger } + #[allow(unused_variables)] fn generate_kernel_boot_args(&mut self, kernel_config: &mut KernelConfigInfo) -> Result<()> { - if !self.is_hotplug { + if self.is_hotplug { return Err(DeviceMgrError::InvalidOperation); } #[cfg(feature = "dbs-virtio-devices")] - let cmdline = kernel_config.kernel_cmdline_mut(); - - #[cfg(feature = "dbs-virtio-devices")] - for device in self.virtio_devices.iter() { - let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_device_info(device)?; - - // as per doc, [virtio_mmio.]device=@: needs to be appended - // to kernel commandline for virtio mmio devices to get recognized - // the size parameter has to be transformed to KiB, so dividing hexadecimal value in - // bytes to 1024; further, the '{}' formatting rust construct will automatically - // transform it to decimal - cmdline - .insert( - "virtio_mmio.device", - &format!("{}K@0x{:08x}:{}", mmio_size / 1024, mmio_base, irq), - ) - .map_err(DeviceMgrError::Cmdline)?; + { + let cmdline = kernel_config.kernel_cmdline_mut(); + + for device in self.virtio_devices.iter() { + let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_device_info(device)?; + + // as per doc, [virtio_mmio.]device=@: needs to be appended + // to kernel commandline for virtio mmio devices to get recognized + // the size parameter has to be transformed to KiB, so dividing hexadecimal value in + // bytes to 1024; further, the '{}' formatting rust construct will automatically + // transform it to decimal + cmdline + .insert( + "virtio_mmio.device", + &format!("{}K@0x{:08x}:{}", mmio_size / 1024, mmio_base, irq), + ) + .map_err(DeviceMgrError::Cmdline)?; + } } Ok(()) From 71db2dd5b8567e741251d1adc840adf7f54b82b8 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 27 Jun 2022 15:24:27 +0800 Subject: [PATCH 0078/1953] hotplug: add room for future acpi hotplug mechanism In order to support ACPI hotplug in the future with the cooperative work from the Kata community, we add ACPI feature and dbs-upcall feature to add room for ACPI hotplug. Signed-off-by: Chao Wu --- src/dragonball/Cargo.toml | 3 ++- src/dragonball/src/device_manager/mod.rs | 19 ++++++++++++------- src/dragonball/src/vcpu/vcpu_manager.rs | 7 ++++--- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 3fa2dc5064c2..3cc17de1ddde 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -45,8 +45,9 @@ slog-term = "2.9.0" slog-async = "2.7.0" [features] +acpi = [] atomic-guest-memory = [] -hotplug = ["dbs-upcall", "virtio-vsock"] +hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] [patch.'crates-io'] diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 0c485963ea2b..2ad26e0d0191 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -29,10 +29,10 @@ use dbs_virtio_devices::{ VirtioDevice, }; -#[cfg(feature = "hotplug")] +#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] use dbs_upcall::{ - MmioDevRequest, DevMgrRequest, DevMgrService, UpcallClient, UpcallClientError, - UpcallClientRequest, UpcallClientResponse + DevMgrRequest, DevMgrService, MmioDevRequest, UpcallClient, UpcallClientError, + UpcallClientRequest, UpcallClientResponse, }; use crate::address_space_manager::GuestAddressSpaceImpl; @@ -90,7 +90,7 @@ pub enum DeviceMgrError { #[error(transparent)] Virtio(virtio::Error), - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] /// Failed to hotplug the device. #[error("failed to hotplug virtual device")] HotplugDevice(#[source] UpcallClientError), @@ -199,7 +199,7 @@ pub struct DeviceOpContext { logger: slog::Logger, is_hotplug: bool, - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_client: Option>>, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec>, @@ -233,7 +233,7 @@ impl DeviceOpContext { address_space, logger, is_hotplug, - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_client: None, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec::new(), @@ -301,7 +301,7 @@ impl DeviceOpContext { } } -#[cfg(feature = "hotplug")] +#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] impl DeviceOpContext { fn call_hotplug_device( &self, @@ -362,6 +362,11 @@ impl DeviceOpContext { } } +#[cfg(all(feature = "hotplug", feature = "acpi"))] +impl DeviceOpContext { + // TODO: We will implement this when we develop ACPI virtualization +} + /// Device manager for virtual machines, which manages all device for a virtual machine. pub struct DeviceManager { io_manager: Arc>, diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index f548035b2c8e..189dfc61555c 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -15,7 +15,7 @@ use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender}; use std::sync::{Arc, Barrier, Mutex, RwLock}; use std::time::Duration; -#[cfg(feature = "hotplug")] +#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] use dbs_upcall::{DevMgrService, UpcallClient}; use dbs_utils::epoll_manager::{EpollManager, EventOps, EventSet, Events, MutEventSubscriber}; use dbs_utils::time::TimestampUs; @@ -206,7 +206,7 @@ pub struct VcpuManager { vcpus_in_action: (VcpuAction, Vec), pub(crate) reset_event_fd: Option, - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_channel: Option>>, // X86 specific fields. @@ -290,7 +290,7 @@ impl VcpuManager { action_sycn_tx: None, vcpus_in_action: (VcpuAction::None, Vec::new()), reset_event_fd: None, - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_channel: None, #[cfg(target_arch = "x86_64")] supported_cpuid, @@ -794,6 +794,7 @@ mod hotplug { #[cfg(all(target_arch = "aarch64", not(test)))] const APIC_VERSION: u8 = 0; + #[cfg(feature = "dbs-upcall")] impl VcpuManager { /// add upcall channel for vcpu manager pub fn set_upcall_channel( From 9c526292e7bf3c97b76edd9195f8e1781393271b Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Fri, 24 Jun 2022 17:26:13 +0800 Subject: [PATCH 0079/1953] runtime-rs:refactor network model with netlink refactor tcfilter with netlink Fixes: #4289 Signed-off-by: Zhongtao Hu --- src/runtime-rs/crates/resource/Cargo.toml | 6 +- .../network/network_model/tc_filter_model.rs | 111 +++++++++--------- 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index e6365b1705c0..7d97af016426 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -12,11 +12,11 @@ cgroups-rs = "0.2.9" futures = "0.3.11" lazy_static = "1.4.0" libc = ">=0.2.39" -netlink-sys = "0.8.2" -netlink-packet-route = "0.11.0" +netlink-sys = "0.8.3" +netlink-packet-route = "0.12.0" nix = "0.16.0" rand = "^0.7.2" -rtnetlink = "0.9.1" +rtnetlink = "0.10.0" scopeguard = "1.0.0" slog = "2.5.2" slog-scope = "4.4.0" diff --git a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs index ae347e717c42..c3bc3542e86c 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs @@ -4,9 +4,10 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use async_trait::async_trait; -use tokio::process::Command; +use rtnetlink::Handle; +use scopeguard::defer; use super::{NetworkModel, NetworkModelType}; use crate::network::NetworkPair; @@ -19,7 +20,6 @@ impl TcFilterModel { Ok(Self {}) } } - #[async_trait] impl NetworkModel for TcFilterModel { fn model_type(&self) -> NetworkModelType { @@ -27,69 +27,74 @@ impl NetworkModel for TcFilterModel { } async fn add(&self, pair: &NetworkPair) -> Result<()> { - let tap_name = &pair.tap.tap_iface.name; - let virt_name = &pair.virt_iface.name; + let (connection, handle, _) = rtnetlink::new_connection().context("new connection")?; + let thread_handler = tokio::spawn(connection); + + defer!({ + thread_handler.abort(); + }); - add_qdisc_ingress(tap_name) + let tap_index = fetch_index(&handle, pair.tap.tap_iface.name.as_str()) .await - .context("add qdisc ingress for tap link")?; - add_qdisc_ingress(virt_name) + .context("fetch tap by index")?; + let virt_index = fetch_index(&handle, pair.virt_iface.name.as_str()) .await - .context("add qdisc ingress")?; + .context("fetch virt by index")?; - add_redirect_tcfilter(tap_name, virt_name) + handle + .qdisc() + .add(tap_index as i32) + .ingress() + .execute() .await - .context("add tc filter for tap")?; - add_redirect_tcfilter(virt_name, tap_name) + .context("add tap ingress")?; + + handle + .qdisc() + .add(virt_index as i32) + .ingress() + .execute() .await - .context("add tc filter")?; - Ok(()) - } + .context("add virt ingress")?; - async fn del(&self, pair: &NetworkPair) -> Result<()> { - del_qdisc(&pair.virt_iface.name) + handle + .traffic_filter(tap_index as i32) + .add() + .protocol(0x0003) + .egress() + .redirect(virt_index) + .execute() .await - .context("del qdisc")?; - Ok(()) - } -} + .context("add tap egress")?; -// TODO: use netlink replace tc command -async fn add_qdisc_ingress(dev: &str) -> Result<()> { - let output = Command::new("/sbin/tc") - .args(&["qdisc", "add", "dev", dev, "handle", "ffff:", "ingress"]) - .output() - .await - .context("add tc")?; - if !output.status.success() { - return Err(anyhow!("{}", String::from_utf8(output.stderr)?)); + handle + .traffic_filter(virt_index as i32) + .add() + .protocol(0x0003) + .egress() + .redirect(tap_index) + .execute() + .await + .context("add virt egress")?; + Ok(()) } - Ok(()) -} -async fn add_redirect_tcfilter(src: &str, dst: &str) -> Result<()> { - let output = Command::new("/sbin/tc") - .args(&[ - "filter", "add", "dev", src, "parent", "ffff:", "protocol", "all", "u32", "match", - "u8", "0", "0", "action", "mirred", "egress", "redirect", "dev", dst, - ]) - .output() - .await - .context("add redirect tcfilter")?; - if !output.status.success() { - return Err(anyhow!("{}", String::from_utf8(output.stderr)?)); + async fn del(&self, pair: &NetworkPair) -> Result<()> { + let (connection, handle, _) = rtnetlink::new_connection().context("new connection")?; + let thread_handler = tokio::spawn(connection); + defer!({ + thread_handler.abort(); + }); + let virt_index = fetch_index(&handle, &pair.virt_iface.name).await?; + handle.qdisc().del(virt_index as i32).execute().await?; + Ok(()) } - Ok(()) } -async fn del_qdisc(dev: &str) -> Result<()> { - let output = Command::new("/sbin/tc") - .args(&["qdisc", "del", "dev", dev, "handle", "ffff:", "ingress"]) - .output() +async fn fetch_index(handle: &Handle, name: &str) -> Result { + let link = crate::network::network_pair::get_link_by_name(handle, name) .await - .context("del qdisc")?; - if !output.status.success() { - return Err(anyhow!("{}", String::from_utf8(output.stderr)?)); - } - Ok(()) + .context("get link by name")?; + let base = link.attrs(); + Ok(base.index) } From 8a697268d061ccdf799c95b89e2cfd9c54086013 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 1 Jun 2022 17:48:18 +0800 Subject: [PATCH 0080/1953] build: makefile for dragonball config use makefile to generate dragonball config file Fixes: #4379 Signed-off-by: Zhongtao Hu --- src/runtime-rs/.gitignore | 1 + src/runtime-rs/Makefile | 266 +++++++++++++++++- src/runtime-rs/arch/x86_64-options.mk | 15 + .../config/configuration-dragonball.toml.in | 249 ++++++++++++++++ 4 files changed, 528 insertions(+), 3 deletions(-) create mode 100644 src/runtime-rs/arch/x86_64-options.mk create mode 100644 src/runtime-rs/config/configuration-dragonball.toml.in diff --git a/src/runtime-rs/.gitignore b/src/runtime-rs/.gitignore index 82cc0b64ca97..0e5a39c11f19 100644 --- a/src/runtime-rs/.gitignore +++ b/src/runtime-rs/.gitignore @@ -1,2 +1,3 @@ target crates/shim/src/config.rs +/config/*.toml diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 5585e844a49a..fb34db9d3bdd 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -8,14 +8,249 @@ # Use the following format: # '##VAR VARIABLE_NAME: help about variable' # '##TARGET TARGET_NAME: help about target' - +PROJECT_TYPE = kata PROJECT_NAME = Kata Containers +PROJECT_TAG = kata-containers PROJECT_URL = https://github.com/kata-containers PROJECT_COMPONENT = containerd-shim-kata-v2 CONTAINERD_RUNTIME_NAME = io.containerd.kata.v2 +include ../../utils.mk + +ARCH_DIR = arch +ARCH_FILE_SUFFIX = -options.mk +ARCH_FILE = $(ARCH_DIR)/$(ARCH)$(ARCH_FILE_SUFFIX) + +ifeq (,$(realpath $(ARCH_FILE))) + $(error "ERROR: invalid architecture: '$(ARCH)'") +else + # Load architecture-dependent settings + include $(ARCH_FILE) +endif + +ifeq ($(PREFIX),) + PREFIX := /usr +endif + +PREFIXDEPS := $(PREFIX) +DBBINDIR := $(PREFIXDEPS)/bin +LIBEXECDIR := $(PREFIXDEPS)/libexec +SHAREDIR := $(PREFIX)/share +DEFAULTSDIR := $(SHAREDIR)/defaults +PROJECT_DIR = $(PROJECT_TAG) +IMAGENAME = $(PROJECT_TAG).img TARGET = $(PROJECT_COMPONENT) +CONFIG_FILE = configuration.toml +HYPERVISOR_DB = dragonball + + +DEFAULT_HYPERVISOR ?= $(HYPERVISOR_DB) + +HYPERVISORS := $(HYPERVISOR_DB) + +DBPATH := $(DBBINDIR)/$(QEMUCMD) +DBVALIDHYPERVISORPATHS := [\"$(DBPATH)\"] + +PKGDATADIR := $(PREFIXDEPS)/share/$(PROJECT_DIR) +KERNELDIR := $(PKGDATADIR) +IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME) +FIRMWAREPATH := +FIRMWAREVOLUMEPATH := + +# Default number of vCPUs +DEFVCPUS := 1 +# Default maximum number of vCPUs +DEFMAXVCPUS := 0 +# Default memory size in MiB +DEFMEMSZ := 2048 +# Default memory slots +# Cases to consider : +# - nvdimm rootfs image +# - preallocated memory +# - vm template memory +# - hugepage memory +DEFMEMSLOTS := 10 +#Default number of bridges +DEFBRIDGES := 1 +DEFENABLEANNOTATIONS := [] +DEFDISABLEGUESTSECCOMP := true +DEFDISABLEGUESTEMPTYDIR := false +#Default experimental features enabled +DEFAULTEXPFEATURES := [] +DEFDISABLESELINUX := false +#Default entropy source +DEFENTROPYSOURCE := /dev/urandom +DEFVALIDENTROPYSOURCES := [\"/dev/urandom\",\"/dev/random\",\"\"] +DEFDISABLEBLOCK := false +DEFSHAREDFS_CLH_VIRTIOFS := virtio-fs +DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs +DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd +ifeq ($(ARCH),x86_64) +DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/virtiofsd +endif +DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"] +# Default DAX mapping cache size in MiB +#if value is 0, DAX is not enabled +DEFVIRTIOFSCACHESIZE ?= 0 +DEFVIRTIOFSCACHE ?= auto +# Format example: +# [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"] +# +# see `virtiofsd -h` for possible options. +# Make sure you quote args. +DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"-o\", \"announce_submounts\"] +DEFENABLEIOTHREADS := false +DEFENABLEVHOSTUSERSTORE := false +DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user +DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"] +DEFFILEMEMBACKEND := "" +DEFVALIDFILEMEMBACKENDS := [\"$(DEFFILEMEMBACKEND)\"] +DEFMSIZE9P := 8192 +DEFVFIOMODE := guest-kernel +# Default cgroup model +DEFSANDBOXCGROUPONLY ?= false +DEFSTATICRESOURCEMGMT ?= false +DEFBINDMOUNTS := [] +SED = sed +CLI_DIR = cmd +SHIMV2 = containerd-shim-kata-v2 +SHIMV2_OUTPUT = $(CURDIR)/$(SHIMV2) +SHIMV2_DIR = $(CLI_DIR)/$(SHIMV2) +MONITOR = kata-monitor +MONITOR_OUTPUT = $(CURDIR)/$(MONITOR) +MONITOR_DIR = $(CLI_DIR)/kata-monitor +SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$') +VERSION := ${shell cat ./VERSION} +# List of configuration files to build and install +CONFIGS = +CONFIG_PATHS = +SYSCONFIG_PATHS = +# List of hypervisors known for the current architecture +KNOWN_HYPERVISORS = +# List of hypervisors known for the current architecture +KNOWN_HYPERVISORS = + +CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) +SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR) +# Main configuration file location for stateless systems +CONFIG_PATH := $(abspath $(CONFDIR)/$(CONFIG_FILE)) +# Secondary configuration file location. Note that this takes precedence +# over CONFIG_PATH. +SYSCONFIG := $(abspath $(SYSCONFDIR)/$(CONFIG_FILE)) +SHAREDIR := $(SHAREDIR) + +ifneq (,$(DBCMD)) + KNOWN_HYPERVISORS += $(HYPERVISOR_DB) + CONFIG_FILE_DB = configuration-dragonball.toml + CONFIG_DB = config/$(CONFIG_FILE_DB) + CONFIG_DB_IN = $(CONFIG_DB).in + CONFIG_PATH_DB = $(abspath $(CONFDIR)/$(CONFIG_FILE_DB)) + CONFIG_PATHS += $(CONFIG_PATH_DB) + SYSCONFIG_DB = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_DB)) + SYSCONFIG_PATHS += $(SYSCONFIG_DB) + CONFIGS += $(CONFIG_DB) + # dragonball-specific options (all should be suffixed by "_dragonball") + DEFMAXVCPUS_DB := 1 + DEFBLOCKSTORAGEDRIVER_DB := virtio-blk + DEFNETWORKMODEL_DB := tcfilter + KERNELPARAMS = console=ttyS1 agent.log_vport=1025 + KERNELTYPE_DB = uncompressed + KERNEL_NAME_DB = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_DB)) + KERNELPATH_DB = $(KERNELDIR)/$(KERNEL_NAME_DB) + DEFSANDBOXCGROUPONLY = true + RUNTIMENAME := virt_container + PIPESIZE := 1 + DBSHAREDFS := inline-virtio-fs +endif + +ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_DB)) + DEFAULT_HYPERVISOR_CONFIG = $(CONFIG_FILE_DB) +endif +# list of variables the user may wish to override +USER_VARS += ARCH +USER_VARS += BINDIR +USER_VARS += CONFIG_DB_IN +USER_VARS += CONFIG_PATH +USER_VARS += DESTDIR +USER_VARS += DEFAULT_HYPERVISOR +USER_VARS += DBCMD +USER_VARS += DBCTLCMD +USER_VARS += DBPATH +USER_VARS += DBVALIDHYPERVISORPATHS +USER_VARS += DBCTLPATH +USER_VARS += DBVALIDCTLPATHS +USER_VARS += SYSCONFIG +USER_VARS += IMAGENAME +USER_VARS += IMAGEPATH +USER_VARS += MACHINETYPE +USER_VARS += KERNELDIR +USER_VARS += KERNELTYPE +USER_VARS += KERNELPATH_DB +USER_VARS += KERNELPATH +USER_VARS += KERNELVIRTIOFSPATH +USER_VARS += FIRMWAREPATH +USER_VARS += FIRMWAREVOLUMEPATH +USER_VARS += MACHINEACCELERATORS +USER_VARS += CPUFEATURES +USER_VARS += DEFMACHINETYPE_CLH +USER_VARS += KERNELPARAMS +USER_VARS += LIBEXECDIR +USER_VARS += LOCALSTATEDIR +USER_VARS += PKGDATADIR +USER_VARS += PKGLIBEXECDIR +USER_VARS += PKGRUNDIR +USER_VARS += PREFIX +USER_VARS += PROJECT_BUG_URL +USER_VARS += PROJECT_NAME +USER_VARS += PROJECT_ORG +USER_VARS += PROJECT_PREFIX +USER_VARS += PROJECT_TAG +USER_VARS += PROJECT_TYPE +USER_VARS += RUNTIME_NAME +USER_VARS += SHAREDIR +USER_VARS += SYSCONFDIR +USER_VARS += DEFVCPUS +USER_VARS += DEFMAXVCPUS +USER_VARS += DEFMAXVCPUS_ACRN +USER_VARS += DEFMAXVCPUS_DB +USER_VARS += DEFMEMSZ +USER_VARS += DEFMEMSLOTS +USER_VARS += DEFBRIDGES +USER_VARS += DEFNETWORKMODEL_DB +USER_VARS += DEFDISABLEGUESTEMPTYDIR +USER_VARS += DEFDISABLEGUESTSECCOMP +USER_VARS += DEFDISABLESELINUX +USER_VARS += DEFAULTEXPFEATURES +USER_VARS += DEFDISABLEBLOCK +USER_VARS += DEFBLOCKSTORAGEDRIVER_DB +USER_VARS += DEFSHAREDFS_CLH_VIRTIOFS +USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS +USER_VARS += DEFVIRTIOFSDAEMON +USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS +USER_VARS += DEFVIRTIOFSCACHESIZE +USER_VARS += DEFVIRTIOFSCACHE +USER_VARS += DEFVIRTIOFSEXTRAARGS +USER_VARS += DEFENABLEANNOTATIONS +USER_VARS += DEFENABLEIOTHREADS +USER_VARS += DEFENABLEVHOSTUSERSTORE +USER_VARS += DEFVHOSTUSERSTOREPATH +USER_VARS += DEFVALIDVHOSTUSERSTOREPATHS +USER_VARS += DEFFILEMEMBACKEND +USER_VARS += DEFVALIDFILEMEMBACKENDS +USER_VARS += DEFMSIZE9P +USER_VARS += DEFENTROPYSOURCE +USER_VARS += DEFVALIDENTROPYSOURCES +USER_VARS += DEFSANDBOXCGROUPONLY +USER_VARS += DEFSTATICRESOURCEMGMT +USER_VARS += DEFBINDMOUNTS +USER_VARS += DEFVFIOMODE +USER_VARS += BUILDFLAGS +USER_VARS += RUNTIMENAME +USER_VARS += HYPERVISOR_DB +USER_VARS += PIPESIZE +USER_VARS += DBSHAREDFS + SOURCES := \ $(shell find . 2>&1 | grep -E '.*\.rs$$') \ Cargo.toml @@ -35,7 +270,6 @@ ifneq ($(EXTRA_RUSTFEATURES),) override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES) endif -include ../../utils.mk TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) @@ -49,6 +283,12 @@ GENERATED_CODE = crates/shim/src/config.rs RUNTIME_NAME=$(TARGET) RUNTIME_VERSION=$(VERSION) +GENERATED_VARS = \ + VERSION \ + CONFIG_DB_IN \ + $(USER_VARS) + + GENERATED_REPLACEMENTS= \ PROJECT_NAME \ RUNTIME_NAME \ @@ -78,6 +318,16 @@ define INSTALL_FILE install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1; endef +define INSTALL_CONFIG + sudo install --mode 0644 -D $1 $(DESTDIR)$2/$(notdir $1); +endef + +# Returns the name of the kernel file to use based on the provided KERNELTYPE. +# $1 : KERNELTYPE (compressed or uncompressed) +define MAKE_KERNEL_NAME +$(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container) +endef + .DEFAULT_GOAL := default ##TARGET default: build code @@ -88,8 +338,13 @@ $(TARGET): $(GENERATED_CODE) $(TARGET_PATH) $(TARGET_PATH): $(SOURCES) | show-summary @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) +GENERATED_FILES += $(CONFIGS) + $(GENERATED_FILES): %: %.in - @sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@" + @sed \ + $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') \ + $(foreach v,$(GENERATED_VARS),-e "s|@$v@|$($v)|g") \ + $< > $@ ##TARGET optimize: optimized build optimize: $(SOURCES) | show-summary show-header @@ -100,6 +355,7 @@ clean: @cargo clean @rm -f $(GENERATED_FILES) @rm -f tarpaulin-report.html + @rm -f $(CONFIGS) vendor: @cargo vendor @@ -163,6 +419,10 @@ codecov: check_tarpaulin codecov-html: check_tarpaulin cargo tarpaulin $(TARPAULIN_ARGS) -o Html +install-configs: $(CONFIGS) + $(foreach f,$(CONFIGS),$(call INSTALL_CONFIG,$f,$(dir $(CONFIG_PATH)))) \ + sudo ln -sf $(DEFAULT_HYPERVISOR_CONFIG) $(DESTDIR)/$(CONFIG_PATH) + .PHONY: \ help \ optimize \ diff --git a/src/runtime-rs/arch/x86_64-options.mk b/src/runtime-rs/arch/x86_64-options.mk new file mode 100644 index 000000000000..522810824a4a --- /dev/null +++ b/src/runtime-rs/arch/x86_64-options.mk @@ -0,0 +1,15 @@ +# Copyright (c) 2019-2022 Alibaba Cloud +# Copyright (c) 2019-2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +MACHINETYPE := q35 +KERNELPARAMS := +MACHINEACCELERATORS := +CPUFEATURES := pmu=off + +QEMUCMD := qemu-system-x86_64 + +# dragonball binary name +DBCMD := dragonball \ No newline at end of file diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in new file mode 100644 index 000000000000..57c4d2951cee --- /dev/null +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -0,0 +1,249 @@ +# Copyright (c) 2019-2022 Alibaba Cloud +# Copyright (c) 2019-2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +# XXX: WARNING: this file is auto-generated. +# XXX: +# XXX: Source file: "@CONFIG_DB_IN@" +# XXX: Project: +# XXX: Name: @PROJECT_NAME@ +# XXX: Type: @PROJECT_TYPE@ + +[hypervisor.dragonball] +path = "@DBPATH@" +ctlpath = "@DBCTLPATH@" +kernel = "@KERNELPATH_DB@" +image = "@IMAGEPATH@" + +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + +# List of valid annotations values for the hypervisor +# Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DBVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @DBVALIDHYPERVISORPATHS@ + +# List of valid annotations values for ctlpath +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: +# valid_ctlpaths = + +# Optional space-separated list of options to pass to the guest kernel. +# For example, use `kernel_params = "vsyscall=emulate"` if you are having +# trouble running pre-2.15 glibc. +# +# WARNING: - any parameter specified here will take priority over the default +# parameter value of the same name used to start the virtual machine. +# Do not set values here unless you understand the impact of doing so as you +# may stop the virtual machine from booting. +# To see the list of default parameters, enable hypervisor debug, create a +# container and look for 'default-kernel-parameters' log entries. +kernel_params = "@KERNELPARAMS@" + +# Path to the firmware. +# If you want that DB uses the default firmware leave this option empty +firmware = "@FIRMWAREPATH@" + + +# Default number of vCPUs per SB/VM: +# unspecified or 0 --> will be set to 1 +# < 0 --> will be set to the actual number of physical cores +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores +default_vcpus = @DEFVCPUS@ + + +# Default maximum number of vCPUs per SB/VM: +# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# > 0 <= number of physical cores --> will be set to the specified number +# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number +# of vCPUs supported by KVM if that number is exceeded +# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when +# the actual number of physical cores is greater than it. +# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU +# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs +# can be added to a SB/VM, but the memory footprint will be big. Another example, with +# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of +# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, +# unless you know what are you doing. +default_maxvcpus = @DEFMAXVCPUS_DB@ + +# Bridges can be used to hot plug devices. +# Limitations: +# * Currently only pci bridges are supported +# * Until 30 devices per bridge can be hot plugged. +# * Until 5 PCI bridges can be cold plugged per VM. +# This limitation could be a bug in the kernel +# Default number of bridges per SB/VM: +# unspecified or 0 --> will be set to @DEFBRIDGES@ +# > 1 <= 5 --> will be set to the specified number +# > 5 --> will be set to 5 +default_bridges = @DEFBRIDGES@ + +# Default memory size in MiB for SB/VM. +# If unspecified then it will be set @DEFMEMSZ@ MiB. +default_memory = @DEFMEMSZ@ + +# Block storage driver to be used for the hypervisor in case the container +# rootfs is backed by a block device. DB only supports virtio-blk. +block_device_driver = "@DEFBLOCKSTORAGEDRIVER_DB@" + +# This option changes the default hypervisor and kernel parameters +# to enable debug output where available. +# +# Default false +#enable_debug = true + +# Disable the customizations done in the runtime when it detects +# that it is running on top a VMM. This will result in the runtime +# behaving as it would when running on bare metal. +# +#disable_nesting_checks = true + +# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. +# Default false +#disable_vhost_net = true + +# Path to OCI hook binaries in the *guest rootfs*. +# This does not affect host-side hooks which must instead be added to +# the OCI spec passed to the runtime. +# +# You can create a rootfs with hooks by customizing the osbuilder scripts: +# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder +# +# Hooks must be stored in a subdirectory of guest_hook_path according to their +# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". +# The agent will scan these directories for executable files and add them, in +# lexicographical order, to the lifecycle of the guest container. +# Hooks are executed in the runtime namespace of the guest. See the official documentation: +# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks +# Warnings will be logged if any error is encountered while scanning for hooks, +# but it will not abort container execution. +#guest_hook_path = "/usr/share/oci/hooks" + +# Shared file system type: +# - virtio-fs (default) +# - virtio-9p +# - virtio-fs-nydus +shared_fs = "@DBSHAREDFS@" + +[agent.@PROJECT_TYPE@] +container_pipe_size=@PIPESIZE@ +# If enabled, make the agent display debug-level messages. +# (default: disabled) +#enable_debug = true + +# Enable agent tracing. +# +# If enabled, the agent will generate OpenTelemetry trace spans. +# +# Notes: +# +# - If the runtime also has tracing enabled, the agent spans will be +# associated with the appropriate runtime parent span. +# - If enabled, the runtime will wait for the container to shutdown, +# increasing the container shutdown time slightly. +# +# (default: disabled) +#enable_tracing = true + +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true + +# Agent connection dialing timeout value in seconds +# (default: 30) +#dial_timeout = 30 + +[runtime] +# If enabled, the runtime will log additional debug messages to the +# system log +# (default: disabled) +#enable_debug = true +# +# Internetworking model +# Determines how the VM should be connected to the +# the container network interface +# Options: +# +# - bridged (Deprecated) +# Uses a linux bridge to interconnect the container interface to +# the VM. Works for most cases except macvlan and ipvlan. +# ***NOTE: This feature has been deprecated with plans to remove this +# feature in the future. Please use other network models listed below. +# +# +# - macvtap +# Used when the Container network interface can be bridged using +# macvtap. +# +# - none +# Used when customize network. Only creates a tap device. No veth pair. +# +# - tcfilter +# Uses tc filter rules to redirect traffic from the network interface +# provided by plugin to a tap interface connected to the VM. +# +internetworking_model="@DEFNETWORKMODEL_DB@" + +name="@RUNTIMENAME@" +hypervisor_name="@HYPERVISOR_DB@" +agent_name="@PROJECT_TYPE@" + +# disable guest seccomp +# Determines whether container seccomp profiles are passed to the virtual +# machine and applied by the kata agent. If set to true, seccomp is not applied +# within the guest +# (default: true) +disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + +# If enabled, the runtime will create opentracing.io traces and spans. +# (See https://www.jaegertracing.io/docs/getting-started). +# (default: disabled) +#enable_tracing = true + +# Set the full url to the Jaeger HTTP Thrift collector. +# The default if not set will be "http://localhost:14268/api/traces" +#jaeger_endpoint = "" + +# Sets the username to be used if basic auth is required for Jaeger. +#jaeger_user = "" + +# Sets the password to be used if basic auth is required for Jaeger. +#jaeger_password = "" + +# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. +# This option may have some potential impacts to your host. It should only be used when you know what you're doing. +# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only +# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge +# (like OVS) directly. +# (default: false) +#disable_new_netns = true + +# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. +# The container cgroups in the host are not created, just one single cgroup per sandbox. +# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. +# The sandbox cgroup is constrained if there is no container type annotation. +# See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType +sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ + +# Enabled experimental feature list, format: ["a", "b"]. +# Experimental features are features not stable enough for production, +# they may break compatibility, and are prepared for a big version bump. +# Supported experimental features: +# (default: []) +experimental=@DEFAULTEXPFEATURES@ + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# enable_pprof = true \ No newline at end of file From 242992e3de14caf840baa19261269638bc4c6dbb Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 7 Jun 2022 15:48:25 +0800 Subject: [PATCH 0081/1953] build: put install methods in utils.mk put install methods in utils.mk to avoid duplication Fixes: #4379 Signed-off-by: Zhongtao Hu --- src/runtime-rs/Makefile | 7 +++---- utils.mk | 42 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index fb34db9d3bdd..e923ad8c71a2 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -250,6 +250,9 @@ USER_VARS += RUNTIMENAME USER_VARS += HYPERVISOR_DB USER_VARS += PIPESIZE USER_VARS += DBSHAREDFS +USER_VARS += KATA_INSTALL_GROUP +USER_VARS += KATA_INSTALL_OWNER +USER_VARS += KATA_INSTALL_CFG_PERMS SOURCES := \ $(shell find . 2>&1 | grep -E '.*\.rs$$') \ @@ -318,10 +321,6 @@ define INSTALL_FILE install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1; endef -define INSTALL_CONFIG - sudo install --mode 0644 -D $1 $(DESTDIR)$2/$(notdir $1); -endef - # Returns the name of the kernel file to use based on the provided KERNELTYPE. # $1 : KERNELTYPE (compressed or uncompressed) define MAKE_KERNEL_NAME diff --git a/utils.mk b/utils.mk index c87da0c06b40..858c14d493e0 100644 --- a/utils.mk +++ b/utils.mk @@ -3,6 +3,23 @@ # SPDX-License-Identifier: Apache-2.0 # +# Note: +# +# Since this file defines rules, it should be included +# in other makefiles *after* their default rule has been defined. + +# Owner for installed files +export KATA_INSTALL_OWNER ?= root + +# Group for installed files +export KATA_INSTALL_GROUP ?= adm + +# Permissions for installed configuration files. +# +# XXX: Note that the permissions MUST be zero for "other" +# XXX: in case the configuration file contains secrets. +export KATA_INSTALL_CFG_PERMS ?= 0640 + # Create a set of standard rules for a project such that: # # - The component depends on its Makefile. @@ -160,3 +177,28 @@ standard_rust_check: cargo clippy --all-targets --all-features --release \ -- \ -D warnings + +# Install a file (full version). +# +# params: +# +# $1 : File to install. +# $2 : Directory path where file will be installed. +# $3 : Permissions to apply to the installed file. +define INSTALL_FILE_FULL + sudo install \ + --mode $3 \ + --owner $(KATA_INSTALL_OWNER) \ + --group $(KATA_INSTALL_GROUP) \ + -D $1 $2/$(notdir $1) || exit 1; +endef + +# Install a configuration file. +# +# params: +# +# $1 : File to install. +# $2 : Directory path where file will be installed. +define INSTALL_CONFIG + $(call INSTALL_FILE_FULL,$1,$2,$(KATA_INSTALL_CFG_PERMS)) +endef \ No newline at end of file From c8a9052063b6cf96267734a998d211b19ce314ff Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 15 Jun 2022 10:41:22 +0800 Subject: [PATCH 0082/1953] build: format files add Enter at the end of the file Fixes: #4379 Signed-off-by: Zhongtao Hu --- src/runtime-rs/arch/x86_64-options.mk | 2 +- src/runtime-rs/config/configuration-dragonball.toml.in | 2 +- utils.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime-rs/arch/x86_64-options.mk b/src/runtime-rs/arch/x86_64-options.mk index 522810824a4a..0e837f065781 100644 --- a/src/runtime-rs/arch/x86_64-options.mk +++ b/src/runtime-rs/arch/x86_64-options.mk @@ -12,4 +12,4 @@ CPUFEATURES := pmu=off QEMUCMD := qemu-system-x86_64 # dragonball binary name -DBCMD := dragonball \ No newline at end of file +DBCMD := dragonball diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index 57c4d2951cee..bda6a8d3a164 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -246,4 +246,4 @@ experimental=@DEFAULTEXPFEATURES@ # If enabled, user can run pprof tools with shim v2 process through kata-monitor. # (default: false) -# enable_pprof = true \ No newline at end of file +# enable_pprof = true diff --git a/utils.mk b/utils.mk index 858c14d493e0..a52bb7362d35 100644 --- a/utils.mk +++ b/utils.mk @@ -201,4 +201,4 @@ endef # $2 : Directory path where file will be installed. define INSTALL_CONFIG $(call INSTALL_FILE_FULL,$1,$2,$(KATA_INSTALL_CFG_PERMS)) -endef \ No newline at end of file +endef From 07231b2f3f19fa90250e7b94c03edcb9949b4150 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 28 Jun 2022 15:41:57 +0800 Subject: [PATCH 0083/1953] runtime-rs:refactor network model with netlink add unit test for tcfilter Fixes: #4289 Signed-off-by: Zhongtao Hu --- src/runtime-rs/crates/resource/Cargo.toml | 2 +- .../resource/src/network/network_model/mod.rs | 2 +- .../network/network_model/tc_filter_model.rs | 2 +- .../network_model/test_network_model.rs | 39 +++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/network/network_model/test_network_model.rs diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 7d97af016426..e5fe51bb2e5a 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -29,5 +29,5 @@ kata-types = { path = "../../../libs/kata-types" } kata-sys-util = { path = "../../../libs/kata-sys-util" } logging = { path = "../../../libs/logging" } oci = { path = "../../../libs/oci" } - +actix-rt = "2.7.0" [features] diff --git a/src/runtime-rs/crates/resource/src/network/network_model/mod.rs b/src/runtime-rs/crates/resource/src/network/network_model/mod.rs index 11cda538ca4d..848e032c0a1a 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/mod.rs @@ -7,7 +7,7 @@ pub mod none_model; pub mod route_model; pub mod tc_filter_model; - +pub mod test_network_model; use std::sync::Arc; use anyhow::{Context, Result}; diff --git a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs index c3bc3542e86c..6c014a7bc083 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs @@ -91,7 +91,7 @@ impl NetworkModel for TcFilterModel { } } -async fn fetch_index(handle: &Handle, name: &str) -> Result { +pub async fn fetch_index(handle: &Handle, name: &str) -> Result { let link = crate::network::network_pair::get_link_by_name(handle, name) .await .context("get link by name")?; diff --git a/src/runtime-rs/crates/resource/src/network/network_model/test_network_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/test_network_model.rs new file mode 100644 index 000000000000..bd1bb628f289 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/network_model/test_network_model.rs @@ -0,0 +1,39 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[cfg(test)] +mod tests { + use crate::network::{ + network_model::{tc_filter_model::fetch_index, TC_FILTER_NET_MODEL_STR}, + network_pair::NetworkPair, + }; + use anyhow::Context; + use scopeguard::defer; + #[actix_rt::test] + async fn test_tc_redirect_network() { + if let Ok((connection, handle, _)) = rtnetlink::new_connection().context("new connection") { + let thread_handler = tokio::spawn(connection); + defer!({ + thread_handler.abort(); + }); + + handle + .link() + .add() + .veth("foo".to_string(), "bar".to_string()); + + if let Ok(net_pair) = + NetworkPair::new(&handle, 1, "bar", TC_FILTER_NET_MODEL_STR, 2).await + { + if let Ok(index) = fetch_index(&handle, "bar").await { + assert!(net_pair.add_network_model().await.is_ok()); + assert!(net_pair.del_network_model().await.is_ok()); + assert!(handle.link().del(index).execute().await.is_ok()); + } + } + } + } +} From 7dad7c89f3d32f8699084cd086683941d914ea3f Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Tue, 28 Jun 2022 09:13:38 +0800 Subject: [PATCH 0084/1953] dragonball: update dbs-xxx dependency change to up-to-date commit ID Fixes: #4543 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/Cargo.toml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 3cc17de1ddde..d1b87beda4c9 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -51,9 +51,11 @@ hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] [patch.'crates-io'] -dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } -dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } -dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } -dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } -dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } -dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "127621db934af5ffba558e44b77afa00cdf62af6" } +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-boot = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-arch = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } From 648d285a2447b89e6fcb5b864149384785cd4707 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Tue, 28 Jun 2022 10:08:29 +0800 Subject: [PATCH 0085/1953] dragonball: add vcpu support for aarch64 add configure() function for aarch64 vcpu Fixes: #4543 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/vcpu/aarch64.rs | 36 +++++++++++++++++++++++++--- src/dragonball/src/vcpu/vcpu_impl.rs | 3 --- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/dragonball/src/vcpu/aarch64.rs b/src/dragonball/src/vcpu/aarch64.rs index 8e78efdc10ac..759774a62090 100644 --- a/src/dragonball/src/vcpu/aarch64.rs +++ b/src/dragonball/src/vcpu/aarch64.rs @@ -8,15 +8,18 @@ use std::sync::mpsc::{channel, Sender}; use std::sync::Arc; +use std::ops::Deref; use crate::IoManagerCached; use dbs_utils::time::TimestampUs; +use dbs_arch::regs; +use dbs_boot::get_fdt_addr; use kvm_ioctls::{VcpuFd, VmFd}; -use vm_memory::GuestAddress; +use vm_memory::{Address, GuestAddress, GuestAddressSpace}; use vmm_sys_util::eventfd::EventFd; use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuStateEvent}; +use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuStateEvent, VcpuError}; use crate::vcpu::VcpuConfig; #[allow(unused)] @@ -83,7 +86,34 @@ impl Vcpu { kernel_load_addr: Option, _pgtable_addr: Option, ) -> Result<()> { - // TODO: add arm vcpu configure() function. issue: #4445 + let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default(); + + // This reads back the kernel's preferred target type. + vm_fd + .get_preferred_target(&mut kvi) + .map_err(VcpuError::VcpuArmPreferredTarget)?; + // We already checked that the capability is supported. + kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2; + // Non-boot cpus are powered off initially. + if self.id > 0 { + kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF; + } + + self.fd.vcpu_init(&kvi).map_err(VcpuError::VcpuArmInit)?; + + if let Some(address) = kernel_load_addr { + regs::setup_regs( + &self.fd, + self.id, + address.raw_value(), + get_fdt_addr(vm_as.memory().deref()), + ) + .map_err(VcpuError::REGSConfiguration)?; + } + + self.mpidr = + regs::read_mpidr(&self.fd).map_err(VcpuError::REGSConfiguration)?; + Ok(()) } diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index 7c39ca280573..eab1a3a5faa2 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -452,9 +452,6 @@ impl Vcpu { Ok(VcpuEmulation::Handled) } VcpuExit::MmioWrite(addr, data) => { - #[cfg(target_arch = "aarch64")] - self.check_boot_complete_signal(addr, data); - let _ = self.io_mgr.mmio_write(addr, data); METRICS.vcpu.exit_mmio_write.inc(); Ok(VcpuEmulation::Handled) From 7120afe4ed43b148337545670c1e997272ba10a5 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Tue, 28 Jun 2022 16:47:54 +0800 Subject: [PATCH 0086/1953] dragonball: add vcpu test function for aarch64 add create_vcpu() function in vcpu test unit for aarch64 Fixes: #4445 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/vcpu/vcpu_impl.rs | 52 ++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index eab1a3a5faa2..d2711cace4ff 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -848,20 +848,52 @@ pub mod tests { (vcpu, rx) } - #[cfg(target_arch = "x86_64")] + #[cfg(target_arch = "aarch64")] + fn create_vcpu() -> (Vcpu, Receiver) { + // Call for kvm too frequently would cause error in some host kernel. + std::thread::sleep(std::time::Duration::from_millis(5)); + + let kvm = Kvm::new().unwrap(); + let vm = Arc::new(kvm.create_vm().unwrap()); + let kvm_context = KvmContext::new(Some(kvm.as_raw_fd())).unwrap(); + let vcpu_fd = Arc::new(vm.create_vcpu(0).unwrap()); + let io_manager = IoManagerCached::new(Arc::new(ArcSwap::new(Arc::new(IoManager::new())))); + let reset_event_fd = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let vcpu_state_event = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let (tx, rx) = channel(); + let time_stamp = TimestampUs::default(); + + let vcpu = Vcpu::new_aarch64( + 0, + vcpu_fd, + io_manager, + reset_event_fd, + vcpu_state_event, + tx, + time_stamp, + false, + ) + .unwrap(); + + (vcpu, rx) + } + #[test] fn test_vcpu_run_emulation() { let (mut vcpu, _) = create_vcpu(); - // Io in - *(EMULATE_RES.lock().unwrap()) = EmulationCase::IoIn; - let res = vcpu.run_emulation(); - assert!(matches!(res, Ok(VcpuEmulation::Handled))); - - // Io out - *(EMULATE_RES.lock().unwrap()) = EmulationCase::IoOut; - let res = vcpu.run_emulation(); - assert!(matches!(res, Ok(VcpuEmulation::Handled))); + #[cfg(target_arch = "x86_64")] + { + // Io in + *(EMULATE_RES.lock().unwrap()) = EmulationCase::IoIn; + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + + // Io out + *(EMULATE_RES.lock().unwrap()) = EmulationCase::IoOut; + let res = vcpu.run_emulation(); + assert!(matches!(res, Ok(VcpuEmulation::Handled))); + } // Mmio read *(EMULATE_RES.lock().unwrap()) = EmulationCase::MmioRead; From 527b73a8e50493acadfca22f0d6fa241d36770c3 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 21:38:49 +0800 Subject: [PATCH 0087/1953] dragonball: remove unused feature in AddressSpaceMgr log_dirty_pages is useless now and will be redesigned to support live migration in the future. Signed-off-by: wllenyj --- src/dragonball/src/address_space_manager.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dragonball/src/address_space_manager.rs b/src/dragonball/src/address_space_manager.rs index dc7650e97a09..2e4276626bac 100644 --- a/src/dragonball/src/address_space_manager.rs +++ b/src/dragonball/src/address_space_manager.rs @@ -27,7 +27,7 @@ use dbs_address_space::{ AddressSpaceRegionType, NumaNode, NumaNodeInfo, MPOL_MF_MOVE, MPOL_PREFERRED, }; use dbs_allocator::Constraint; -use kvm_bindings::{kvm_userspace_memory_region, KVM_MEM_LOG_DIRTY_PAGES}; +use kvm_bindings::kvm_userspace_memory_region; use kvm_ioctls::VmFd; use log::{debug, error, info, warn}; use nix::sys::mman; @@ -245,6 +245,11 @@ impl AddressSpaceMgr { self.address_space.is_some() } + /// Gets address space. + pub fn address_space(&self) -> Option<&AddressSpace> { + self.address_space.as_ref() + } + /// Create the address space for a virtual machine. /// /// This method is designed to be called when starting up a virtual machine instead of at @@ -393,11 +398,8 @@ impl AddressSpaceMgr { let host_addr = mmap_reg .get_host_address(MemoryRegionAddress(0)) .map_err(|_e| AddressManagerError::InvalidOperation)?; - let flags = if param.dirty_page_logging { - KVM_MEM_LOG_DIRTY_PAGES - } else { - 0 - }; + let flags = 0u32; + let mem_region = kvm_userspace_memory_region { slot: slot as u32, guest_phys_addr: reg.start_addr().raw_value(), From cfd5dae47ccc578c075b727248b54a6183226dcf Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 21:42:00 +0800 Subject: [PATCH 0088/1953] dragonball: add vm struct The vm struct to manage resources and control states of an virtual machine instance. Signed-off-by: wllenyj Signed-off-by: jingshan Signed-off-by: Liu Jiang Signed-off-by: Chao Wu --- .../src/device_manager/console_manager.rs | 10 + src/dragonball/src/device_manager/mod.rs | 76 ++- src/dragonball/src/error.rs | 117 +++- src/dragonball/src/vcpu/mod.rs | 1 + src/dragonball/src/vcpu/vcpu_impl.rs | 2 +- src/dragonball/src/vm/aarch64.rs | 148 +++++ src/dragonball/src/vm/kernel_config.rs | 2 +- src/dragonball/src/vm/mod.rs | 610 ++++++++++++++++++ src/dragonball/src/vm/x86_64.rs | 276 ++++++++ 9 files changed, 1236 insertions(+), 6 deletions(-) create mode 100644 src/dragonball/src/vm/aarch64.rs create mode 100644 src/dragonball/src/vm/x86_64.rs diff --git a/src/dragonball/src/device_manager/console_manager.rs b/src/dragonball/src/device_manager/console_manager.rs index 617b98b167b5..388eea78b8d9 100644 --- a/src/dragonball/src/device_manager/console_manager.rs +++ b/src/dragonball/src/device_manager/console_manager.rs @@ -351,6 +351,16 @@ pub struct DmesgWriter { logger: slog::Logger, } +impl DmesgWriter { + /// Creates a new instance. + pub fn new(logger: slog::Logger) -> Self { + Self { + buf: BytesMut::with_capacity(1024), + logger: logger.new(slog::o!("subsystem" => "dmesg")), + } + } +} + impl io::Write for DmesgWriter { /// 0000000 [ 0 . 0 3 4 9 1 6 ] R /// 5b 20 20 20 20 30 2e 30 33 34 39 31 36 5d 20 52 diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 2ad26e0d0191..011a1607dc83 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -8,6 +8,8 @@ use std::sync::{Arc, Mutex, MutexGuard}; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; +#[cfg(target_arch = "aarch64")] +use dbs_arch::{DeviceType, MMIODeviceInfo}; use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext}; use dbs_device::resources::Resource; use dbs_device::DeviceIo; @@ -20,6 +22,8 @@ use kvm_ioctls::VmFd; use dbs_device::resources::ResourceConstraint; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices as virtio; +#[cfg(feature = "virtio-vsock")] +use dbs_virtio_devices::vsock::backend::VsockInnerConnector; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::{ mmio::{ @@ -38,7 +42,8 @@ use dbs_upcall::{ use crate::address_space_manager::GuestAddressSpaceImpl; use crate::error::StartMicrovmError; use crate::resource_manager::ResourceManager; -use crate::vm::KernelConfigInfo; +use crate::vm::{KernelConfigInfo, Vm}; +use crate::IoManagerCached; /// Virtual machine console device manager. pub mod console_manager; @@ -240,6 +245,10 @@ impl DeviceOpContext { } } + pub(crate) fn create_boot_ctx(vm: &Vm, epoll_mgr: Option) -> Self { + Self::new(epoll_mgr, vm.device_manager(), None, None, false) + } + pub(crate) fn get_vm_as(&self) -> Result { match self.vm_as.as_ref() { Some(v) => Ok(v.clone()), @@ -303,6 +312,23 @@ impl DeviceOpContext { #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] impl DeviceOpContext { + pub(crate) fn create_hotplug_ctx(vm: &Vm, epoll_mgr: Option) -> Self { + let vm_as = vm.vm_as().expect("VM should have memory ready").clone(); + + let vm_config = vm.vm_config().clone(); + + let mut ctx = Self::new( + epoll_mgr, + vm.device_manager(), + Some(vm_as), + vm.vm_address_space().cloned(), + true, + ); + ctx.upcall_client = vm.upcall_client().clone(); + + ctx + } + fn call_hotplug_device( &self, req: DevMgrRequest, @@ -380,6 +406,8 @@ pub struct DeviceManager { pub(crate) legacy_manager: Option, #[cfg(feature = "virtio-vsock")] pub(crate) vsock_manager: VsockDeviceMgr, + #[cfg(target_arch = "aarch64")] + mmio_device_info: HashMap<(DeviceType, String), MMIODeviceInfo>, } impl DeviceManager { @@ -401,9 +429,16 @@ impl DeviceManager { legacy_manager: None, #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), + #[cfg(target_arch = "aarch64")] + mmio_device_info: HashMap::new(), } } + /// Get the underlying IoManager to dispatch IO read/write requests. + pub fn io_manager(&self) -> IoManagerCached { + IoManagerCached::new(self.io_manager.clone()) + } + /// Create the underline interrupt manager for the device manager. pub fn create_interrupt_manager(&mut self) -> Result<()> { self.irq_manager @@ -494,6 +529,12 @@ impl DeviceManager { self.con_manager.reset_console() } + #[cfg(target_arch = "aarch64")] + /// Return mmio device info for FDT build. + pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { + Some(&self.mmio_device_info) + } + /// Create all registered devices when booting the associated virtual machine. pub fn create_devices( &mut self, @@ -524,6 +565,21 @@ impl DeviceManager { Ok(()) } + /// Start all registered devices when booting the associated virtual machine. + pub fn start_devices(&mut self) -> std::result::Result<(), StartMicrovmError> { + Ok(()) + } + + /// Remove all devices when shutdown the associated virtual machine + pub fn remove_devices( + &mut self, + _vm_as: GuestAddressSpaceImpl, + _epoll_mgr: EpollManager, + _address_space: Option<&AddressSpace>, + ) -> Result<()> { + Ok(()) + } + #[cfg(target_arch = "x86_64")] /// Get the underlying eventfd for vm exit notification. pub fn get_reset_eventfd(&self) -> Result { @@ -689,3 +745,21 @@ impl DeviceManager { } } } + +#[cfg(feature = "hotplug")] +impl DeviceManager { + /// Get Unix Domain Socket path for the vsock device. + pub fn get_vsock_inner_connector(&mut self) -> Option { + #[cfg(feature = "virtio-vsock")] + { + self.vsock_manager + .get_default_connector() + .map(|d| Some(d)) + .unwrap_or(None) + } + #[cfg(not(feature = "virtio-vsock"))] + { + return None; + } + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 2858103a4869..b201c1d779ed 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,7 +12,10 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; +use crate::address_space_manager; use crate::device_manager; +use crate::vcpu; +use crate::vm; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; @@ -23,8 +26,20 @@ pub type Result = std::result::Result; /// of the host (for example if Dragonball doesn't have permissions to open the KVM fd). #[derive(Debug, thiserror::Error)] pub enum Error { + /// Empty AddressSpace from parameters. + #[error("Empty AddressSpace from parameters")] + AddressSpace, + + /// The zero page extends past the end of guest_mem. + #[error("the guest zero page extends past the end of guest memory")] + ZeroPagePastRamEnd, + + /// Error writing the zero page of guest memory. + #[error("failed to write to guest zero page")] + ZeroPageSetup, + /// Failure occurs in issuing KVM ioctls and errors will be returned from kvm_ioctls lib. - #[error("failure in issuing KVM ioctl command")] + #[error("failure in issuing KVM ioctl command: {0}")] Kvm(#[source] kvm_ioctls::Error), /// The host kernel reports an unsupported KVM API version. @@ -32,17 +47,30 @@ pub enum Error { KvmApiVersion(i32), /// Cannot initialize the KVM context due to missing capabilities. - #[error("missing KVM capability")] + #[error("missing KVM capability: {0:?}")] KvmCap(kvm_ioctls::Cap), #[cfg(target_arch = "x86_64")] - #[error("failed to configure MSRs")] + #[error("failed to configure MSRs: {0:?}")] /// Cannot configure MSRs GuestMSRs(dbs_arch::msr::Error), /// MSR inner error #[error("MSR inner error")] Msr(vmm_sys_util::fam::Error), + + /// Error writing MP table to memory. + #[cfg(target_arch = "x86_64")] + #[error("failed to write MP table to guest memory: {0}")] + MpTableSetup(#[source] dbs_boot::mptable::Error), + + /// Fail to boot system + #[error("failed to boot system: {0}")] + BootSystem(#[source] dbs_boot::Error), + + /// Cannot open the VM file descriptor. + #[error(transparent)] + Vm(vm::VmError), } /// Errors associated with starting the instance. @@ -52,6 +80,48 @@ pub enum StartMicrovmError { #[error("failure while reading from EventFd file descriptor")] EventFd, + /// The start command was issued more than once. + #[error("the virtual machine is already running")] + MicroVMAlreadyRunning, + + /// Cannot start the VM because the kernel was not configured. + #[error("cannot start the virtual machine without kernel configuration")] + MissingKernelConfig, + + #[cfg(feature = "hotplug")] + /// Upcall initialize miss vsock device. + #[error("the upcall client needs a virtio-vsock device for communication")] + UpcallMissVsock, + + /// Upcall is not ready + #[error("the upcall client is not ready")] + UpcallNotReady, + + /// Configuration passed in is invalidate. + #[error("invalid virtual machine configuration: {0} ")] + ConfigureInvalid(String), + + /// This error is thrown by the minimal boot loader implementation. + /// It is related to a faulty memory configuration. + #[error("failure while configuring boot information for the virtual machine: {0}")] + ConfigureSystem(#[source] Error), + + /// Cannot configure the VM. + #[error("failure while configuring the virtual machine: {0}")] + ConfigureVm(#[source] vm::VmError), + + /// Cannot load initrd. + #[error("cannot load Initrd into guest memory: {0}")] + InitrdLoader(#[from] LoadInitrdError), + + /// Cannot load kernel due to invalid memory configuration or invalid kernel image. + #[error("cannot load guest kernel into guest memory: {0}")] + KernelLoader(#[source] linux_loader::loader::Error), + + /// Cannot load command line string. + #[error("failure while configuring guest kernel commandline: {0}")] + LoadCommandline(#[source] linux_loader::loader::Error), + /// The device manager was not configured. #[error("the device manager failed to manage devices: {0}")] DeviceManager(#[source] device_manager::DeviceMgrError), @@ -69,4 +139,45 @@ pub enum StartMicrovmError { /// Cannot initialize a MMIO Vsock Device or add a device to the MMIO Bus. #[error("failure while registering virtio-vsock device: {0}")] RegisterVsockDevice(#[source] device_manager::DeviceMgrError), + + /// Address space manager related error, e.g.cannot access guest address space manager. + #[error("address space manager related error: {0}")] + AddressManagerError(#[source] address_space_manager::AddressManagerError), + + /// Cannot create a new vCPU file descriptor. + #[error("vCPU related error: {0}")] + Vcpu(#[source] vcpu::VcpuManagerError), + + #[cfg(feature = "hotplug")] + /// Upcall initialize Error. + #[error("failure while initializing the upcall client: {0}")] + UpcallInitError(#[source] dbs_upcall::UpcallClientError), + + #[cfg(feature = "hotplug")] + /// Upcall connect Error. + #[error("failure while connecting the upcall client: {0}")] + UpcallConnectError(#[source] dbs_upcall::UpcallClientError), +} + +/// Errors associated with starting the instance. +#[derive(Debug, thiserror::Error)] +pub enum StopMicrovmError { + /// Guest memory has not been initialized. + #[error("Guest memory has not been initialized")] + GuestMemoryNotInitialized, + + /// Cannnot remove devices + #[error("Failed to remove devices in device_manager {0}")] + DeviceManager(#[source] device_manager::DeviceMgrError), +} + +/// Errors associated with loading initrd +#[derive(Debug, thiserror::Error)] +pub enum LoadInitrdError { + /// Cannot load initrd due to an invalid memory configuration. + #[error("failed to load the initrd image to guest memory")] + LoadInitrd, + /// Cannot load initrd due to an invalid image. + #[error("failed to read the initrd image: {0}")] + ReadInitrd(#[source] std::io::Error), } diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs index d1075e734d38..fe66883245de 100644 --- a/src/dragonball/src/vcpu/mod.rs +++ b/src/dragonball/src/vcpu/mod.rs @@ -6,6 +6,7 @@ mod sm; pub mod vcpu_impl; pub mod vcpu_manager; +pub use vcpu_manager::{VcpuManager, VcpuManagerError}; #[cfg(target_arch = "x86_64")] use dbs_arch::cpuid::VpmuFeatureLevel; diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index d2711cace4ff..2dbccf66d059 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -964,7 +964,7 @@ pub mod tests { #[cfg(target_arch = "x86_64")] #[test] fn test_vcpu_check_io_port_info() { - let (vcpu, receiver) = create_vcpu(); + let (vcpu, _receiver) = create_vcpu(); // boot complete signal let res = vcpu diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs new file mode 100644 index 000000000000..02af4a69171f --- /dev/null +++ b/src/dragonball/src/vm/aarch64.rs @@ -0,0 +1,148 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::collections::HashMap; +use std::ops::Deref; + +use dbs_arch::gic::GICDevice; +use dbs_arch::{DeviceInfoForFDT, DeviceType}; +use dbs_boot::InitrdConfig; +use dbs_utils::epoll_manager::EpollManager; +use dbs_utils::time::TimestampUs; +use std::fmt::Debug; +use vm_memory::{GuestAddressSpace, GuestMemory}; +use vmm_sys_util::eventfd::EventFd; + +use super::{Vm, VmError}; +use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; +use crate::error::Error; +use crate::StartMicrovmError; +use linux_loader::loader::Cmdline; + +/// Configures the system and should be called once per vm before starting vcpu threads. +/// For aarch64, we only setup the FDT. +/// +/// # Arguments +/// +/// * `guest_mem` - The memory to be used by the guest. +/// * `cmdline` - The kernel commandline. +/// * `vcpu_mpidr` - Array of MPIDR register values per vcpu. +/// * `device_info` - A hashmap containing the attached devices for building FDT device nodes. +/// * `gic_device` - The GIC device. +/// * `initrd` - Information about an optional initrd. +pub fn configure_system( + guest_mem: &M, + cmdline: &str, + vcpu_mpidr: Vec, + device_info: Option<&HashMap<(DeviceType, String), T>>, + gic_device: &Box, + initrd: &Option, +) -> super::Result<()> { + dbs_boot::fdt::create_fdt( + guest_mem, + vcpu_mpidr, + cmdline, + device_info, + gic_device, + initrd, + ) + .map_err(Error::BootSystem)?; + Ok(()) +} + +#[cfg(target_arch = "aarch64")] +impl Vm { + /// Gets a reference to the irqchip of the VM + pub fn get_irqchip(&self) -> &Box { + &self.irqchip_handle.as_ref().unwrap() + } + + /// Initialize the virtual machine instance. + /// + /// It initialize the virtual machine instance by: + /// 1) initialize virtual machine global state and configuration. + /// 2) create system devices, such as interrupt controller. + /// 3) create and start IO devices, such as serial, console, block, net, vsock etc. + /// 4) create and initialize vCPUs. + /// 5) configure CPU power management features. + /// 6) load guest kernel image. + pub fn init_microvm( + &mut self, + epoll_mgr: EpollManager, + vm_as: GuestAddressSpaceImpl, + request_ts: TimestampUs, + ) -> std::result::Result<(), StartMicrovmError> { + let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; + // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before + // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP + // was already initialized. + // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. + + let reset_eventfd = + EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?; + self.reset_eventfd = Some( + reset_eventfd + .try_clone() + .map_err(|_| StartMicrovmError::EventFd)?, + ); + + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .set_reset_event_fd(reset_eventfd); + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) + .map_err(StartMicrovmError::Vcpu)?; + + self.setup_interrupt_controller()?; + self.init_devices(epoll_mgr)?; + + Ok(()) + } + + /// Creates the irq chip in-kernel device model. + pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { + let vcpu_count = self.vm_config.vcpu_count; + + self.irqchip_handle = Some( + dbs_arch::gic::create_gic(&self.fd, vcpu_count.into()) + .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, + ); + + Ok(()) + } + + /// Execute system architecture specific configurations. + /// + /// 1) set guest kernel boot parameters + /// 2) setup FDT data structs. + pub fn configure_system_arch( + &self, + vm_memory: &GuestMemoryImpl, + cmdline: &Cmdline, + initrd: Option, + ) -> std::result::Result<(), StartMicrovmError> { + let vcpu_manager = self.vcpu_manager().map_err(StartMicrovmError::Vcpu)?; + let vcpu_mpidr = vcpu_manager + .vcpus() + .into_iter() + .map(|cpu| cpu.get_mpidr()) + .collect(); + + let guest_memory = vm_memory.memory(); + configure_system( + guest_memory, + cmdline.as_str(), + vcpu_mpidr, + self.device_manager.get_mmio_device_info(), + self.get_irqchip(), + &initrd, + ) + .map_err(StartMicrovmError::ConfigureSystem) + } +} diff --git a/src/dragonball/src/vm/kernel_config.rs b/src/dragonball/src/vm/kernel_config.rs index f266b48c4d52..466fd8849cc9 100644 --- a/src/dragonball/src/vm/kernel_config.rs +++ b/src/dragonball/src/vm/kernel_config.rs @@ -8,7 +8,7 @@ pub struct KernelConfigInfo { /// The descriptor to the kernel file. kernel_file: File, /// The descriptor to the initrd file, if there is one - initrd_file: Option, + pub initrd_file: Option, /// The commandline for guest kernel. cmdline: linux_loader::cmdline::Cmdline, } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index e09ec316bb41..db10e593e614 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -1,11 +1,69 @@ // Copyright (C) 2021 Alibaba Cloud. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::io::{self, Read, Seek, SeekFrom}; +use std::ops::Deref; +use std::os::unix::io::RawFd; +use std::sync::{Arc, Mutex, RwLock}; + +use dbs_address_space::AddressSpace; +#[cfg(target_arch = "aarch64")] +use dbs_arch::gic::GICDevice; +use dbs_boot::InitrdConfig; +#[cfg(feature = "hotplug")] +use dbs_upcall::{DevMgrService, UpcallClient}; +use dbs_utils::epoll_manager::EpollManager; +use dbs_utils::time::TimestampUs; +use kvm_ioctls::VmFd; +use linux_loader::loader::{KernelLoader, KernelLoaderResult}; +use seccompiler::BpfProgram; use serde_derive::{Deserialize, Serialize}; +use slog::{error, info}; +use vm_memory::{Bytes, GuestAddress, GuestAddressSpace}; +use vmm_sys_util::eventfd::EventFd; + +use crate::address_space_manager::{ + AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl, + GuestMemoryImpl, +}; +use crate::api::v1::{InstanceInfo, InstanceState}; +use crate::device_manager::console_manager::DmesgWriter; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::error::{LoadInitrdError, Result, StartMicrovmError, StopMicrovmError}; +use crate::kvm_context::KvmContext; +use crate::resource_manager::ResourceManager; +use crate::vcpu::{VcpuManager, VcpuManagerError}; +#[cfg(target_arch = "aarch64")] +use dbs_arch::gic::Error as GICError; mod kernel_config; pub use self::kernel_config::KernelConfigInfo; +#[cfg(target_arch = "aarch64")] +#[path = "aarch64.rs"] +mod aarch64; + +#[cfg(target_arch = "x86_64")] +#[path = "x86_64.rs"] +mod x86_64; + +/// Errors associated with virtual machine instance related operations. +#[derive(Debug, thiserror::Error)] +pub enum VmError { + /// Cannot configure the IRQ. + #[error("failed to configure IRQ fot the virtual machine: {0}")] + Irq(#[source] kvm_ioctls::Error), + + /// Cannot configure the microvm. + #[error("failed to initialize the virtual machine: {0}")] + VmSetup(#[source] kvm_ioctls::Error), + + /// Cannot setup GIC + #[cfg(target_arch = "aarch64")] + #[error("failed to configure GIC")] + SetupGIC(GICError), +} + /// Configuration information for user defined NUMA nodes. #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] pub struct NumaRegionInfo { @@ -94,3 +152,555 @@ impl Default for VmConfigInfo { } } } + +/// Struct to manage resources and control states of an virtual machine instance. +/// +/// An `Vm` instance holds a resources assigned to a virtual machine instance, such as CPU, memory, +/// devices etc. When an `Vm` instance gets deconstructed, all resources assigned should be +/// released. +/// +/// We have explicit build the object model as: +/// |---Vmm API Server--<-1:1-> HTTP API Server +/// | |----------<-1:1-> Shimv2/CRI API Server +/// | +/// Vmm <-1:N-> Vm <-1:1-> Address Space Manager <-1:N-> GuestMemory +/// ^ ^---1:1-> Device Manager <-1:N-> Device +/// | ^---1:1-> Resource Manager +/// | ^---1:N-> Vcpu +/// |---<-1:N-> Event Manager +pub struct Vm { + fd: Arc, + kvm: KvmContext, + + address_space: AddressSpaceMgr, + device_manager: DeviceManager, + epoll_manager: EpollManager, + resource_manager: Arc, + vcpu_manager: Option>>, + logger: slog::Logger, + /// Config of virtual machine + vm_config: VmConfigInfo, + kernel_config: Option, + shared_info: Arc>, + reset_eventfd: Option, + dmesg_fifo: Option>, + start_instance_request_ts: u64, + start_instance_request_cpu_ts: u64, + start_instance_downtime: u64, + + // Arm specific fields. + // On aarch64 we need to keep around the fd obtained by creating the VGIC device. + #[cfg(target_arch = "aarch64")] + irqchip_handle: Option>, + + #[cfg(feature = "hotplug")] + upcall_client: Option>>, +} + +impl Vm { + /// Constructs a new `Vm` instance using the given `Kvm` instance. + pub fn new( + kvm_fd: Option, + api_shared_info: Arc>, + epoll_manager: EpollManager, + ) -> Result { + let id = api_shared_info.read().unwrap().id.clone(); + let logger = slog_scope::logger().new(slog::o!("id" => id)); + + let kvm = KvmContext::new(kvm_fd)?; + let fd = Arc::new(kvm.create_vm()?); + + let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots()))); + + let device_manager = DeviceManager::new( + fd.clone(), + resource_manager.clone(), + epoll_manager.clone(), + &logger, + ); + + Ok(Vm { + fd, + kvm, + address_space: AddressSpaceMgr::default(), + device_manager, + epoll_manager, + resource_manager, + vcpu_manager: None, + logger, + vm_config: Default::default(), + kernel_config: None, + shared_info: api_shared_info, + reset_eventfd: None, + dmesg_fifo: None, + start_instance_request_ts: 0, + start_instance_request_cpu_ts: 0, + start_instance_downtime: 0, + #[cfg(target_arch = "aarch64")] + irqchip_handle: None, + #[cfg(feature = "hotplug")] + upcall_client: None, + }) + } + + /// Gets a reference to the kvm file descriptor owned by this VM. + pub fn vm_fd(&self) -> &VmFd { + &self.fd + } + + /// Gets a reference to the address_space.address_space for guest memory owned by this VM. + pub fn vm_address_space(&self) -> Option<&AddressSpace> { + self.address_space.get_address_space() + } + + /// Gets a reference to the device manager by this VM. + pub fn device_manager(&self) -> &DeviceManager { + &self.device_manager + } + + /// Gets a reference to the address space for guest memory owned by this VM. + /// + /// Note that `GuestMemory` does not include any device memory that may have been added after + /// this VM was constructed. + pub fn vm_as(&self) -> Option<&GuestAddressSpaceImpl> { + self.address_space.get_vm_as() + } + + /// Get a immutable reference to the virtual machine configuration information. + pub fn vm_config(&self) -> &VmConfigInfo { + &self.vm_config + } + + /// Set the virtual machine configuration information. + pub fn set_vm_config(&mut self, config: VmConfigInfo) { + self.vm_config = config; + } + + /// Set guest kernel boot configurations. + pub fn set_kernel_config(&mut self, kernel_config: KernelConfigInfo) { + self.kernel_config = Some(kernel_config); + } + + /// Get virtual machine shared instance information. + pub fn shared_info(&self) -> &Arc> { + &self.shared_info + } + + /// Get a reference to EpollManager. + pub fn epoll_manager(&self) -> &EpollManager { + &self.epoll_manager + } + + /// Get eventfd for exit notification. + pub fn get_reset_eventfd(&self) -> Option<&EventFd> { + self.reset_eventfd.as_ref() + } + + /// Check whether the VM has been initialized. + pub fn is_vm_initialized(&self) -> bool { + let instance_state = { + // Use expect() to crash if the other thread poisoned this lock. + let shared_info = self.shared_info.read() + .expect("Failed to determine if instance is initialized because shared info couldn't be read due to poisoned lock"); + shared_info.state + }; + instance_state != InstanceState::Uninitialized + } + + /// Check whether the VM instance is running. + pub fn is_vm_running(&self) -> bool { + let instance_state = { + // Use expect() to crash if the other thread poisoned this lock. + let shared_info = self.shared_info.read() + .expect("Failed to determine if instance is initialized because shared info couldn't be read due to poisoned lock"); + shared_info.state + }; + instance_state == InstanceState::Running + } + + /// returns true if system upcall service is ready + pub fn is_upcall_client_ready(&self) -> bool { + #[cfg(feature = "hotplug")] + { + if let Some(upcall_client) = self.upcall_client() { + return upcall_client.is_ready(); + } + } + + false + } + + /// Create device operation context. + /// vm is not running, return false + /// vm is running, but hotplug feature is not enable, return error + /// vm is running, but upcall initialize failed, return error + /// vm is running, upcall initialize OK, return true + pub fn create_device_op_context( + &mut self, + epoll_mgr: Option, + ) -> std::result::Result { + if !self.is_vm_initialized() { + Ok(DeviceOpContext::create_boot_ctx(self, epoll_mgr)) + } else { + #[cfg(feature = "hotplug")] + { + if self.upcall_client().is_none() { + Err(StartMicrovmError::UpcallMissVsock) + } else if self.is_upcall_client_ready() { + Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) + } else { + Err(StartMicrovmError::UpcallNotReady) + } + } + #[cfg(not(feature = "hotplug"))] + { + Err(StartMicrovmError::MicroVMAlreadyRunning) + } + } + } + + /// Save VM instance exit state + pub fn vm_exit(&self, exit_code: i32) { + if let Ok(mut info) = self.shared_info.write() { + info.state = InstanceState::Exited(exit_code); + } else { + error!( + self.logger, + "Failed to save exit state, couldn't be written due to poisoned lock" + ); + } + } + + /// Reset the console into canonical mode. + pub fn reset_console(&self) -> std::result::Result<(), DeviceMgrError> { + self.device_manager.reset_console() + } + + fn get_dragonball_info(&self) -> (String, String) { + let guard = self.shared_info.read().unwrap(); + let instance_id = guard.id.clone(); + let dragonball_version = guard.vmm_version.clone(); + + (dragonball_version, instance_id) + } + + fn init_dmesg_logger(&mut self) { + let writer = self.dmesg_logger(); + self.dmesg_fifo = Some(writer); + } + + /// dmesg write to logger + pub fn dmesg_logger(&self) -> Box { + Box::new(DmesgWriter::new(self.logger.clone())) + } + + pub(crate) fn check_health(&self) -> std::result::Result<(), StartMicrovmError> { + if self.kernel_config.is_none() { + return Err(StartMicrovmError::MissingKernelConfig); + } + Ok(()) + } + + pub(crate) fn init_vcpu_manager( + &mut self, + vm_as: GuestAddressSpaceImpl, + vcpu_seccomp_filter: BpfProgram, + ) -> std::result::Result<(), VcpuManagerError> { + let vcpu_manager = VcpuManager::new( + self.fd.clone(), + &self.kvm, + &self.vm_config, + vm_as, + vcpu_seccomp_filter, + self.shared_info.clone(), + self.device_manager.io_manager(), + self.epoll_manager.clone(), + )?; + self.vcpu_manager = Some(vcpu_manager); + + Ok(()) + } + + /// get the cpu manager's reference + pub fn vcpu_manager( + &self, + ) -> std::result::Result, VcpuManagerError> { + self.vcpu_manager + .as_ref() + .ok_or(VcpuManagerError::VcpuManagerNotInitialized) + .map(|mgr| mgr.lock().unwrap()) + } + + /// Pause all vcpus and record the instance downtime + pub fn pause_all_vcpus_with_downtime(&mut self) -> std::result::Result<(), VcpuManagerError> { + let ts = TimestampUs::default(); + self.start_instance_downtime = ts.time_us; + + self.vcpu_manager()?.pause_all_vcpus()?; + + Ok(()) + } + + /// Resume all vcpus and calc the intance downtime + pub fn resume_all_vcpus_with_downtime(&mut self) -> std::result::Result<(), VcpuManagerError> { + self.vcpu_manager()?.resume_all_vcpus()?; + + Ok(()) + } + + pub(crate) fn init_guest_memory(&mut self) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: initializing guest memory..."); + + // We are not allowing reinitialization of vm guest memory. + if self.address_space.is_initialized() { + return Ok(()); + } + // vcpu boot up require local memory. reserve 100 MiB memory + let mem_size = (self.vm_config.mem_size_mib as u64) << 20; + let reserve_memory_bytes = self.vm_config.reserve_memory_bytes; + if reserve_memory_bytes > (mem_size >> 1) as u64 { + return Err(StartMicrovmError::ConfigureInvalid(String::from( + "invalid reserve_memory_bytes", + ))); + } + + let mem_type = self.vm_config.mem_type.clone(); + let mut mem_file_path = String::from(""); + if mem_type == "hugetlbfs" { + let shared_info = self.shared_info.read() + .expect("Failed to determine if instance is initialized because shared info couldn't be read due to poisoned lock"); + mem_file_path.push_str("/dragonball/"); + mem_file_path.push_str(shared_info.id.as_str()); + } + + // init default regions. + let mut numa_regions = Vec::with_capacity(1); + let mut vcpu_ids: Vec = Vec::new(); + + for i in 0..self.vm_config().max_vcpu_count { + vcpu_ids.push(i as u32); + } + let numa_node = NumaRegionInfo { + size: self.vm_config.mem_size_mib as u64, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids, + }; + numa_regions.push(numa_node); + + info!( + self.logger, + "VM: mem_type:{} mem_file_path:{}, mem_size:{}, reserve_memory_bytes:{}, \ + numa_regions:{:?}", + mem_type, + mem_file_path, + mem_size, + reserve_memory_bytes, + numa_regions, + ); + + let mut address_space_param = AddressSpaceMgrBuilder::new(&mem_type, &mem_file_path) + .map_err(StartMicrovmError::AddressManagerError)?; + address_space_param.set_kvm_vm_fd(self.fd.clone()); + self.address_space + .create_address_space(&self.resource_manager, &numa_regions, address_space_param) + .map_err(StartMicrovmError::AddressManagerError)?; + + info!(self.logger, "VM: initializing guest memory done"); + Ok(()) + } + + fn init_devices( + &mut self, + epoll_manager: EpollManager, + ) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: initializing devices ..."); + + let com1_sock_path = self.vm_config.serial_path.clone(); + let kernel_config = self + .kernel_config + .as_mut() + .ok_or(StartMicrovmError::MissingKernelConfig)?; + + info!(self.logger, "VM: create interrupt manager"); + self.device_manager + .create_interrupt_manager() + .map_err(StartMicrovmError::DeviceManager)?; + + info!(self.logger, "VM: create devices"); + let vm_as = + self.address_space + .get_vm_as() + .ok_or(StartMicrovmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + ))?; + self.device_manager.create_devices( + vm_as.clone(), + epoll_manager, + kernel_config, + com1_sock_path, + self.dmesg_fifo.take(), + self.address_space.address_space(), + )?; + + info!(self.logger, "VM: start devices"); + self.device_manager.start_devices()?; + + info!(self.logger, "VM: initializing devices done"); + Ok(()) + } + + /// Remove devices when shutdown vm + pub fn remove_devices(&mut self) -> std::result::Result<(), StopMicrovmError> { + info!(self.logger, "VM: remove devices"); + let vm_as = self + .address_space + .get_vm_as() + .ok_or(StopMicrovmError::GuestMemoryNotInitialized)?; + + self.device_manager + .remove_devices( + vm_as.clone(), + self.epoll_manager.clone(), + self.address_space.address_space(), + ) + .map_err(StopMicrovmError::DeviceManager) + } + + fn load_kernel( + &mut self, + vm_memory: &GuestMemoryImpl, + ) -> std::result::Result { + // This is the easy way out of consuming the value of the kernel_cmdline. + + let kernel_config = self + .kernel_config + .as_mut() + .ok_or(StartMicrovmError::MissingKernelConfig)?; + + let high_mem_addr = GuestAddress(dbs_boot::get_kernel_start()); + + #[cfg(target_arch = "x86_64")] + return linux_loader::loader::elf::Elf::load( + vm_memory, + None, + kernel_config.kernel_file_mut(), + Some(high_mem_addr), + ) + .map_err(StartMicrovmError::KernelLoader); + + #[cfg(target_arch = "aarch64")] + return linux_loader::loader::pe::PE::load( + vm_memory, + Some(GuestAddress(dbs_boot::get_kernel_start())), + kernel_config.kernel_file_mut(), + Some(high_mem_addr), + ) + .map_err(StartMicrovmError::KernelLoader); + } + + /// Loads the initrd from a file into the given memory slice. + /// + /// * `vm_memory` - The guest memory the initrd is written to. + /// * `image` - The initrd image. + /// + /// Returns the result of initrd loading + fn load_initrd( + &self, + vm_memory: &GuestMemoryImpl, + image: &mut F, + ) -> std::result::Result + where + F: Read + Seek, + { + use crate::error::LoadInitrdError::*; + + let size: usize; + // Get the image size + match image.seek(SeekFrom::End(0)) { + Err(e) => return Err(ReadInitrd(e)), + Ok(0) => { + return Err(ReadInitrd(io::Error::new( + io::ErrorKind::InvalidData, + "Initrd image seek returned a size of zero", + ))) + } + Ok(s) => size = s as usize, + }; + // Go back to the image start + image.seek(SeekFrom::Start(0)).map_err(ReadInitrd)?; + + // Get the target address + let address = dbs_boot::initrd_load_addr(vm_memory, size as u64).map_err(|_| LoadInitrd)?; + + // Load the image into memory + vm_memory + .read_from(GuestAddress(address), image, size) + .map_err(|_| LoadInitrd)?; + + Ok(InitrdConfig { + address: GuestAddress(address), + size, + }) + } + + fn init_configure_system( + &mut self, + vm_as: &GuestAddressSpaceImpl, + ) -> std::result::Result<(), StartMicrovmError> { + let vm_memory = vm_as.memory(); + let kernel_config = self + .kernel_config + .as_ref() + .ok_or(StartMicrovmError::MissingKernelConfig)?; + //let cmdline = kernel_config.cmdline.clone(); + let initrd: Option = match &kernel_config.initrd_file { + Some(f) => { + let initrd_file = f.try_clone(); + if initrd_file.is_err() { + return Err(StartMicrovmError::InitrdLoader( + LoadInitrdError::ReadInitrd(io::Error::from(io::ErrorKind::InvalidData)), + )); + } + let res = self.load_initrd(vm_memory.deref(), &mut initrd_file.unwrap())?; + Some(res) + } + None => None, + }; + + self.configure_system_arch(vm_memory.deref(), kernel_config.kernel_cmdline(), initrd) + } +} + +#[cfg(feature = "hotplug")] +impl Vm { + /// Get upcall client. + pub fn upcall_client(&self) -> &Option>> { + &self.upcall_client + } + + /// initialize upcall client for guest os + fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + // get vsock inner connector for upcall + let inner_connector = self + .device_manager + .get_vsock_inner_connector() + .ok_or(StartMicrovmError::UpcallMissVsock)?; + + let mut upcall_client = UpcallClient::new( + inner_connector, + self.epoll_manager.clone(), + DevMgrService::default(), + ) + .map_err(StartMicrovmError::UpcallInitError)?; + + upcall_client + .connect() + .map_err(StartMicrovmError::UpcallConnectError)?; + + self.upcall_client = Some(Arc::new(upcall_client)); + + info!(self.logger, "upcall client init success"); + Ok(()) + } +} diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs new file mode 100644 index 000000000000..a55996282174 --- /dev/null +++ b/src/dragonball/src/vm/x86_64.rs @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::convert::TryInto; +use std::mem; +use std::ops::Deref; + +use dbs_address_space::AddressSpace; +use dbs_boot::{add_e820_entry, bootparam, layout, mptable, BootParamsWrapper, InitrdConfig}; +use dbs_utils::epoll_manager::EpollManager; +use dbs_utils::time::TimestampUs; +use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY}; +use slog::info; +use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; + +use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; +use crate::error::{Error, Result, StartMicrovmError}; +use crate::vm::{Vm, VmError}; + +use linux_loader::cmdline::Cmdline; + +/// Configures the system and should be called once per vm before starting vcpu +/// threads. +/// +/// # Arguments +/// +/// * `guest_mem` - The memory to be used by the guest. +/// * `cmdline_addr` - Address in `guest_mem` where the kernel command line was +/// loaded. +/// * `cmdline_size` - Size of the kernel command line in bytes including the +/// null terminator. +/// * `initrd` - Information about where the ramdisk image was loaded in the +/// `guest_mem`. +/// * `boot_cpus` - Number of virtual CPUs the guest will have at boot time. +/// * `max_cpus` - Max number of virtual CPUs the guest will have. +/// * `rsv_mem_bytes` - Reserve memory from microVM.. +#[allow(clippy::too_many_arguments)] +pub fn configure_system( + guest_mem: &M, + address_space: Option<&AddressSpace>, + cmdline_addr: GuestAddress, + cmdline_size: usize, + initrd: &Option, + boot_cpus: u8, + max_cpus: u8, + rsv_mem_bytes: u64, +) -> super::Result<()> { + const KERNEL_BOOT_FLAG_MAGIC: u16 = 0xaa55; + const KERNEL_HDR_MAGIC: u32 = 0x5372_6448; + const KERNEL_LOADER_OTHER: u8 = 0xff; + const KERNEL_MIN_ALIGNMENT_BYTES: u32 = 0x0100_0000; // Must be non-zero. + + let mmio_start = GuestAddress(layout::MMIO_LOW_START); + let mmio_end = GuestAddress(layout::MMIO_LOW_END); + let himem_start = GuestAddress(layout::HIMEM_START); + + // Note that this puts the mptable at the last 1k of Linux's 640k base RAM + mptable::setup_mptable(guest_mem, boot_cpus, max_cpus).map_err(Error::MpTableSetup)?; + + let mut params: BootParamsWrapper = BootParamsWrapper(bootparam::boot_params::default()); + + params.0.hdr.type_of_loader = KERNEL_LOADER_OTHER; + params.0.hdr.boot_flag = KERNEL_BOOT_FLAG_MAGIC; + params.0.hdr.header = KERNEL_HDR_MAGIC; + params.0.hdr.cmd_line_ptr = cmdline_addr.raw_value() as u32; + params.0.hdr.cmdline_size = cmdline_size as u32; + params.0.hdr.kernel_alignment = KERNEL_MIN_ALIGNMENT_BYTES; + if let Some(initrd_config) = initrd { + params.0.hdr.ramdisk_image = initrd_config.address.raw_value() as u32; + params.0.hdr.ramdisk_size = initrd_config.size as u32; + } + + add_e820_entry(&mut params.0, 0, layout::EBDA_START, bootparam::E820_RAM) + .map_err(Error::BootSystem)?; + + let mem_end = address_space.ok_or(Error::AddressSpace)?.last_addr(); + if mem_end < mmio_start { + add_e820_entry( + &mut params.0, + himem_start.raw_value() as u64, + // it's safe to use unchecked_offset_from because + // mem_end > himem_start + mem_end.unchecked_offset_from(himem_start) as u64 + 1, + bootparam::E820_RAM, + ) + .map_err(Error::BootSystem)?; + } else { + add_e820_entry( + &mut params.0, + himem_start.raw_value(), + // it's safe to use unchecked_offset_from because + // end_32bit_gap_start > himem_start + mmio_start.unchecked_offset_from(himem_start), + bootparam::E820_RAM, + ) + .map_err(Error::BootSystem)?; + if mem_end > mmio_end { + add_e820_entry( + &mut params.0, + mmio_end.raw_value() + 1, + // it's safe to use unchecked_offset_from because mem_end > mmio_end + mem_end.unchecked_offset_from(mmio_end) as u64, + bootparam::E820_RAM, + ) + .map_err(Error::BootSystem)?; + } + } + + // reserve memory from microVM. + if rsv_mem_bytes > 0 { + add_e820_entry( + &mut params.0, + mem_end.raw_value().max(mmio_end.raw_value()) + 1, + rsv_mem_bytes, + bootparam::E820_RESERVED, + ) + .map_err(Error::BootSystem)?; + } + + let zero_page_addr = GuestAddress(layout::ZERO_PAGE_START); + guest_mem + .checked_offset(zero_page_addr, mem::size_of::()) + .ok_or(Error::ZeroPagePastRamEnd)?; + guest_mem + .write_obj(params, zero_page_addr) + .map_err(|_| Error::ZeroPageSetup)?; + + Ok(()) +} + +impl Vm { + /// Get the status of in-kernel PIT. + pub fn get_pit_state(&self) -> Result { + self.fd.get_pit2().map_err(|e| Error::Vm(VmError::Irq(e))) + } + + /// Set the status of in-kernel PIT. + pub fn set_pit_state(&self, pit_state: &kvm_pit_state2) -> Result<()> { + self.fd + .set_pit2(pit_state) + .map_err(|e| Error::Vm(VmError::Irq(e))) + } + + /// Get the status of in-kernel ioapic. + pub fn get_irqchip_state(&self, chip_id: u32) -> Result { + let mut irqchip: kvm_irqchip = kvm_irqchip { + chip_id, + ..kvm_irqchip::default() + }; + self.fd + .get_irqchip(&mut irqchip) + .map(|_| irqchip) + .map_err(|e| Error::Vm(VmError::Irq(e))) + } + + /// Set the status of in-kernel ioapic. + pub fn set_irqchip_state(&self, irqchip: &kvm_irqchip) -> Result<()> { + self.fd + .set_irqchip(irqchip) + .map_err(|e| Error::Vm(VmError::Irq(e))) + } +} + +impl Vm { + /// Initialize the virtual machine instance. + /// + /// It initialize the virtual machine instance by: + /// 1) initialize virtual machine global state and configuration. + /// 2) create system devices, such as interrupt controller, PIT etc. + /// 3) create and start IO devices, such as serial, console, block, net, vsock etc. + /// 4) create and initialize vCPUs. + /// 5) configure CPU power management features. + /// 6) load guest kernel image. + pub fn init_microvm( + &mut self, + epoll_mgr: EpollManager, + vm_as: GuestAddressSpaceImpl, + request_ts: TimestampUs, + ) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: start initializing microvm ..."); + + self.init_tss()?; + // For x86_64 we need to create the interrupt controller before calling `KVM_CREATE_VCPUS` + // while on aarch64 we need to do it the other way around. + self.setup_interrupt_controller()?; + self.create_pit()?; + self.init_devices(epoll_mgr)?; + + let reset_event_fd = self.device_manager.get_reset_eventfd().unwrap(); + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .set_reset_event_fd(reset_event_fd) + .map_err(StartMicrovmError::Vcpu)?; + + if self.vm_config.cpu_pm == "on" { + // TODO: add cpu_pm support. issue #4590. + info!(self.logger, "VM: enable CPU disable_idle_exits capability"); + } + + let vm_memory = vm_as.memory(); + let kernel_loader_result = self.load_kernel(vm_memory.deref())?; + + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) + .map_err(StartMicrovmError::Vcpu)?; + + info!(self.logger, "VM: initializing microvm done"); + Ok(()) + } + + /// Execute system architecture specific configurations. + /// + /// 1) set guest kernel boot parameters + /// 2) setup BIOS configuration data structs, mainly implement the MPSpec. + pub fn configure_system_arch( + &self, + vm_memory: &GuestMemoryImpl, + cmdline: &Cmdline, + initrd: Option, + ) -> std::result::Result<(), StartMicrovmError> { + let cmdline_addr = GuestAddress(dbs_boot::layout::CMDLINE_START); + linux_loader::loader::load_cmdline(vm_memory, cmdline_addr, cmdline) + .map_err(StartMicrovmError::LoadCommandline)?; + + configure_system( + vm_memory, + self.address_space.address_space(), + cmdline_addr, + cmdline.as_str().len() + 1, + &initrd, + self.vm_config.vcpu_count, + self.vm_config.max_vcpu_count, + self.vm_config.reserve_memory_bytes, + ) + .map_err(StartMicrovmError::ConfigureSystem) + } + + /// Initializes the guest memory. + pub(crate) fn init_tss(&mut self) -> std::result::Result<(), StartMicrovmError> { + self.fd + .set_tss_address(dbs_boot::layout::KVM_TSS_ADDRESS.try_into().unwrap()) + .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) + } + + /// Creates the irq chip and an in-kernel device model for the PIT. + pub(crate) fn setup_interrupt_controller( + &mut self, + ) -> std::result::Result<(), StartMicrovmError> { + self.fd + .create_irq_chip() + .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) + } + + /// Creates an in-kernel device model for the PIT. + pub(crate) fn create_pit(&self) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: create pit"); + // We need to enable the emulation of a dummy speaker port stub so that writing to port 0x61 + // (i.e. KVM_SPEAKER_BASE_ADDRESS) does not trigger an exit to user space. + let pit_config = kvm_pit_config { + flags: KVM_PIT_SPEAKER_DUMMY, + ..kvm_pit_config::default() + }; + + // Safe because we know that our file is a VM fd, we know the kernel will only read the + // correct amount of memory from our pointer, and we verify the return result. + self.fd + .create_pit2(pit_config) + .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) + } +} From 4d234f5742e437cfa9bf4e1a0aaa6250df9f9904 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 22 May 2022 14:42:32 +0800 Subject: [PATCH 0089/1953] dragonball: refactor code layout Refactored some code layout. Signed-off-by: Jiang Liu --- .../src/device_manager/console_manager.rs | 2 +- src/dragonball/src/device_manager/mod.rs | 43 +- src/dragonball/src/error.rs | 5 +- src/dragonball/src/vcpu/mod.rs | 7 +- src/dragonball/src/vm/aarch64.rs | 51 ++- src/dragonball/src/vm/kernel_config.rs | 7 +- src/dragonball/src/vm/mod.rs | 407 +++++++++--------- src/dragonball/src/vm/x86_64.rs | 22 +- 8 files changed, 276 insertions(+), 268 deletions(-) diff --git a/src/dragonball/src/device_manager/console_manager.rs b/src/dragonball/src/device_manager/console_manager.rs index 388eea78b8d9..4ebad58167c6 100644 --- a/src/dragonball/src/device_manager/console_manager.rs +++ b/src/dragonball/src/device_manager/console_manager.rs @@ -353,7 +353,7 @@ pub struct DmesgWriter { impl DmesgWriter { /// Creates a new instance. - pub fn new(logger: slog::Logger) -> Self { + pub fn new(logger: &slog::Logger) -> Self { Self { buf: BytesMut::with_capacity(1024), logger: logger.new(slog::o!("subsystem" => "dmesg")), diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 011a1607dc83..07698d4b887d 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -22,8 +22,6 @@ use kvm_ioctls::VmFd; use dbs_device::resources::ResourceConstraint; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices as virtio; -#[cfg(feature = "virtio-vsock")] -use dbs_virtio_devices::vsock::backend::VsockInnerConnector; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::{ mmio::{ @@ -38,6 +36,8 @@ use dbs_upcall::{ DevMgrRequest, DevMgrService, MmioDevRequest, UpcallClient, UpcallClientError, UpcallClientRequest, UpcallClientResponse, }; +#[cfg(feature = "hotplug")] +use dbs_virtio_devices::vsock::backend::VsockInnerConnector; use crate::address_space_manager::GuestAddressSpaceImpl; use crate::error::StartMicrovmError; @@ -99,6 +99,10 @@ pub enum DeviceMgrError { /// Failed to hotplug the device. #[error("failed to hotplug virtual device")] HotplugDevice(#[source] UpcallClientError), + + /// Failed to free device resource. + #[error("failed to free device resources: {0}")] + ResourceError(#[source] crate::resource_manager::ResourceError), } /// Specialized version of `std::result::Result` for device manager operations. @@ -315,8 +319,6 @@ impl DeviceOpContext { pub(crate) fn create_hotplug_ctx(vm: &Vm, epoll_mgr: Option) -> Self { let vm_as = vm.vm_as().expect("VM should have memory ready").clone(); - let vm_config = vm.vm_config().clone(); - let mut ctx = Self::new( epoll_mgr, vm.device_manager(), @@ -325,7 +327,6 @@ impl DeviceOpContext { true, ); ctx.upcall_client = vm.upcall_client().clone(); - ctx } @@ -404,10 +405,10 @@ pub struct DeviceManager { pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, + #[cfg(target_arch = "aarch64")] + pub(crate) mmio_device_info: HashMap<(DeviceType, String), MMIODeviceInfo>, #[cfg(feature = "virtio-vsock")] pub(crate) vsock_manager: VsockDeviceMgr, - #[cfg(target_arch = "aarch64")] - mmio_device_info: HashMap<(DeviceType, String), MMIODeviceInfo>, } impl DeviceManager { @@ -425,12 +426,13 @@ impl DeviceManager { res_manager, vm_fd, logger: logger.new(slog::o!()), + con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, - #[cfg(feature = "virtio-vsock")] - vsock_manager: VsockDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), + #[cfg(feature = "virtio-vsock")] + vsock_manager: VsockDeviceMgr::default(), } } @@ -452,6 +454,7 @@ impl DeviceManager { } /// Create legacy devices associted virtual machine + #[allow(unused_variables)] pub fn create_legacy_devices( &mut self, ctx: &mut DeviceOpContext, @@ -529,12 +532,6 @@ impl DeviceManager { self.con_manager.reset_console() } - #[cfg(target_arch = "aarch64")] - /// Return mmio device info for FDT build. - pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { - Some(&self.mmio_device_info) - } - /// Create all registered devices when booting the associated virtual machine. pub fn create_devices( &mut self, @@ -579,8 +576,10 @@ impl DeviceManager { ) -> Result<()> { Ok(()) } +} - #[cfg(target_arch = "x86_64")] +#[cfg(target_arch = "x86_64")] +impl DeviceManager { /// Get the underlying eventfd for vm exit notification. pub fn get_reset_eventfd(&self) -> Result { if let Some(legacy) = self.legacy_manager.as_ref() { @@ -595,6 +594,14 @@ impl DeviceManager { } } +#[cfg(target_arch = "aarch64")] +impl DeviceManager { + /// Return mmio device info for FDT build. + pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { + Some(&self.mmio_device_info) + } +} + #[cfg(feature = "dbs-virtio-devices")] impl DeviceManager { fn get_virtio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { @@ -694,7 +701,9 @@ impl DeviceManager { // unregister Resource manager let resources = device.get_assigned_resources(); - ctx.res_manager.free_device_resources(&resources); + ctx.res_manager + .free_device_resources(&resources) + .map_err(DeviceMgrError::ResourceError)?; Ok(()) } diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index b201c1d779ed..c50e50b256f2 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,10 +12,7 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; -use crate::address_space_manager; -use crate::device_manager; -use crate::vcpu; -use crate::vm; +use crate::{address_space_manager, device_manager, vcpu, vm}; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs index fe66883245de..5b8ed397fff6 100644 --- a/src/dragonball/src/vcpu/mod.rs +++ b/src/dragonball/src/vcpu/mod.rs @@ -4,13 +4,14 @@ // SPDX-License-Identifier: Apache-2.0 mod sm; -pub mod vcpu_impl; -pub mod vcpu_manager; -pub use vcpu_manager::{VcpuManager, VcpuManagerError}; +mod vcpu_impl; +mod vcpu_manager; #[cfg(target_arch = "x86_64")] use dbs_arch::cpuid::VpmuFeatureLevel; +pub use vcpu_manager::{VcpuManager, VcpuManagerError}; + /// vcpu config collection pub struct VcpuConfig { /// initial vcpu count diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 02af4a69171f..739b1515c641 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -7,6 +7,7 @@ // found in the THIRD-PARTY file. use std::collections::HashMap; +use std::fmt::Debug; use std::ops::Deref; use dbs_arch::gic::GICDevice; @@ -14,15 +15,13 @@ use dbs_arch::{DeviceInfoForFDT, DeviceType}; use dbs_boot::InitrdConfig; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; -use std::fmt::Debug; +use linux_loader::loader::Cmdline; use vm_memory::{GuestAddressSpace, GuestMemory}; use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; -use crate::error::Error; -use crate::StartMicrovmError; -use linux_loader::loader::Cmdline; +use crate::error::{Error, StartMicrovmError}; /// Configures the system and should be called once per vm before starting vcpu threads. /// For aarch64, we only setup the FDT. @@ -35,7 +34,7 @@ use linux_loader::loader::Cmdline; /// * `device_info` - A hashmap containing the attached devices for building FDT device nodes. /// * `gic_device` - The GIC device. /// * `initrd` - Information about an optional initrd. -pub fn configure_system( +fn configure_system( guest_mem: &M, cmdline: &str, vcpu_mpidr: Vec, @@ -62,6 +61,18 @@ impl Vm { &self.irqchip_handle.as_ref().unwrap() } + /// Creates the irq chip in-kernel device model. + pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { + let vcpu_count = self.vm_config.vcpu_count; + + self.irqchip_handle = Some( + dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) + .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, + ); + + Ok(()) + } + /// Initialize the virtual machine instance. /// /// It initialize the virtual machine instance by: @@ -76,13 +87,7 @@ impl Vm { epoll_mgr: EpollManager, vm_as: GuestAddressSpaceImpl, request_ts: TimestampUs, - ) -> std::result::Result<(), StartMicrovmError> { - let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; - // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before - // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP - // was already initialized. - // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. - + ) -> Result<(), StartMicrovmError> { let reset_eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?; self.reset_eventfd = Some( @@ -90,33 +95,25 @@ impl Vm { .try_clone() .map_err(|_| StartMicrovmError::EventFd)?, ); - self.vcpu_manager() .map_err(StartMicrovmError::Vcpu)? .set_reset_event_fd(reset_eventfd); + + // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before + // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP + // was already initialized. + // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. + let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; self.vcpu_manager() .map_err(StartMicrovmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) .map_err(StartMicrovmError::Vcpu)?; - self.setup_interrupt_controller()?; self.init_devices(epoll_mgr)?; Ok(()) } - /// Creates the irq chip in-kernel device model. - pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { - let vcpu_count = self.vm_config.vcpu_count; - - self.irqchip_handle = Some( - dbs_arch::gic::create_gic(&self.fd, vcpu_count.into()) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, - ); - - Ok(()) - } - /// Execute system architecture specific configurations. /// /// 1) set guest kernel boot parameters @@ -133,8 +130,8 @@ impl Vm { .into_iter() .map(|cpu| cpu.get_mpidr()) .collect(); - let guest_memory = vm_memory.memory(); + configure_system( guest_memory, cmdline.as_str(), diff --git a/src/dragonball/src/vm/kernel_config.rs b/src/dragonball/src/vm/kernel_config.rs index 466fd8849cc9..4798d8da3d18 100644 --- a/src/dragonball/src/vm/kernel_config.rs +++ b/src/dragonball/src/vm/kernel_config.rs @@ -8,7 +8,7 @@ pub struct KernelConfigInfo { /// The descriptor to the kernel file. kernel_file: File, /// The descriptor to the initrd file, if there is one - pub initrd_file: Option, + initrd_file: Option, /// The commandline for guest kernel. cmdline: linux_loader::cmdline::Cmdline, } @@ -32,6 +32,11 @@ impl KernelConfigInfo { &mut self.kernel_file } + /// Get an immutable reference to the initrd file. + pub fn initrd_file(&self) -> Option<&File> { + self.initrd_file.as_ref() + } + /// Get a mutable reference to the initrd file. pub fn initrd_file_mut(&mut self) -> Option<&mut File> { self.initrd_file.as_mut() diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index db10e593e614..fff044e3aa21 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -10,8 +10,6 @@ use dbs_address_space::AddressSpace; #[cfg(target_arch = "aarch64")] use dbs_arch::gic::GICDevice; use dbs_boot::InitrdConfig; -#[cfg(feature = "hotplug")] -use dbs_upcall::{DevMgrService, UpcallClient}; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; use kvm_ioctls::VmFd; @@ -22,6 +20,9 @@ use slog::{error, info}; use vm_memory::{Bytes, GuestAddress, GuestAddressSpace}; use vmm_sys_util::eventfd::EventFd; +#[cfg(feature = "hotplug")] +use dbs_upcall::{DevMgrService, UpcallClient}; + use crate::address_space_manager::{ AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl, GuestMemoryImpl, @@ -61,7 +62,7 @@ pub enum VmError { /// Cannot setup GIC #[cfg(target_arch = "aarch64")] #[error("failed to configure GIC")] - SetupGIC(GICError), + SetupGIC(dbs_arch::gic::Error), } /// Configuration information for user defined NUMA nodes. @@ -169,21 +170,21 @@ impl Default for VmConfigInfo { /// | ^---1:N-> Vcpu /// |---<-1:N-> Event Manager pub struct Vm { - fd: Arc, + epoll_manager: EpollManager, kvm: KvmContext, + shared_info: Arc>, address_space: AddressSpaceMgr, device_manager: DeviceManager, - epoll_manager: EpollManager, + dmesg_fifo: Option>, + kernel_config: Option, + logger: slog::Logger, + reset_eventfd: Option, resource_manager: Arc, vcpu_manager: Option>>, - logger: slog::Logger, - /// Config of virtual machine vm_config: VmConfigInfo, - kernel_config: Option, - shared_info: Arc>, - reset_eventfd: Option, - dmesg_fifo: Option>, + vm_fd: Arc, + start_instance_request_ts: u64, start_instance_request_cpu_ts: u64, start_instance_downtime: u64, @@ -191,7 +192,7 @@ pub struct Vm { // Arm specific fields. // On aarch64 we need to keep around the fd obtained by creating the VGIC device. #[cfg(target_arch = "aarch64")] - irqchip_handle: Option>, + irqchip_handle: Option>, #[cfg(feature = "hotplug")] upcall_client: Option>>, @@ -206,36 +207,36 @@ impl Vm { ) -> Result { let id = api_shared_info.read().unwrap().id.clone(); let logger = slog_scope::logger().new(slog::o!("id" => id)); - let kvm = KvmContext::new(kvm_fd)?; - let fd = Arc::new(kvm.create_vm()?); - + let vm_fd = Arc::new(kvm.create_vm()?); let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots()))); - let device_manager = DeviceManager::new( - fd.clone(), + vm_fd.clone(), resource_manager.clone(), epoll_manager.clone(), &logger, ); Ok(Vm { - fd, + epoll_manager, kvm, + shared_info: api_shared_info, + address_space: AddressSpaceMgr::default(), device_manager, - epoll_manager, + dmesg_fifo: None, + kernel_config: None, + logger, + reset_eventfd: None, resource_manager, vcpu_manager: None, - logger, vm_config: Default::default(), - kernel_config: None, - shared_info: api_shared_info, - reset_eventfd: None, - dmesg_fifo: None, + vm_fd, + start_instance_request_ts: 0, start_instance_request_cpu_ts: 0, start_instance_downtime: 0, + #[cfg(target_arch = "aarch64")] irqchip_handle: None, #[cfg(feature = "hotplug")] @@ -243,9 +244,29 @@ impl Vm { }) } - /// Gets a reference to the kvm file descriptor owned by this VM. - pub fn vm_fd(&self) -> &VmFd { - &self.fd + /// Gets a reference to the device manager by this VM. + pub fn device_manager(&self) -> &DeviceManager { + &self.device_manager + } + + /// Get a reference to EpollManager. + pub fn epoll_manager(&self) -> &EpollManager { + &self.epoll_manager + } + + /// Get eventfd for exit notification. + pub fn get_reset_eventfd(&self) -> Option<&EventFd> { + self.reset_eventfd.as_ref() + } + + /// Set guest kernel boot configurations. + pub fn set_kernel_config(&mut self, kernel_config: KernelConfigInfo) { + self.kernel_config = Some(kernel_config); + } + + /// Get virtual machine shared instance information. + pub fn shared_info(&self) -> &Arc> { + &self.shared_info } /// Gets a reference to the address_space.address_space for guest memory owned by this VM. @@ -253,11 +274,6 @@ impl Vm { self.address_space.get_address_space() } - /// Gets a reference to the device manager by this VM. - pub fn device_manager(&self) -> &DeviceManager { - &self.device_manager - } - /// Gets a reference to the address space for guest memory owned by this VM. /// /// Note that `GuestMemory` does not include any device memory that may have been added after @@ -276,24 +292,21 @@ impl Vm { self.vm_config = config; } - /// Set guest kernel boot configurations. - pub fn set_kernel_config(&mut self, kernel_config: KernelConfigInfo) { - self.kernel_config = Some(kernel_config); - } - - /// Get virtual machine shared instance information. - pub fn shared_info(&self) -> &Arc> { - &self.shared_info + /// Gets a reference to the kvm file descriptor owned by this VM. + pub fn vm_fd(&self) -> &VmFd { + &self.vm_fd } - /// Get a reference to EpollManager. - pub fn epoll_manager(&self) -> &EpollManager { - &self.epoll_manager - } + /// returns true if system upcall service is ready + pub fn is_upcall_client_ready(&self) -> bool { + #[cfg(feature = "hotplug")] + { + if let Some(upcall_client) = self.upcall_client() { + return upcall_client.is_ready(); + } + } - /// Get eventfd for exit notification. - pub fn get_reset_eventfd(&self) -> Option<&EventFd> { - self.reset_eventfd.as_ref() + false } /// Check whether the VM has been initialized. @@ -318,16 +331,16 @@ impl Vm { instance_state == InstanceState::Running } - /// returns true if system upcall service is ready - pub fn is_upcall_client_ready(&self) -> bool { - #[cfg(feature = "hotplug")] - { - if let Some(upcall_client) = self.upcall_client() { - return upcall_client.is_ready(); - } + /// Save VM instance exit state + pub fn vm_exit(&self, exit_code: i32) { + if let Ok(mut info) = self.shared_info.write() { + info.state = InstanceState::Exited(exit_code); + } else { + error!( + self.logger, + "Failed to save exit state, couldn't be written due to poisoned lock" + ); } - - false } /// Create device operation context. @@ -359,55 +372,30 @@ impl Vm { } } - /// Save VM instance exit state - pub fn vm_exit(&self, exit_code: i32) { - if let Ok(mut info) = self.shared_info.write() { - info.state = InstanceState::Exited(exit_code); - } else { - error!( - self.logger, - "Failed to save exit state, couldn't be written due to poisoned lock" - ); + pub(crate) fn check_health(&self) -> std::result::Result<(), StartMicrovmError> { + if self.kernel_config.is_none() { + return Err(StartMicrovmError::MissingKernelConfig); } + Ok(()) } - /// Reset the console into canonical mode. - pub fn reset_console(&self) -> std::result::Result<(), DeviceMgrError> { - self.device_manager.reset_console() - } - - fn get_dragonball_info(&self) -> (String, String) { + pub(crate) fn get_dragonball_info(&self) -> (String, String) { let guard = self.shared_info.read().unwrap(); let instance_id = guard.id.clone(); let dragonball_version = guard.vmm_version.clone(); (dragonball_version, instance_id) } +} - fn init_dmesg_logger(&mut self) { - let writer = self.dmesg_logger(); - self.dmesg_fifo = Some(writer); - } - - /// dmesg write to logger - pub fn dmesg_logger(&self) -> Box { - Box::new(DmesgWriter::new(self.logger.clone())) - } - - pub(crate) fn check_health(&self) -> std::result::Result<(), StartMicrovmError> { - if self.kernel_config.is_none() { - return Err(StartMicrovmError::MissingKernelConfig); - } - Ok(()) - } - +impl Vm { pub(crate) fn init_vcpu_manager( &mut self, vm_as: GuestAddressSpaceImpl, vcpu_seccomp_filter: BpfProgram, ) -> std::result::Result<(), VcpuManagerError> { let vcpu_manager = VcpuManager::new( - self.fd.clone(), + self.vm_fd.clone(), &self.kvm, &self.vm_config, vm_as, @@ -422,7 +410,7 @@ impl Vm { } /// get the cpu manager's reference - pub fn vcpu_manager( + pub(crate) fn vcpu_manager( &self, ) -> std::result::Result, VcpuManagerError> { self.vcpu_manager @@ -448,13 +436,85 @@ impl Vm { Ok(()) } + pub(crate) fn init_devices( + &mut self, + epoll_manager: EpollManager, + ) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: initializing devices ..."); + + let com1_sock_path = self.vm_config.serial_path.clone(); + let kernel_config = self + .kernel_config + .as_mut() + .ok_or(StartMicrovmError::MissingKernelConfig)?; + + info!(self.logger, "VM: create interrupt manager"); + self.device_manager + .create_interrupt_manager() + .map_err(StartMicrovmError::DeviceManager)?; + + info!(self.logger, "VM: create devices"); + let vm_as = + self.address_space + .get_vm_as() + .ok_or(StartMicrovmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + ))?; + self.device_manager.create_devices( + vm_as.clone(), + epoll_manager, + kernel_config, + com1_sock_path, + self.dmesg_fifo.take(), + self.address_space.address_space(), + )?; + + info!(self.logger, "VM: start devices"); + self.device_manager.start_devices()?; + + info!(self.logger, "VM: initializing devices done"); + Ok(()) + } + + /// Remove devices when shutdown vm + pub fn remove_devices(&mut self) -> std::result::Result<(), StopMicrovmError> { + info!(self.logger, "VM: remove devices"); + let vm_as = self + .address_space + .get_vm_as() + .ok_or(StopMicrovmError::GuestMemoryNotInitialized)?; + + self.device_manager + .remove_devices( + vm_as.clone(), + self.epoll_manager.clone(), + self.address_space.address_space(), + ) + .map_err(StopMicrovmError::DeviceManager) + } + + /// Reset the console into canonical mode. + pub fn reset_console(&self) -> std::result::Result<(), DeviceMgrError> { + self.device_manager.reset_console() + } + + pub(crate) fn init_dmesg_logger(&mut self) { + let writer = self.dmesg_logger(); + self.dmesg_fifo = Some(writer); + } + + /// dmesg write to logger + fn dmesg_logger(&self) -> Box { + Box::new(DmesgWriter::new(&self.logger)) + } + pub(crate) fn init_guest_memory(&mut self) -> std::result::Result<(), StartMicrovmError> { info!(self.logger, "VM: initializing guest memory..."); - // We are not allowing reinitialization of vm guest memory. if self.address_space.is_initialized() { return Ok(()); } + // vcpu boot up require local memory. reserve 100 MiB memory let mem_size = (self.vm_config.mem_size_mib as u64) << 20; let reserve_memory_bytes = self.vm_config.reserve_memory_bytes; @@ -473,13 +533,13 @@ impl Vm { mem_file_path.push_str(shared_info.id.as_str()); } - // init default regions. - let mut numa_regions = Vec::with_capacity(1); let mut vcpu_ids: Vec = Vec::new(); - for i in 0..self.vm_config().max_vcpu_count { vcpu_ids.push(i as u32); } + + // init default regions. + let mut numa_regions = Vec::with_capacity(1); let numa_node = NumaRegionInfo { size: self.vm_config.mem_size_mib as u64, host_numa_node_id: None, @@ -501,7 +561,7 @@ impl Vm { let mut address_space_param = AddressSpaceMgrBuilder::new(&mem_type, &mem_file_path) .map_err(StartMicrovmError::AddressManagerError)?; - address_space_param.set_kvm_vm_fd(self.fd.clone()); + address_space_param.set_kvm_vm_fd(self.vm_fd.clone()); self.address_space .create_address_space(&self.resource_manager, &numa_regions, address_space_param) .map_err(StartMicrovmError::AddressManagerError)?; @@ -510,93 +570,31 @@ impl Vm { Ok(()) } - fn init_devices( + fn init_configure_system( &mut self, - epoll_manager: EpollManager, + vm_as: &GuestAddressSpaceImpl, ) -> std::result::Result<(), StartMicrovmError> { - info!(self.logger, "VM: initializing devices ..."); - - let com1_sock_path = self.vm_config.serial_path.clone(); - let kernel_config = self - .kernel_config - .as_mut() - .ok_or(StartMicrovmError::MissingKernelConfig)?; - - info!(self.logger, "VM: create interrupt manager"); - self.device_manager - .create_interrupt_manager() - .map_err(StartMicrovmError::DeviceManager)?; - - info!(self.logger, "VM: create devices"); - let vm_as = - self.address_space - .get_vm_as() - .ok_or(StartMicrovmError::AddressManagerError( - AddressManagerError::GuestMemoryNotInitialized, - ))?; - self.device_manager.create_devices( - vm_as.clone(), - epoll_manager, - kernel_config, - com1_sock_path, - self.dmesg_fifo.take(), - self.address_space.address_space(), - )?; - - info!(self.logger, "VM: start devices"); - self.device_manager.start_devices()?; - - info!(self.logger, "VM: initializing devices done"); - Ok(()) - } - - /// Remove devices when shutdown vm - pub fn remove_devices(&mut self) -> std::result::Result<(), StopMicrovmError> { - info!(self.logger, "VM: remove devices"); - let vm_as = self - .address_space - .get_vm_as() - .ok_or(StopMicrovmError::GuestMemoryNotInitialized)?; - - self.device_manager - .remove_devices( - vm_as.clone(), - self.epoll_manager.clone(), - self.address_space.address_space(), - ) - .map_err(StopMicrovmError::DeviceManager) - } - - fn load_kernel( - &mut self, - vm_memory: &GuestMemoryImpl, - ) -> std::result::Result { - // This is the easy way out of consuming the value of the kernel_cmdline. - + let vm_memory = vm_as.memory(); let kernel_config = self .kernel_config - .as_mut() + .as_ref() .ok_or(StartMicrovmError::MissingKernelConfig)?; + //let cmdline = kernel_config.cmdline.clone(); + let initrd: Option = match kernel_config.initrd_file() { + Some(f) => { + let initrd_file = f.try_clone(); + if initrd_file.is_err() { + return Err(StartMicrovmError::InitrdLoader( + LoadInitrdError::ReadInitrd(io::Error::from(io::ErrorKind::InvalidData)), + )); + } + let res = self.load_initrd(vm_memory.deref(), &mut initrd_file.unwrap())?; + Some(res) + } + None => None, + }; - let high_mem_addr = GuestAddress(dbs_boot::get_kernel_start()); - - #[cfg(target_arch = "x86_64")] - return linux_loader::loader::elf::Elf::load( - vm_memory, - None, - kernel_config.kernel_file_mut(), - Some(high_mem_addr), - ) - .map_err(StartMicrovmError::KernelLoader); - - #[cfg(target_arch = "aarch64")] - return linux_loader::loader::pe::PE::load( - vm_memory, - Some(GuestAddress(dbs_boot::get_kernel_start())), - kernel_config.kernel_file_mut(), - Some(high_mem_addr), - ) - .map_err(StartMicrovmError::KernelLoader); + self.configure_system_arch(vm_memory.deref(), kernel_config.kernel_cmdline(), initrd) } /// Loads the initrd from a file into the given memory slice. @@ -644,49 +642,46 @@ impl Vm { }) } - fn init_configure_system( + fn load_kernel( &mut self, - vm_as: &GuestAddressSpaceImpl, - ) -> std::result::Result<(), StartMicrovmError> { - let vm_memory = vm_as.memory(); + vm_memory: &GuestMemoryImpl, + ) -> std::result::Result { + // This is the easy way out of consuming the value of the kernel_cmdline. let kernel_config = self .kernel_config - .as_ref() + .as_mut() .ok_or(StartMicrovmError::MissingKernelConfig)?; - //let cmdline = kernel_config.cmdline.clone(); - let initrd: Option = match &kernel_config.initrd_file { - Some(f) => { - let initrd_file = f.try_clone(); - if initrd_file.is_err() { - return Err(StartMicrovmError::InitrdLoader( - LoadInitrdError::ReadInitrd(io::Error::from(io::ErrorKind::InvalidData)), - )); - } - let res = self.load_initrd(vm_memory.deref(), &mut initrd_file.unwrap())?; - Some(res) - } - None => None, - }; + let high_mem_addr = GuestAddress(dbs_boot::get_kernel_start()); - self.configure_system_arch(vm_memory.deref(), kernel_config.kernel_cmdline(), initrd) + #[cfg(target_arch = "x86_64")] + return linux_loader::loader::elf::Elf::load( + vm_memory, + None, + kernel_config.kernel_file_mut(), + Some(high_mem_addr), + ) + .map_err(StartMicrovmError::KernelLoader); + + #[cfg(target_arch = "aarch64")] + return linux_loader::loader::pe::PE::load( + vm_memory, + Some(GuestAddress(dbs_boot::get_kernel_start())), + kernel_config.kernel_file_mut(), + Some(high_mem_addr), + ) + .map_err(StartMicrovmError::KernelLoader); } } #[cfg(feature = "hotplug")] impl Vm { - /// Get upcall client. - pub fn upcall_client(&self) -> &Option>> { - &self.upcall_client - } - /// initialize upcall client for guest os - fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub(crate) fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { // get vsock inner connector for upcall let inner_connector = self .device_manager .get_vsock_inner_connector() .ok_or(StartMicrovmError::UpcallMissVsock)?; - let mut upcall_client = UpcallClient::new( inner_connector, self.epoll_manager.clone(), @@ -697,10 +692,14 @@ impl Vm { upcall_client .connect() .map_err(StartMicrovmError::UpcallConnectError)?; - self.upcall_client = Some(Arc::new(upcall_client)); info!(self.logger, "upcall client init success"); Ok(()) } + + /// Get upcall client. + pub fn upcall_client(&self) -> &Option>> { + &self.upcall_client + } } diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index a55996282174..74336b6da0d9 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -15,6 +15,7 @@ use dbs_boot::{add_e820_entry, bootparam, layout, mptable, BootParamsWrapper, In use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY}; +use linux_loader::cmdline::Cmdline; use slog::info; use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; @@ -22,8 +23,6 @@ use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, Result, StartMicrovmError}; use crate::vm::{Vm, VmError}; -use linux_loader::cmdline::Cmdline; - /// Configures the system and should be called once per vm before starting vcpu /// threads. /// @@ -40,7 +39,7 @@ use linux_loader::cmdline::Cmdline; /// * `max_cpus` - Max number of virtual CPUs the guest will have. /// * `rsv_mem_bytes` - Reserve memory from microVM.. #[allow(clippy::too_many_arguments)] -pub fn configure_system( +fn configure_system( guest_mem: &M, address_space: Option<&AddressSpace>, cmdline_addr: GuestAddress, @@ -136,12 +135,14 @@ pub fn configure_system( impl Vm { /// Get the status of in-kernel PIT. pub fn get_pit_state(&self) -> Result { - self.fd.get_pit2().map_err(|e| Error::Vm(VmError::Irq(e))) + self.vm_fd + .get_pit2() + .map_err(|e| Error::Vm(VmError::Irq(e))) } /// Set the status of in-kernel PIT. pub fn set_pit_state(&self, pit_state: &kvm_pit_state2) -> Result<()> { - self.fd + self.vm_fd .set_pit2(pit_state) .map_err(|e| Error::Vm(VmError::Irq(e))) } @@ -152,7 +153,7 @@ impl Vm { chip_id, ..kvm_irqchip::default() }; - self.fd + self.vm_fd .get_irqchip(&mut irqchip) .map(|_| irqchip) .map_err(|e| Error::Vm(VmError::Irq(e))) @@ -160,7 +161,7 @@ impl Vm { /// Set the status of in-kernel ioapic. pub fn set_irqchip_state(&self, irqchip: &kvm_irqchip) -> Result<()> { - self.fd + self.vm_fd .set_irqchip(irqchip) .map_err(|e| Error::Vm(VmError::Irq(e))) } @@ -204,7 +205,6 @@ impl Vm { let vm_memory = vm_as.memory(); let kernel_loader_result = self.load_kernel(vm_memory.deref())?; - self.vcpu_manager() .map_err(StartMicrovmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) @@ -243,7 +243,7 @@ impl Vm { /// Initializes the guest memory. pub(crate) fn init_tss(&mut self) -> std::result::Result<(), StartMicrovmError> { - self.fd + self.vm_fd .set_tss_address(dbs_boot::layout::KVM_TSS_ADDRESS.try_into().unwrap()) .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } @@ -252,7 +252,7 @@ impl Vm { pub(crate) fn setup_interrupt_controller( &mut self, ) -> std::result::Result<(), StartMicrovmError> { - self.fd + self.vm_fd .create_irq_chip() .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } @@ -269,7 +269,7 @@ impl Vm { // Safe because we know that our file is a VM fd, we know the kernel will only read the // correct amount of memory from our pointer, and we verify the return result. - self.fd + self.vm_fd .create_pit2(pit_config) .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } From 5c1ccc376b5c46414c6fcda7d6183cfb274d16d6 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Sun, 15 May 2022 23:45:56 +0800 Subject: [PATCH 0090/1953] dragonball: add Vmm struct The Vmm struct is global coordinator to manage API servers, virtual machines etc. Signed-off-by: wllenyj --- src/dragonball/src/api/v1/boot_source.rs | 59 +++++++ src/dragonball/src/api/v1/mod.rs | 10 ++ src/dragonball/src/api/v1/vmm_action.rs | 148 ++++++++++++++++ src/dragonball/src/error.rs | 21 +++ src/dragonball/src/event_manager.rs | 169 ++++++++++++++++++ src/dragonball/src/lib.rs | 5 + src/dragonball/src/vmm.rs | 215 +++++++++++++++++++++++ 7 files changed, 627 insertions(+) create mode 100644 src/dragonball/src/api/v1/boot_source.rs create mode 100644 src/dragonball/src/api/v1/vmm_action.rs create mode 100644 src/dragonball/src/event_manager.rs create mode 100644 src/dragonball/src/vmm.rs diff --git a/src/dragonball/src/api/v1/boot_source.rs b/src/dragonball/src/api/v1/boot_source.rs new file mode 100644 index 000000000000..e7de030438cf --- /dev/null +++ b/src/dragonball/src/api/v1/boot_source.rs @@ -0,0 +1,59 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_derive::{Deserialize, Serialize}; + +/// Default guest kernel command line: +/// - `reboot=k` shut down the guest on reboot, instead of well... rebooting; +/// - `panic=1` on panic, reboot after 1 second; +/// - `pci=off` do not scan for PCI devices (ser boot time); +/// - `nomodules` disable loadable kernel module support; +/// - `8250.nr_uarts=0` disable 8250 serial interface; +/// - `i8042.noaux` do not probe the i8042 controller for an attached mouse (ser boot time); +/// - `i8042.nomux` do not probe i8042 for a multiplexing controller (ser boot time); +/// - `i8042.nopnp` do not use ACPIPnP to discover KBD/AUX controllers (ser boot time); +/// - `i8042.dumbkbd` do not attempt to control kbd state via the i8042 (ser boot time). +pub const DEFAULT_KERNEL_CMDLINE: &str = "reboot=k panic=1 pci=off nomodules 8250.nr_uarts=0 \ + i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd"; + +/// Strongly typed data structure used to configure the boot source of the microvm. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, Default)] +#[serde(deny_unknown_fields)] +pub struct BootSourceConfig { + /// Path of the kernel image. + /// We only support uncompressed kernel for Dragonball. + pub kernel_path: String, + /// Path of the initrd, if there is one. + /// ps. rootfs is set in BlockDeviceConfigInfo + pub initrd_path: Option, + /// The boot arguments to pass to the kernel. + #[serde(skip_serializing_if = "Option::is_none")] + pub boot_args: Option, +} + +/// Errors associated with actions on `BootSourceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum BootSourceConfigError { + /// The virutal machine instance ID is invalid. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + + /// The kernel file cannot be opened. + #[error( + "the kernel file cannot be opened due to invalid kernel path or invalid permissions: {0}" + )] + InvalidKernelPath(#[source] std::io::Error), + + /// The initrd file cannot be opened. + #[error("the initrd file cannot be opened due to invalid path or invalid permissions: {0}")] + InvalidInitrdPath(#[source] std::io::Error), + + /// The kernel command line is invalid. + #[error("the kernel command line is invalid: {0}")] + InvalidKernelCommandLine(#[source] linux_loader::cmdline::Error), + + /// The boot source cannot be update post boot. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, +} diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs index f25fb8436423..c1ab3f5d322e 100644 --- a/src/dragonball/src/api/v1/mod.rs +++ b/src/dragonball/src/api/v1/mod.rs @@ -3,5 +3,15 @@ //! API Version 1 related data structures to configure the vmm. +mod vmm_action; +pub use self::vmm_action::{ + VmmAction, VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, +}; + +/// Wrapper for configuring the microVM boot source. +mod boot_source; +pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KERNEL_CMDLINE}; + +/// Wrapper over the microVM general information. mod instance_info; pub use self::instance_info::{InstanceInfo, InstanceState}; diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs new file mode 100644 index 000000000000..1d7ac7e63035 --- /dev/null +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -0,0 +1,148 @@ +// Copyright (C) 2020-2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::fs::File; +use std::sync::mpsc::{Receiver, Sender, TryRecvError}; + +use log::{debug, error, warn}; +use vmm_sys_util::eventfd::EventFd; + +use crate::error::Result; +use crate::event_manager::EventManager; +use crate::vm::{KernelConfigInfo, VmConfigInfo}; +use crate::vmm::Vmm; + +use super::*; + +/// Wrapper for all errors associated with VMM actions. +#[derive(Debug, thiserror::Error)] +pub enum VmmActionError { + /// The action `ConfigureBootSource` failed either because of bad user input or an internal + /// error. + #[error("failed to configure boot source for VM: {0}")] + BootSource(#[source] BootSourceConfigError), +} + +/// This enum represents the public interface of the VMM. Each action contains various +/// bits of information (ids, paths, etc.). +#[derive(Clone, Debug, PartialEq)] +pub enum VmmAction { + /// Configure the boot source of the microVM using as input the `ConfigureBootSource`. This + /// action can only be called before the microVM has booted. + ConfigureBootSource(BootSourceConfig), +} + +/// The enum represents the response sent by the VMM in case of success. The response is either +/// empty, when no data needs to be sent, or an internal VMM structure. +#[derive(Debug)] +pub enum VmmData { + /// No data is sent on the channel. + Empty, +} + +/// Request data type used to communicate between the API and the VMM. +pub type VmmRequest = Box; + +/// Data type used to communicate between the API and the VMM. +pub type VmmRequestResult = std::result::Result; + +/// Response data type used to communicate between the API and the VMM. +pub type VmmResponse = Box; + +/// VMM Service to handle requests from the API server. +/// +/// There are two levels of API servers as below: +/// API client <--> VMM API Server <--> VMM Core +pub struct VmmService { + from_api: Receiver, + to_api: Sender, + machine_config: VmConfigInfo, +} + +impl VmmService { + /// Create a new VMM API server instance. + pub fn new(from_api: Receiver, to_api: Sender) -> Self { + VmmService { + from_api, + to_api, + machine_config: VmConfigInfo::default(), + } + } + + /// Handle requests from the HTTP API Server and send back replies. + pub fn run_vmm_action(&mut self, vmm: &mut Vmm, _event_mgr: &mut EventManager) -> Result<()> { + let request = match self.from_api.try_recv() { + Ok(t) => *t, + Err(TryRecvError::Empty) => { + warn!("Got a spurious notification from api thread"); + return Ok(()); + } + Err(TryRecvError::Disconnected) => { + panic!("The channel's sending half was disconnected. Cannot receive data."); + } + }; + debug!("receive vmm action: {:?}", request); + + let response = match request { + VmmAction::ConfigureBootSource(boot_source_body) => { + self.configure_boot_source(vmm, boot_source_body) + } + }; + + debug!("send vmm response: {:?}", response); + self.send_response(response) + } + + fn send_response(&self, result: VmmRequestResult) -> Result<()> { + self.to_api + .send(Box::new(result)) + .map_err(|_| ()) + .expect("vmm: one-shot API result channel has been closed"); + + Ok(()) + } + + fn configure_boot_source( + &self, + vmm: &mut Vmm, + boot_source_config: BootSourceConfig, + ) -> VmmRequestResult { + use super::BootSourceConfigError::{ + InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath, InvalidVMID, + UpdateNotAllowedPostBoot, + }; + use super::VmmActionError::BootSource; + + let vm = vmm.get_vm_by_id_mut("").ok_or(BootSource(InvalidVMID))?; + if vm.is_vm_initialized() { + return Err(BootSource(UpdateNotAllowedPostBoot)); + } + + let kernel_file = File::open(&boot_source_config.kernel_path) + .map_err(|e| BootSource(InvalidKernelPath(e)))?; + + let initrd_file = match boot_source_config.initrd_path { + None => None, + Some(ref path) => Some(File::open(path).map_err(|e| BootSource(InvalidInitrdPath(e)))?), + }; + + let mut cmdline = linux_loader::cmdline::Cmdline::new(dbs_boot::layout::CMDLINE_MAX_SIZE); + let boot_args = boot_source_config + .boot_args + .clone() + .unwrap_or_else(|| String::from(DEFAULT_KERNEL_CMDLINE)); + cmdline + .insert_str(boot_args) + .map_err(|e| BootSource(InvalidKernelCommandLine(e)))?; + + let kernel_config = KernelConfigInfo::new(kernel_file, initrd_file, cmdline); + vm.set_kernel_config(kernel_config); + + Ok(VmmData::Empty) + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index c50e50b256f2..9cb27fdd73e2 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -178,3 +178,24 @@ pub enum LoadInitrdError { #[error("failed to read the initrd image: {0}")] ReadInitrd(#[source] std::io::Error), } + +/// A dedicated error type to glue with the vmm_epoll crate. +#[derive(Debug, thiserror::Error)] +pub enum EpollError { + /// Generic internal error. + #[error("unclassfied internal error")] + InternalError, + + /// Errors from the epoll subsystem. + #[error("failed to issue epoll syscall: {0}")] + EpollMgr(#[from] dbs_utils::epoll_manager::Error), + + /// Generic IO errors. + #[error(transparent)] + IOError(std::io::Error), + + #[cfg(feature = "dbs-virtio-devices")] + /// Errors from virtio devices. + #[error("failed to manager Virtio device: {0}")] + VirtIoDevice(#[source] VirtIoError), +} diff --git a/src/dragonball/src/event_manager.rs b/src/dragonball/src/event_manager.rs new file mode 100644 index 000000000000..cd0da10c87ac --- /dev/null +++ b/src/dragonball/src/event_manager.rs @@ -0,0 +1,169 @@ +// Copyright (C) 2020-2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +//! Event manager to manage and handle IO events and requests from API server . + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use dbs_utils::epoll_manager::{ + EpollManager, EventOps, EventSet, Events, MutEventSubscriber, SubscriberId, +}; +use log::{error, warn}; +use vmm_sys_util::eventfd::EventFd; + +use crate::error::{EpollError, Result}; +use crate::vmm::Vmm; + +// Statically assigned epoll slot for VMM events. +pub(crate) const EPOLL_EVENT_EXIT: u32 = 0; +pub(crate) const EPOLL_EVENT_API_REQUEST: u32 = 1; + +/// Shared information between vmm::vmm_thread_event_loop() and VmmEpollHandler. +pub(crate) struct EventContext { + pub api_event_fd: EventFd, + pub api_event_flag: bool, + pub exit_evt_flag: bool, +} + +impl EventContext { + /// Create a new instance of [`EventContext`]. + pub fn new(api_event_fd: EventFd) -> Result { + Ok(EventContext { + api_event_fd, + api_event_flag: false, + exit_evt_flag: false, + }) + } +} + +/// Event manager for VMM to handle API requests and IO events. +pub struct EventManager { + epoll_mgr: EpollManager, + subscriber_id: SubscriberId, + vmm_event_count: Arc, +} + +impl Drop for EventManager { + fn drop(&mut self) { + // Vmm -> Vm -> EpollManager -> VmmEpollHandler -> Vmm + // We need to remove VmmEpollHandler to break the circular reference + // so that Vmm can drop. + self.epoll_mgr + .remove_subscriber(self.subscriber_id) + .map_err(|e| { + error!("event_manager: remove_subscriber err. {:?}", e); + e + }) + .ok(); + } +} + +impl EventManager { + /// Create a new event manager associated with the VMM object. + pub fn new(vmm: &Arc>, epoll_mgr: EpollManager) -> Result { + let vmm_event_count = Arc::new(AtomicUsize::new(0)); + let handler: Box = Box::new(VmmEpollHandler { + vmm: vmm.clone(), + vmm_event_count: vmm_event_count.clone(), + }); + let subscriber_id = epoll_mgr.add_subscriber(handler); + + Ok(EventManager { + epoll_mgr, + subscriber_id, + vmm_event_count, + }) + } + + /// Get the underlying epoll event manager. + pub fn epoll_manager(&self) -> EpollManager { + self.epoll_mgr.clone() + } + + /// Registry the eventfd for exit notification. + pub fn register_exit_eventfd( + &mut self, + exit_evt: &EventFd, + ) -> std::result::Result<(), EpollError> { + let events = Events::with_data(exit_evt, EPOLL_EVENT_EXIT, EventSet::IN); + + self.epoll_mgr + .add_event(self.subscriber_id, events) + .map_err(EpollError::EpollMgr) + } + + /// Poll pending events and invoke registered event handler. + /// + /// # Arguments: + /// * max_events: maximum number of pending events to handle + /// * timeout: maximum time in milliseconds to wait + pub fn handle_events(&self, timeout: i32) -> std::result::Result { + self.epoll_mgr + .handle_events(timeout) + .map_err(EpollError::EpollMgr) + } + + /// Fetch the VMM event count and reset it to zero. + pub fn fetch_vmm_event_count(&self) -> usize { + self.vmm_event_count.swap(0, Ordering::AcqRel) + } +} + +struct VmmEpollHandler { + vmm: Arc>, + vmm_event_count: Arc, +} + +impl MutEventSubscriber for VmmEpollHandler { + fn process(&mut self, events: Events, _ops: &mut EventOps) { + // Do not try to recover when the lock has already been poisoned. + // And be careful to avoid deadlock between process() and vmm::vmm_thread_event_loop(). + let mut vmm = self.vmm.lock().unwrap(); + + match events.data() { + EPOLL_EVENT_API_REQUEST => { + if let Err(e) = vmm.event_ctx.api_event_fd.read() { + error!("event_manager: failed to read API eventfd, {:?}", e); + } + vmm.event_ctx.api_event_flag = true; + self.vmm_event_count.fetch_add(1, Ordering::AcqRel); + } + EPOLL_EVENT_EXIT => { + let vm = vmm.get_vm_by_id("").unwrap(); + match vm.get_reset_eventfd() { + Some(ev) => { + if let Err(e) = ev.read() { + error!("event_manager: failed to read exit eventfd, {:?}", e); + } + } + None => warn!("event_manager: leftover exit event in epoll context!"), + } + vmm.event_ctx.exit_evt_flag = true; + self.vmm_event_count.fetch_add(1, Ordering::AcqRel); + } + _ => error!("event_manager: unknown epoll slot number {}", events.data()), + } + } + + fn init(&mut self, ops: &mut EventOps) { + // Do not expect poisoned lock. + let vmm = self.vmm.lock().unwrap(); + let events = Events::with_data( + &vmm.event_ctx.api_event_fd, + EPOLL_EVENT_API_REQUEST, + EventSet::IN, + ); + if let Err(e) = ops.add(events) { + error!( + "event_manager: failed to register epoll event for API server, {:?}", + e + ); + } + } +} diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index 6bf0b9298a1b..bd58159ac208 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -32,8 +32,13 @@ pub mod vcpu; /// Virtual machine manager for virtual machines. pub mod vm; +mod event_manager; mod io_manager; +mod vmm; + +pub use self::error::StartMicrovmError; pub use self::io_manager::IoManagerCached; +pub use self::vmm::Vmm; /// Success exit code. pub const EXIT_CODE_OK: u8 = 0; diff --git a/src/dragonball/src/vmm.rs b/src/dragonball/src/vmm.rs new file mode 100644 index 000000000000..a2c75ff7166f --- /dev/null +++ b/src/dragonball/src/vmm.rs @@ -0,0 +1,215 @@ +// Copyright (C) 2020-2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::os::unix::io::RawFd; +use std::sync::{Arc, Mutex, RwLock}; + +use dbs_utils::epoll_manager::EpollManager; +use log::{error, info, warn}; +use seccompiler::BpfProgram; +use vmm_sys_util::eventfd::EventFd; + +use crate::api::v1::{InstanceInfo, VmmService}; +use crate::error::{EpollError, Result}; +use crate::event_manager::{EventContext, EventManager}; +use crate::vm::Vm; +use crate::{EXIT_CODE_GENERIC_ERROR, EXIT_CODE_OK}; + +/// Global coordinator to manage API servers, virtual machines, upgrade etc. +/// +/// Originally firecracker assumes an VMM only manages an VM, and doesn't distinguish VMM and VM. +/// Thus caused a mixed and confusion design. Now we have explicit build the object model as: +/// |---Vmm API Server--<-1:1-> HTTP API Server +/// | |----------<-1:1-> Shimv2/CRI API Server +/// | +/// Vmm <-1:N-> Vm <-1:1-> Address Space Manager <-1:N-> GuestMemory +/// ^ ^---1:1-> Device Manager <-1:N-> Device +/// | ^---1:1-> Resource Manager +/// | ^---1:N-> Vcpu +/// |---<-1:N-> Event Manager +pub struct Vmm { + pub(crate) event_ctx: EventContext, + epoll_manager: EpollManager, + + // Will change to a HashMap when enabling 1 VMM with multiple VMs. + vm: Vm, + + vcpu_seccomp_filter: BpfProgram, + vmm_seccomp_filter: BpfProgram, +} + +impl Vmm { + /// Create a Virtual Machine Monitor instance. + pub fn new( + api_shared_info: Arc>, + api_event_fd: EventFd, + vmm_seccomp_filter: BpfProgram, + vcpu_seccomp_filter: BpfProgram, + kvm_fd: Option, + ) -> Result { + let epoll_manager = EpollManager::default(); + Self::new_with_epoll_manager( + api_shared_info, + api_event_fd, + epoll_manager, + vmm_seccomp_filter, + vcpu_seccomp_filter, + kvm_fd, + ) + } + + /// Create a Virtual Machine Monitor instance with a epoll_manager. + pub fn new_with_epoll_manager( + api_shared_info: Arc>, + api_event_fd: EventFd, + epoll_manager: EpollManager, + vmm_seccomp_filter: BpfProgram, + vcpu_seccomp_filter: BpfProgram, + kvm_fd: Option, + ) -> Result { + let vm = Vm::new(kvm_fd, api_shared_info, epoll_manager.clone())?; + let event_ctx = EventContext::new(api_event_fd)?; + + Ok(Vmm { + event_ctx, + epoll_manager, + vm, + vcpu_seccomp_filter, + vmm_seccomp_filter, + }) + } + + /// Get a reference to a virtual machine managed by the VMM. + pub fn get_vm_by_id(&self, _id: &str) -> Option<&Vm> { + Some(&self.vm) + } + + /// Get a mutable reference to a virtual machine managed by the VMM. + pub fn get_vm_by_id_mut(&mut self, _id: &str) -> Option<&mut Vm> { + Some(&mut self.vm) + } + + /// Get the seccomp rules for vCPU threads. + pub fn vcpu_seccomp_filter(&self) -> BpfProgram { + self.vcpu_seccomp_filter.clone() + } + + /// Get the seccomp rules for VMM threads. + pub fn vmm_seccomp_filter(&self) -> BpfProgram { + self.vmm_seccomp_filter.clone() + } + + /// Run the event loop to service API requests. + /// + /// # Arguments + /// + /// * `vmm` - An Arc reference to the global Vmm instance. + /// * `service` - VMM Service provider. + pub fn run_vmm_event_loop(vmm: Arc>, mut service: VmmService) -> i32 { + let epoll_mgr = vmm.lock().unwrap().epoll_manager.clone(); + let mut event_mgr = + EventManager::new(&vmm, epoll_mgr).expect("Cannot create epoll manager"); + + 'poll: loop { + match event_mgr.handle_events(-1) { + Ok(_) => { + // Check whether there are pending vmm events. + if event_mgr.fetch_vmm_event_count() == 0 { + continue; + } + + let mut v = vmm.lock().unwrap(); + if v.event_ctx.api_event_flag { + // The run_vmm_action() needs to access event_mgr, so it could + // not be handled in EpollHandler::handle_events(). It has been + // delayed to the main loop. + v.event_ctx.api_event_flag = false; + service + .run_vmm_action(&mut v, &mut event_mgr) + .unwrap_or_else(|_| { + warn!("got spurious notification from api thread"); + }); + } + if v.event_ctx.exit_evt_flag { + info!("Gracefully terminated VMM control loop"); + return v.stop(EXIT_CODE_OK as i32); + } + } + Err(e) => { + error!("Abruptly exited VMM control loop: {:?}", e); + if let EpollError::EpollMgr(dbs_utils::epoll_manager::Error::Epoll(e)) = e { + if e.errno() == libc::EAGAIN || e.errno() == libc::EINTR { + continue 'poll; + } + } + return vmm.lock().unwrap().stop(EXIT_CODE_GENERIC_ERROR as i32); + } + } + } + } + + /// Waits for all vCPUs to exit and terminates the Dragonball process. + fn stop(&mut self, exit_code: i32) -> i32 { + info!("Vmm is stopping."); + if let Some(vm) = self.get_vm_by_id_mut("") { + if vm.is_vm_initialized() { + if let Err(e) = vm.remove_devices() { + warn!("failed to remove devices: {:?}", e); + } + + if let Err(e) = vm.reset_console() { + warn!("Cannot set canonical mode for the terminal. {:?}", e); + } + + // Now, we use exit_code instead of invoking _exit to + // terminate process, so all of vcpu threads should be stopped + // prior to vmm event loop. + match vm.vcpu_manager() { + Ok(mut mgr) => { + if let Err(e) = mgr.exit_all_vcpus() { + warn!("Failed to exit vcpu thread. {:?}", e); + } + } + Err(e) => warn!("Failed to get vcpu manager {:?}", e), + } + + // save exit state to VM, instead of exit process. + vm.vm_exit(exit_code); + } + } + + exit_code + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + pub fn create_vmm_instance() -> Vmm { + let info = Arc::new(RwLock::new(InstanceInfo::default())); + let event_fd = EventFd::new(libc::EFD_NONBLOCK).unwrap(); + let seccomp_filter: BpfProgram = Vec::new(); + let epoll_manager = EpollManager::default(); + + Vmm::new_with_epoll_manager( + info, + event_fd, + epoll_manager, + seccomp_filter.clone(), + seccomp_filter, + None, + ) + .unwrap() + } + + #[test] + fn test_create_vmm_instance() { + create_vmm_instance(); + } +} From 95fa0c70c33c33781afaeef0349b1b199975a2e4 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 00:29:41 +0800 Subject: [PATCH 0091/1953] dragonball: add start microvm support We add microvm start related support in thie pull request. Signed-off-by: Liu Jiang Signed-off-by: wllenyj Signed-off-by: jingshan Signed-off-by: Chao Wu --- src/dragonball/src/api/v1/boot_source.rs | 4 - src/dragonball/src/api/v1/vmm_action.rs | 60 ++++++++++-- src/dragonball/src/error.rs | 6 +- src/dragonball/src/event_manager.rs | 12 +-- src/dragonball/src/vm/aarch64.rs | 13 +++ src/dragonball/src/vm/mod.rs | 119 ++++++++++++++++++++--- src/dragonball/src/vm/x86_64.rs | 17 ++++ src/dragonball/src/vmm.rs | 6 +- 8 files changed, 202 insertions(+), 35 deletions(-) diff --git a/src/dragonball/src/api/v1/boot_source.rs b/src/dragonball/src/api/v1/boot_source.rs index e7de030438cf..0094c8d7c7b0 100644 --- a/src/dragonball/src/api/v1/boot_source.rs +++ b/src/dragonball/src/api/v1/boot_source.rs @@ -35,10 +35,6 @@ pub struct BootSourceConfig { /// Errors associated with actions on `BootSourceConfig`. #[derive(Debug, thiserror::Error)] pub enum BootSourceConfigError { - /// The virutal machine instance ID is invalid. - #[error("the virtual machine instance ID is invalid")] - InvalidVMID, - /// The kernel file cannot be opened. #[error( "the kernel file cannot be opened due to invalid kernel path or invalid permissions: {0}" diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 1d7ac7e63035..7f8a8b98db8e 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -12,7 +12,7 @@ use std::sync::mpsc::{Receiver, Sender, TryRecvError}; use log::{debug, error, warn}; use vmm_sys_util::eventfd::EventFd; -use crate::error::Result; +use crate::error::{Result, StartMicrovmError, StopMicrovmError}; use crate::event_manager::EventManager; use crate::vm::{KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; @@ -22,19 +22,39 @@ use super::*; /// Wrapper for all errors associated with VMM actions. #[derive(Debug, thiserror::Error)] pub enum VmmActionError { + /// Invalid virtual machine instance ID. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + /// The action `ConfigureBootSource` failed either because of bad user input or an internal /// error. #[error("failed to configure boot source for VM: {0}")] BootSource(#[source] BootSourceConfigError), + + /// The action `StartMicroVm` failed either because of bad user input or an internal error. + #[error("failed to boot the VM: {0}")] + StartMicroVm(#[source] StartMicroVmError), + + /// The action `StopMicroVm` failed either because of bad user input or an internal error. + #[error("failed to shutdown the VM: {0}")] + StopMicrovm(#[source] StopMicrovmError), } /// This enum represents the public interface of the VMM. Each action contains various /// bits of information (ids, paths, etc.). #[derive(Clone, Debug, PartialEq)] pub enum VmmAction { - /// Configure the boot source of the microVM using as input the `ConfigureBootSource`. This - /// action can only be called before the microVM has booted. + /// Configure the boot source of the microVM using `BootSourceConfig`. + /// This action can only be called before the microVM has booted. ConfigureBootSource(BootSourceConfig), + + /// Launch the microVM. This action can only be called before the microVM has booted. + StartMicroVm, + + /// Shutdown the vmicroVM. This action can only be called after the microVM has booted. + /// When vmm is used as the crate by the other process, which is need to + /// shutdown the vcpu threads and destory all of the object. + ShutdownMicroVm, } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -75,7 +95,7 @@ impl VmmService { } /// Handle requests from the HTTP API Server and send back replies. - pub fn run_vmm_action(&mut self, vmm: &mut Vmm, _event_mgr: &mut EventManager) -> Result<()> { + pub fn run_vmm_action(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> Result<()> { let request = match self.from_api.try_recv() { Ok(t) => *t, Err(TryRecvError::Empty) => { @@ -92,6 +112,8 @@ impl VmmService { VmmAction::ConfigureBootSource(boot_source_body) => { self.configure_boot_source(vmm, boot_source_body) } + VmmAction::StartMicroVm => self.start_microvm(vmm, event_mgr), + VmmAction::ShutdownMicroVm => self.shutdown_microvm(vmm), }; debug!("send vmm response: {:?}", response); @@ -113,12 +135,14 @@ impl VmmService { boot_source_config: BootSourceConfig, ) -> VmmRequestResult { use super::BootSourceConfigError::{ - InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath, InvalidVMID, + InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath, UpdateNotAllowedPostBoot, }; use super::VmmActionError::BootSource; - let vm = vmm.get_vm_by_id_mut("").ok_or(BootSource(InvalidVMID))?; + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(BootSource(UpdateNotAllowedPostBoot)); } @@ -145,4 +169,28 @@ impl VmmService { Ok(VmmData::Empty) } + + fn start_microvm(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> VmmRequestResult { + use self::StartMicrovmError::MicroVMAlreadyRunning; + use self::VmmActionError::StartMicrovm; + + let vmm_seccomp_filter = vmm.vmm_seccomp_filter(); + let vcpu_seccomp_filter = vmm.vcpu_seccomp_filter(); + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + if vm.is_vm_initialized() { + return Err(StartMicrovm(MicroVMAlreadyRunning)); + } + + vm.start_microvm(event_mgr, vmm_seccomp_filter, vcpu_seccomp_filter) + .map(|_| VmmData::Empty) + .map_err(StartMicrovm) + } + + fn shutdown_microvm(&mut self, vmm: &mut Vmm) -> VmmRequestResult { + vmm.event_ctx.exit_evt_triggered = true; + + Ok(VmmData::Empty) + } } diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 9cb27fdd73e2..f6b37c867d88 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -72,11 +72,15 @@ pub enum Error { /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] -pub enum StartMicrovmError { +pub enum StartMicroVmError { /// Cannot read from an Event file descriptor. #[error("failure while reading from EventFd file descriptor")] EventFd, + /// Cannot add event to Epoll. + #[error("failure while registering epoll event for file descriptor")] + RegisterEvent, + /// The start command was issued more than once. #[error("the virtual machine is already running")] MicroVMAlreadyRunning, diff --git a/src/dragonball/src/event_manager.rs b/src/dragonball/src/event_manager.rs index cd0da10c87ac..9a2ad9d1c701 100644 --- a/src/dragonball/src/event_manager.rs +++ b/src/dragonball/src/event_manager.rs @@ -27,8 +27,8 @@ pub(crate) const EPOLL_EVENT_API_REQUEST: u32 = 1; /// Shared information between vmm::vmm_thread_event_loop() and VmmEpollHandler. pub(crate) struct EventContext { pub api_event_fd: EventFd, - pub api_event_flag: bool, - pub exit_evt_flag: bool, + pub api_event_triggered: bool, + pub exit_evt_triggered: bool, } impl EventContext { @@ -36,8 +36,8 @@ impl EventContext { pub fn new(api_event_fd: EventFd) -> Result { Ok(EventContext { api_event_fd, - api_event_flag: false, - exit_evt_flag: false, + api_event_triggered: false, + exit_evt_triggered: false, }) } } @@ -131,7 +131,7 @@ impl MutEventSubscriber for VmmEpollHandler { if let Err(e) = vmm.event_ctx.api_event_fd.read() { error!("event_manager: failed to read API eventfd, {:?}", e); } - vmm.event_ctx.api_event_flag = true; + vmm.event_ctx.api_event_triggered = true; self.vmm_event_count.fetch_add(1, Ordering::AcqRel); } EPOLL_EVENT_EXIT => { @@ -144,7 +144,7 @@ impl MutEventSubscriber for VmmEpollHandler { } None => warn!("event_manager: leftover exit event in epoll context!"), } - vmm.event_ctx.exit_evt_flag = true; + vmm.event_ctx.exit_evt_triggered = true; self.vmm_event_count.fetch_add(1, Ordering::AcqRel); } _ => error!("event_manager: unknown epoll slot number {}", events.data()), diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 739b1515c641..25450f380d74 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -22,6 +22,7 @@ use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, StartMicrovmError}; +use crate::event_manager::EventManager; /// Configures the system and should be called once per vm before starting vcpu threads. /// For aarch64, we only setup the FDT. @@ -142,4 +143,16 @@ impl Vm { ) .map_err(StartMicrovmError::ConfigureSystem) } + + pub(crate) fn register_events( + &mut self, + event_mgr: &mut EventManager, + ) -> std::result::Result<(), StartMicrovmError> { + let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?; + event_mgr + .register_exit_eventfd(reset_evt) + .map_err(|_| StartMicrovmError::RegisterEvent)?; + + Ok(()) + } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index fff044e3aa21..1089c99bf279 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -31,6 +31,7 @@ use crate::api::v1::{InstanceInfo, InstanceState}; use crate::device_manager::console_manager::DmesgWriter; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; use crate::error::{LoadInitrdError, Result, StartMicrovmError, StopMicrovmError}; +use crate::event_manager::EventManager; use crate::kvm_context::KvmContext; use crate::resource_manager::ResourceManager; use crate::vcpu::{VcpuManager, VcpuManagerError}; @@ -355,20 +356,7 @@ impl Vm { if !self.is_vm_initialized() { Ok(DeviceOpContext::create_boot_ctx(self, epoll_mgr)) } else { - #[cfg(feature = "hotplug")] - { - if self.upcall_client().is_none() { - Err(StartMicrovmError::UpcallMissVsock) - } else if self.is_upcall_client_ready() { - Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) - } else { - Err(StartMicrovmError::UpcallNotReady) - } - } - #[cfg(not(feature = "hotplug"))] - { - Err(StartMicrovmError::MicroVMAlreadyRunning) - } + self.create_device_hotplug_context(epoll_mgr) } } @@ -671,12 +659,70 @@ impl Vm { ) .map_err(StartMicrovmError::KernelLoader); } + + /// Set up the initial microVM state and start the vCPU threads. + /// + /// This is the main entrance of the Vm object, to bring up the virtual machine instance into + /// running state. + pub fn start_microvm( + &mut self, + event_mgr: &mut EventManager, + vmm_seccomp_filter: BpfProgram, + vcpu_seccomp_filter: BpfProgram, + ) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: received instance start command"); + if self.is_vm_initialized() { + return Err(StartMicrovmError::MicroVMAlreadyRunning); + } + + let request_ts = TimestampUs::default(); + self.start_instance_request_ts = request_ts.time_us; + self.start_instance_request_cpu_ts = request_ts.cputime_us; + + self.init_dmesg_logger(); + self.check_health()?; + + // Use expect() to crash if the other thread poisoned this lock. + self.shared_info + .write() + .expect("Failed to start microVM because shared info couldn't be written due to poisoned lock") + .state = InstanceState::Starting; + + self.init_guest_memory()?; + let vm_as = self.vm_as().cloned().ok_or(StartMicrovmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + ))?; + + self.init_vcpu_manager(vm_as.clone(), vcpu_seccomp_filter) + .map_err(StartMicrovmError::Vcpu)?; + self.init_microvm(event_mgr.epoll_manager(), vm_as.clone(), request_ts)?; + self.init_configure_system(&vm_as)?; + self.init_upcall()?; + + info!(self.logger, "VM: register events"); + self.register_events(event_mgr)?; + + info!(self.logger, "VM: start vcpus"); + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .start_boot_vcpus(vmm_seccomp_filter) + .map_err(StartMicrovmError::Vcpu)?; + + // Use expect() to crash if the other thread poisoned this lock. + self.shared_info + .write() + .expect("Failed to start microVM because shared info couldn't be written due to poisoned lock") + .state = InstanceState::Running; + + info!(self.logger, "VM started"); + Ok(()) + } } #[cfg(feature = "hotplug")] impl Vm { /// initialize upcall client for guest os - pub(crate) fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + fn new_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { // get vsock inner connector for upcall let inner_connector = self .device_manager @@ -698,8 +744,51 @@ impl Vm { Ok(()) } + fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM upcall init"); + if let Err(e) = self.new_upcall() { + info!( + self.logger, + "VM upcall init failed, no support hotplug: {}", e + ); + Err(e) + } else { + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .set_upcall_channel(self.upcall_client().clone()); + Ok(()) + } + } + /// Get upcall client. pub fn upcall_client(&self) -> &Option>> { &self.upcall_client } + + fn create_device_hotplug_context( + &self, + epoll_mgr: Option, + ) -> std::result::Result { + if self.upcall_client().is_none() { + Err(StartMicrovmError::UpcallMissVsock) + } else if self.is_upcall_client_ready() { + Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) + } else { + Err(StartMicrovmError::UpcallNotReady) + } + } +} + +#[cfg(not(feature = "hotplug"))] +impl Vm { + fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + Ok(()) + } + + fn create_device_hotplug_context( + &self, + _epoll_mgr: Option, + ) -> std::result::Result { + Err(StartMicrovmError::MicroVMAlreadyRunning) + } } diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index 74336b6da0d9..6bd74acc09d9 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -21,6 +21,7 @@ use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, Result, StartMicrovmError}; +use crate::event_manager::EventManager; use crate::vm::{Vm, VmError}; /// Configures the system and should be called once per vm before starting vcpu @@ -273,4 +274,20 @@ impl Vm { .create_pit2(pit_config) .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } + + pub(crate) fn register_events( + &mut self, + event_mgr: &mut EventManager, + ) -> std::result::Result<(), StartMicrovmError> { + let reset_evt = self + .device_manager + .get_reset_eventfd() + .map_err(StartMicrovmError::DeviceManager)?; + event_mgr + .register_exit_eventfd(&reset_evt) + .map_err(|_| StartMicrovmError::RegisterEvent)?; + self.reset_eventfd = Some(reset_evt); + + Ok(()) + } } diff --git a/src/dragonball/src/vmm.rs b/src/dragonball/src/vmm.rs index a2c75ff7166f..362415dbd9dd 100644 --- a/src/dragonball/src/vmm.rs +++ b/src/dragonball/src/vmm.rs @@ -124,18 +124,18 @@ impl Vmm { } let mut v = vmm.lock().unwrap(); - if v.event_ctx.api_event_flag { + if v.event_ctx.api_event_triggered { // The run_vmm_action() needs to access event_mgr, so it could // not be handled in EpollHandler::handle_events(). It has been // delayed to the main loop. - v.event_ctx.api_event_flag = false; + v.event_ctx.api_event_triggered = false; service .run_vmm_action(&mut v, &mut event_mgr) .unwrap_or_else(|_| { warn!("got spurious notification from api thread"); }); } - if v.event_ctx.exit_evt_flag { + if v.event_ctx.exit_evt_triggered { info!("Gracefully terminated VMM control loop"); return v.stop(EXIT_CODE_OK as i32); } From 89b9ba86032691f92edfc8b50d6b22067a836228 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 01:42:19 +0800 Subject: [PATCH 0092/1953] dragonball: add set_vm_configuration api Set virtual machine configuration configurations. Signed-off-by: wllenyj --- src/dragonball/src/api/v1/machine_config.rs | 86 +++++++++++ src/dragonball/src/api/v1/mod.rs | 4 + src/dragonball/src/api/v1/vmm_action.rs | 154 +++++++++++++++++++- src/dragonball/src/vm/mod.rs | 9 +- 4 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 src/dragonball/src/api/v1/machine_config.rs diff --git a/src/dragonball/src/api/v1/machine_config.rs b/src/dragonball/src/api/v1/machine_config.rs new file mode 100644 index 000000000000..e4ae2286798c --- /dev/null +++ b/src/dragonball/src/api/v1/machine_config.rs @@ -0,0 +1,86 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/// We only support this number of vcpus for now. Mostly because we have set all vcpu related metrics as u8 +/// and breaking u8 will take extra efforts. +pub const MAX_SUPPORTED_VCPUS: u8 = 254; + +/// Memory hotplug value should have alignment in this size (unit: MiB) +pub const MEMORY_HOTPLUG_ALIGHMENT: u8 = 64; + +/// Errors associated with configuring the microVM. +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum VmConfigError { + /// Cannot update the configuration of the microvm post boot. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The max vcpu count is invalid. + #[error("the vCPU number shouldn't large than {}", MAX_SUPPORTED_VCPUS)] + VcpuCountExceedsMaximum, + + /// The vcpu count is invalid. When hyperthreading is enabled, the `cpu_count` must be either + /// 1 or an even number. + #[error( + "the vCPU number '{0}' can only be 1 or an even number when hyperthreading is enabled" + )] + InvalidVcpuCount(u8), + + /// The threads_per_core is invalid. It should be either 1 or 2. + #[error("the threads_per_core number '{0}' can only be 1 or 2")] + InvalidThreadsPerCore(u8), + + /// The cores_per_die is invalid. It should be larger than 0. + #[error("the cores_per_die number '{0}' can only be larger than 0")] + InvalidCoresPerDie(u8), + + /// The dies_per_socket is invalid. It should be larger than 0. + #[error("the dies_per_socket number '{0}' can only be larger than 0")] + InvalidDiesPerSocket(u8), + + /// The socket number is invalid. It should be either 1 or 2. + #[error("the socket number '{0}' can only be 1 or 2")] + InvalidSocket(u8), + + /// max vcpu count inferred from cpu topology(threads_per_core * cores_per_die * dies_per_socket * sockets) should be larger or equal to vcpu_count + #[error("the max vcpu count inferred from cpu topology '{0}' (threads_per_core * cores_per_die * dies_per_socket * sockets) should be larger or equal to vcpu_count")] + InvalidCpuTopology(u8), + + /// The max vcpu count is invalid. + #[error( + "the max vCPU number '{0}' shouldn't less than vCPU count and can only be 1 or an even number when hyperthreading is enabled" + )] + InvalidMaxVcpuCount(u8), + + /// The memory size is invalid. The memory can only be an unsigned integer. + #[error("the memory size 0x{0:x}MiB is invalid")] + InvalidMemorySize(usize), + + /// The hotplug memory size is invalid. The memory can only be an unsigned integer. + #[error( + "the hotplug memory size '{0}' (MiB) is invalid, must be multiple of {}", + MEMORY_HOTPLUG_ALIGHMENT + )] + InvalidHotplugMemorySize(usize), + + /// The memory type is invalid. + #[error("the memory type '{0}' is invalid")] + InvalidMemType(String), + + /// The memory file path is invalid. + #[error("the memory file path is invalid")] + InvalidMemFilePath(String), + + /// NUMA region memory size is invalid + #[error("Total size of memory in NUMA regions: {0}, should matches memory size in config")] + InvalidNumaRegionMemorySize(usize), + + /// NUMA region vCPU count is invalid + #[error("Total counts of vCPUs in NUMA regions: {0}, should matches max vcpu count in config")] + InvalidNumaRegionCpuCount(u16), + + /// NUMA region vCPU count is invalid + #[error("Max id of vCPUs in NUMA regions: {0}, should matches max vcpu count in config")] + InvalidNumaRegionCpuMaxId(u16), +} diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs index c1ab3f5d322e..2077c982e95f 100644 --- a/src/dragonball/src/api/v1/mod.rs +++ b/src/dragonball/src/api/v1/mod.rs @@ -15,3 +15,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER /// Wrapper over the microVM general information. mod instance_info; pub use self::instance_info::{InstanceInfo, InstanceState}; + +/// Wrapper for configuring the memory and CPU of the microVM. +mod machine_config; +pub use self::machine_config::{VmConfigError, MAX_SUPPORTED_VCPUS}; diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 7f8a8b98db8e..754d877b4984 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -9,12 +9,12 @@ use std::fs::File; use std::sync::mpsc::{Receiver, Sender, TryRecvError}; -use log::{debug, error, warn}; +use log::{debug, error, info, warn}; use vmm_sys_util::eventfd::EventFd; use crate::error::{Result, StartMicrovmError, StopMicrovmError}; use crate::event_manager::EventManager; -use crate::vm::{KernelConfigInfo, VmConfigInfo}; +use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; use super::*; @@ -38,6 +38,11 @@ pub enum VmmActionError { /// The action `StopMicroVm` failed either because of bad user input or an internal error. #[error("failed to shutdown the VM: {0}")] StopMicrovm(#[source] StopMicrovmError), + + /// One of the actions `GetVmConfiguration` or `SetVmConfiguration` failed either because of bad + /// input or an internal error. + #[error("failed to set configuration for the VM: {0}")] + MachineConfig(#[source] VmConfigError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -55,6 +60,13 @@ pub enum VmmAction { /// When vmm is used as the crate by the other process, which is need to /// shutdown the vcpu threads and destory all of the object. ShutdownMicroVm, + + /// Get the configuration of the microVM. + GetVmConfiguration, + + /// Set the microVM configuration (memory & vcpu) using `VmConfig` as input. This + /// action can only be called before the microVM has booted. + SetVmConfiguration(VmConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -63,6 +75,8 @@ pub enum VmmAction { pub enum VmmData { /// No data is sent on the channel. Empty, + /// The microVM configuration represented by `VmConfigInfo`. + MachineConfiguration(Box), } /// Request data type used to communicate between the API and the VMM. @@ -114,6 +128,12 @@ impl VmmService { } VmmAction::StartMicroVm => self.start_microvm(vmm, event_mgr), VmmAction::ShutdownMicroVm => self.shutdown_microvm(vmm), + VmmAction::GetVmConfiguration => Ok(VmmData::MachineConfiguration(Box::new( + self.machine_config.clone(), + ))), + VmmAction::SetVmConfiguration(machine_config) => { + self.set_vm_configuration(vmm, machine_config) + } }; debug!("send vmm response: {:?}", response); @@ -193,4 +213,134 @@ impl VmmService { Ok(VmmData::Empty) } + + /// Set virtual machine configuration configurations. + pub fn set_vm_configuration( + &mut self, + vmm: &mut Vmm, + machine_config: VmConfigInfo, + ) -> VmmRequestResult { + use self::VmConfigError::*; + use self::VmmActionError::MachineConfig; + + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + if vm.is_vm_initialized() { + return Err(MachineConfig(UpdateNotAllowedPostBoot)); + } + + // If the check is successful, set it up together. + let mut config = vm.vm_config().clone(); + if config.vcpu_count != machine_config.vcpu_count { + let vcpu_count = machine_config.vcpu_count; + // Check that the vcpu_count value is >=1. + if vcpu_count == 0 { + return Err(MachineConfig(InvalidVcpuCount(vcpu_count))); + } + config.vcpu_count = vcpu_count; + } + + if config.cpu_topology != machine_config.cpu_topology { + let cpu_topology = &machine_config.cpu_topology; + // Check if dies_per_socket, cores_per_die, threads_per_core and socket number is valid + if cpu_topology.threads_per_core < 1 || cpu_topology.threads_per_core > 2 { + return Err(MachineConfig(InvalidThreadsPerCore( + cpu_topology.threads_per_core, + ))); + } + let vcpu_count_from_topo = cpu_topology + .sockets + .checked_mul(cpu_topology.dies_per_socket) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))? + .checked_mul(cpu_topology.cores_per_die) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))? + .checked_mul(cpu_topology.threads_per_core) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))?; + if vcpu_count_from_topo > MAX_SUPPORTED_VCPUS { + return Err(MachineConfig(VcpuCountExceedsMaximum)); + } + if vcpu_count_from_topo < config.vcpu_count { + return Err(MachineConfig(InvalidCpuTopology(vcpu_count_from_topo))); + } + config.cpu_topology = cpu_topology.clone(); + } else { + // the same default + let mut default_cpu_topology = CpuTopology { + threads_per_core: 1, + cores_per_die: config.vcpu_count, + dies_per_socket: 1, + sockets: 1, + }; + if machine_config.max_vcpu_count > config.vcpu_count { + default_cpu_topology.cores_per_die = machine_config.max_vcpu_count; + } + config.cpu_topology = default_cpu_topology; + } + let cpu_topology = &config.cpu_topology; + let max_vcpu_from_topo = cpu_topology.threads_per_core + * cpu_topology.cores_per_die + * cpu_topology.dies_per_socket + * cpu_topology.sockets; + // If the max_vcpu_count inferred by cpu_topology is not equal to + // max_vcpu_count, max_vcpu_count will be changed. currently, max vcpu size + // is used when cpu_topology is not defined and help define the cores_per_die + // for the default cpu topology. + let mut max_vcpu_count = machine_config.max_vcpu_count; + if max_vcpu_count < config.vcpu_count { + return Err(MachineConfig(InvalidMaxVcpuCount(max_vcpu_count))); + } + if max_vcpu_from_topo != max_vcpu_count { + max_vcpu_count = max_vcpu_from_topo; + info!("Since max_vcpu_count is not equal to cpu topo information, we have changed the max vcpu count to {}", max_vcpu_from_topo); + } + config.max_vcpu_count = max_vcpu_count; + + config.cpu_pm = machine_config.cpu_pm; + config.mem_type = machine_config.mem_type; + + let mem_size_mib_value = machine_config.mem_size_mib; + // Support 1TB memory at most, 2MB aligned for huge page. + if mem_size_mib_value == 0 || mem_size_mib_value > 0x10_0000 || mem_size_mib_value % 2 != 0 + { + return Err(MachineConfig(InvalidMemorySize(mem_size_mib_value))); + } + config.mem_size_mib = mem_size_mib_value; + + config.mem_file_path = machine_config.mem_file_path.clone(); + + let reserve_memory_bytes = machine_config.reserve_memory_bytes; + // Reserved memory must be 2MB aligned and less than half of the total memory. + if reserve_memory_bytes % 0x200000 != 0 + || reserve_memory_bytes > (config.mem_size_mib as u64) << 20 + { + return Err(MachineConfig(InvalidReservedMemorySize( + reserve_memory_bytes as usize >> 20, + ))); + } + config.reserve_memory_bytes = reserve_memory_bytes; + if config.mem_type == "hugetlbfs" && config.mem_file_path.is_empty() { + return Err(MachineConfig(InvalidMemFilePath("".to_owned()))); + } + config.vpmu_feature = machine_config.vpmu_feature; + + let vm_id = vm.shared_info().read().unwrap().id.clone(); + let serial_path = match machine_config.serial_path { + Some(value) => value, + None => { + if config.serial_path.is_none() { + String::from("/run/dragonball/") + &vm_id + "_com1" + } else { + // Safe to unwrap() because we have checked it has a value. + config.serial_path.as_ref().unwrap().clone() + } + } + }; + config.serial_path = Some(serial_path); + + vm.set_vm_config(config.clone()); + self.machine_config = config; + + Ok(VmmData::Empty) + } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 1089c99bf279..7c83aabb4c0f 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -689,9 +689,12 @@ impl Vm { .state = InstanceState::Starting; self.init_guest_memory()?; - let vm_as = self.vm_as().cloned().ok_or(StartMicrovmError::AddressManagerError( - AddressManagerError::GuestMemoryNotInitialized, - ))?; + let vm_as = self + .vm_as() + .cloned() + .ok_or(StartMicrovmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + ))?; self.init_vcpu_manager(vm_as.clone(), vcpu_seccomp_filter) .map_err(StartMicrovmError::Vcpu)?; From a1593322bd013e12127fd24a0a6216ee9996d1ef Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 02:10:58 +0800 Subject: [PATCH 0093/1953] dragonball: add vsock api to api server Enables vsock to use the api for device configuration. Signed-off-by: wllenyj --- src/dragonball/src/api/v1/vmm_action.rs | 50 +++++++++++++++++++++++++ src/dragonball/src/vm/mod.rs | 5 +++ 2 files changed, 55 insertions(+) diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 754d877b4984..d4d598da89a3 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -17,6 +17,9 @@ use crate::event_manager::EventManager; use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; +#[cfg(feature = "virtio-vsock")] +use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; + use super::*; /// Wrapper for all errors associated with VMM actions. @@ -43,6 +46,11 @@ pub enum VmmActionError { /// input or an internal error. #[error("failed to set configuration for the VM: {0}")] MachineConfig(#[source] VmConfigError), + + #[cfg(feature = "virtio-vsock")] + /// The action `InsertVsockDevice` failed either because of bad user input or an internal error. + #[error("failed to add virtio-vsock device: {0}")] + Vsock(#[source] VsockDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -67,6 +75,12 @@ pub enum VmmAction { /// Set the microVM configuration (memory & vcpu) using `VmConfig` as input. This /// action can only be called before the microVM has booted. SetVmConfiguration(VmConfigInfo), + + #[cfg(feature = "virtio-vsock")] + /// Add a new vsock device or update one that already exists using the + /// `VsockDeviceConfig` as input. This action can only be called before the microVM has + /// booted. The response is sent using the `OutcomeSender`. + InsertVsockDevice(VsockDeviceConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -134,6 +148,8 @@ impl VmmService { VmmAction::SetVmConfiguration(machine_config) => { self.set_vm_configuration(vmm, machine_config) } + #[cfg(feature = "virtio-vsock")] + VmmAction::InsertVsockDevice(vsock_cfg) => self.add_vsock_device(vmm, vsock_cfg), }; debug!("send vmm response: {:?}", response); @@ -343,4 +359,38 @@ impl VmmService { Ok(VmmData::Empty) } + + #[cfg(feature = "virtio-vsock")] + fn add_vsock_device(&self, vmm: &mut Vmm, config: VsockDeviceConfigInfo) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + if vm.is_vm_initialized() { + return Err(VmmActionError::Vsock( + VsockDeviceError::UpdateNotAllowedPostBoot, + )); + } + + // VMADDR_CID_ANY (-1U) means any address for binding; + // VMADDR_CID_HYPERVISOR (0) is reserved for services built into the hypervisor; + // VMADDR_CID_RESERVED (1) must not be used; + // VMADDR_CID_HOST (2) is the well-known address of the host. + if config.guest_cid <= 2 { + return Err(VmmActionError::Vsock(VsockDeviceError::GuestCIDInvalid( + config.guest_cid, + ))); + } + + info!("add_vsock_device: {:?}", config); + let ctx = vm.create_device_op_context(None).map_err(|e| { + info!("create device op context error: {:?}", e); + VmmActionError::Vsock(VsockDeviceError::UpdateNotAllowedPostBoot) + })?; + + vm.device_manager_mut() + .vsock_manager + .insert_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Vsock) + } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 7c83aabb4c0f..4063b77708e4 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -250,6 +250,11 @@ impl Vm { &self.device_manager } + /// Gets a mutable reference to the device manager by this VM. + pub fn device_manager_mut(&mut self) -> &mut DeviceManager { + &mut self.device_manager + } + /// Get a reference to EpollManager. pub fn epoll_manager(&self) -> &EpollManager { &self.epoll_manager From 090de2dae2d19a08d32f91bfaba167a715a56682 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Tue, 24 May 2022 00:11:25 +0800 Subject: [PATCH 0094/1953] dragonball: fix the clippy errors. fix clippy errors and do fmt in this PR. Signed-off-by: wllenyj --- src/dragonball/src/address_space_manager.rs | 20 +++++++++---------- src/dragonball/src/config_manager.rs | 15 +++++++++++++- .../src/device_manager/console_manager.rs | 14 ++++++------- src/dragonball/src/vcpu/vcpu_manager.rs | 12 ++++++----- 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/dragonball/src/address_space_manager.rs b/src/dragonball/src/address_space_manager.rs index 2e4276626bac..5a54c0da01c1 100644 --- a/src/dragonball/src/address_space_manager.rs +++ b/src/dragonball/src/address_space_manager.rs @@ -642,7 +642,7 @@ impl AddressSpaceMgr { let node = self .numa_nodes .entry(guest_numa_node_id) - .or_insert(NumaNode::new()); + .or_insert_with(NumaNode::new); node.add_info(&NumaNodeInfo { base: region.start_addr(), size: region.len(), @@ -736,7 +736,7 @@ mod tests { .unwrap(); val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x1)).unwrap(); assert_eq!(val, 0xa5); - val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x0)).unwrap(); + val = gmem.read_obj(GuestAddress(GUEST_MEM_START)).unwrap(); assert_eq!(val, 1); val = gmem.read_obj(GuestAddress(GUEST_MEM_START + 0x2)).unwrap(); assert_eq!(val, 3); @@ -837,7 +837,7 @@ mod tests { size: mem_size >> 20, host_numa_node_id: None, guest_numa_node_id: Some(0), - vcpu_ids: cpu_vec.clone(), + vcpu_ids: cpu_vec, }]; let mut builder = AddressSpaceMgrBuilder::new("hugeshmem", "").unwrap(); builder.toggle_prealloc(true); @@ -852,9 +852,9 @@ mod tests { assert_eq!(builder.mem_type, "shmem"); assert_eq!(builder.mem_file, "/tmp/shmem"); assert_eq!(builder.mem_index, 0); - assert_eq!(builder.mem_suffix, true); - assert_eq!(builder.mem_prealloc, false); - assert_eq!(builder.dirty_page_logging, false); + assert!(builder.mem_suffix); + assert!(!builder.mem_prealloc); + assert!(!builder.dirty_page_logging); assert!(builder.vmfd.is_none()); assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem0"); @@ -867,10 +867,10 @@ mod tests { assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem"); assert_eq!(builder.mem_index, 3); - builder.toggle_prealloc(true); - builder.toggle_dirty_page_logging(true); - assert_eq!(builder.mem_prealloc, true); - assert_eq!(builder.dirty_page_logging, true); + builder.set_prealloc(true); + builder.set_dirty_page_logging(true); + assert!(builder.mem_prealloc); + assert!(builder.dirty_page_logging); } #[test] diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs index de16ab0fbffa..e25b792e7bff 100644 --- a/src/dragonball/src/config_manager.rs +++ b/src/dragonball/src/config_manager.rs @@ -178,6 +178,15 @@ where info_list: Vec>, } +impl Default for DeviceConfigInfos +where + T: ConfigItem + Clone, +{ + fn default() -> Self { + Self::new() + } +} + impl DeviceConfigInfos where T: ConfigItem + Clone, @@ -221,12 +230,16 @@ where } } - #[allow(dead_code)] /// Get number of device configuration information objects. pub fn len(&self) -> usize { self.info_list.len() } + /// Returns true if the device configuration information objects is empty. + pub fn is_empty(&self) -> bool { + self.info_list.len() == 0 + } + /// Add a device configuration information object at the tail. pub fn push(&mut self, info: DeviceConfigInfo) { self.info_list.push(info); diff --git a/src/dragonball/src/device_manager/console_manager.rs b/src/dragonball/src/device_manager/console_manager.rs index 4ebad58167c6..1e3b2a2f2267 100644 --- a/src/dragonball/src/device_manager/console_manager.rs +++ b/src/dragonball/src/device_manager/console_manager.rs @@ -426,13 +426,13 @@ mod tests { }; writer.flush().unwrap(); - writer.write("".as_bytes()).unwrap(); - writer.write("\n".as_bytes()).unwrap(); - writer.write("\n\n".as_bytes()).unwrap(); - writer.write("\n\n\n".as_bytes()).unwrap(); - writer.write("12\n23\n34\n56".as_bytes()).unwrap(); - writer.write("78".as_bytes()).unwrap(); - writer.write("90\n".as_bytes()).unwrap(); + writer.write_all("".as_bytes()).unwrap(); + writer.write_all("\n".as_bytes()).unwrap(); + writer.write_all("\n\n".as_bytes()).unwrap(); + writer.write_all("\n\n\n".as_bytes()).unwrap(); + writer.write_all("12\n23\n34\n56".as_bytes()).unwrap(); + writer.write_all("78".as_bytes()).unwrap(); + writer.write_all("90\n".as_bytes()).unwrap(); writer.flush().unwrap(); } diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 189dfc61555c..278a88ecea1e 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -784,14 +784,13 @@ impl VcpuManager { mod hotplug { use std::cmp::Ordering; + use dbs_upcall::{CpuDevRequest, DevMgrRequest}; + use super::*; - #[cfg(not(test))] - use dbs_upcall::CpuDevRequest; - use dbs_upcall::{DevMgrRequest, DevMgrResponse, UpcallClientRequest, UpcallClientResponse}; - #[cfg(all(target_arch = "x86_64", not(test)))] + #[cfg(all(target_arch = "x86_64"))] use dbs_boot::mptable::APIC_VERSION; - #[cfg(all(target_arch = "aarch64", not(test)))] + #[cfg(all(target_arch = "aarch64"))] const APIC_VERSION: u8 = 0; #[cfg(feature = "dbs-upcall")] @@ -933,6 +932,9 @@ mod hotplug { upcall_client: Arc>, request: DevMgrRequest, ) -> std::result::Result<(), VcpuManagerError> { + // This is used to fix clippy warnings. + use dbs_upcall::{DevMgrResponse, UpcallClientRequest, UpcallClientResponse}; + let vcpu_state_event = self.vcpu_state_event.try_clone().unwrap(); let vcpu_state_sender = self.vcpu_state_sender.clone(); From 42ea854eb6f4ba6a590f16349eadbdfbe4e6d421 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 3 May 2022 13:49:44 +0800 Subject: [PATCH 0095/1953] docs: kata 3.0 Architecture An introduction for kata 3.0 architecture design Fixes:#4193 Signed-off-by: Zhongtao Hu Signed-off-by: Quanwei Zhou Signed-off-by: Christophe de Dinechin --- README.md | 1 + docs/design/architecture_3.0/README.md | 93 ++++++++++++++++++ .../architecture_3.0/images/architecture.png | Bin 0 -> 102890 bytes .../architecture_3.0/images/built_in_vmm.png | Bin 0 -> 67370 bytes .../architecture_3.0/images/framework.png | Bin 0 -> 139722 bytes .../images/not_built_in_vmm.png | Bin 0 -> 73380 bytes .../images/resourceManager.png | Bin 0 -> 142678 bytes .../images/source_code/kata_3.0_images.drawio | 1 + 8 files changed, 95 insertions(+) create mode 100644 docs/design/architecture_3.0/README.md create mode 100644 docs/design/architecture_3.0/images/architecture.png create mode 100644 docs/design/architecture_3.0/images/built_in_vmm.png create mode 100644 docs/design/architecture_3.0/images/framework.png create mode 100644 docs/design/architecture_3.0/images/not_built_in_vmm.png create mode 100644 docs/design/architecture_3.0/images/resourceManager.png create mode 100644 docs/design/architecture_3.0/images/source_code/kata_3.0_images.drawio diff --git a/README.md b/README.md index e66917251e05..4a7a6ea9a7bf 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ See the [official documentation](docs) including: - [Developer guide](docs/Developer-Guide.md) - [Design documents](docs/design) - [Architecture overview](docs/design/architecture) + - [Architecture 3.0 overview](docs/design/architecture_3.0/) ## Configuration diff --git a/docs/design/architecture_3.0/README.md b/docs/design/architecture_3.0/README.md new file mode 100644 index 000000000000..6211796474df --- /dev/null +++ b/docs/design/architecture_3.0/README.md @@ -0,0 +1,93 @@ +# Kata 3.0 Architecture +## Overview +In cloud-native scenarios, there is an increased demand for container startup speed, resource consumption, stability, and security. To achieve this, we would like to introduce you to a solid and secure Rust version kata-runtime. We chose Rust because it is designed to prevent developers from accidentally introducing flaws in their code that can lead to buffer overflows, missing pointer checks, integer range errors, or other memory-related vulnerabilities. Besides, in contrast to Go, Rust makes a variety of design trade-offs in order to obtain the fastest feasible execution performance. +Also, we provide the following designs: + +- Turn key solution with builtin Dragonball Sandbox +- Async io to reduce resource consumption +- Extensible framework for multiple services, runtimes and hypervisors +- Lifecycle management for sandbox and container associated resources +## Design +### Architecture +![architecture](./images/architecture.png) +### Built-in VMM +#### Why Need Built-in VMM +![not_builtin_vmm](./images/not_built_in_vmm.png) +As shown in the figure, runtime and VMM are separate processes. The runtime process forks the VMM process and interacts through the inter-process RPC. Typically, process interaction consumes more resources than peers within the process, and it will result in relatively low efficiency. At the same time, the cost of resource operation and maintenance should be considered. For example, when performing resource recovery under abnormal conditions, the exception of any process must be detected by others and activate the appropriate resource recovery process. If there are additional processes, the recovery becomes even more difficult. +#### How To Support Built-in VMM +We provide Dragonball Sandbox to enable built-in VMM by integrating VMM's function into the Rust library. We could perform VMM-related functionalities by using the library. Because runtime and VMM are in the same process, there is a benefit in terms of message processing speed and API synchronization. It can also guarantee the consistency of the runtime and the VMM life cycle, reducing resource recovery and exception handling maintenance, as shown in the figure: +![builtin_vmm](./images/built_in_vmm.png) +### Async Support +#### Why Need Async +**Async is already in stable Rust and allows us to write async code** + +- Async provides significantly reduced CPU and memory overhead, especially for workloads with a large amount of IO-bound tasks +- Async is zero-cost in Rust, which means that you only pay for what you use. Specifically, you can use async without heap allocations and dynamic dispatch, which greatly improves efficiency +- For more (see [Why Async?](https://rust-lang.github.io/async-book/01_getting_started/02_why_async.html) and [The State of Asynchronous Rust](https://rust-lang.github.io/async-book/01_getting_started/03_state_of_async_rust.html)). + +**There may be several problems if implementing kata-runtime with Sync Rust** + +- Too many threads with a new TTRPC connection + - TTRPC threads: reaper thread(1) + listener thread(1) + client handler(2) +- Add 3 I/O threads with a new container +- In Sync mode, implementing a timeout mechanism is challenging. For example, in TTRPC API interaction, the timeout mechanism is difficult to align with Golang +#### How To Support Async +The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the tokio thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block io and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable. + +**For N tokio worker threads and M containers** + +- Sync runtime(both OS thread and tokio task are OS thread but without tokio worker thread) OS thread number: 4 + 12*M +- Async runtime(only OS thread is OS thread) OS thread number: 2 + N +```shell +├─ main(OS thread) +├─ async-logger(OS thread) +└─ tokio worker(N * OS thread) + ├─ agent log forwarder(1 * tokio task) + ├─ health check thread(1 * tokio task) + ├─ TTRPC reaper thread(M * tokio task) + ├─ TTRPC listener thread(M * tokio task) + ├─ TTRPC client handler thread(7 * M * tokio task) + ├─ container stdin io thread(M * tokio task) + ├─ container stdin io thread(M * tokio task) + └─ container stdin io thread(M * tokio task) +``` +### Extensible Framework +The Rust version kata-runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration. + +![framework](./images/framework.png) +### Resource Manager +In our case, there will be a variety of resources, and every resource has several subtypes. Especially for Virt-Container, every subtype of resource has different operations. And there may be dependencies, such as the share-fs rootfs and the share-fs volume will use share-fs resources to share files to the VM. Currently, network and share-fs are regarded as sandbox resources, while rootfs, volume, and cgroup are regarded as container resources. Also, we abstract a common interface for each resource and use subclass operations to evaluate the differences between different subtypes. +![resource manager](./images/resourceManager.png) + +## Roadmap + +- Stage 1: provide basic features（current delivered） +- Stage 2: support common features +- Stage 3: support full features + +| **Class** | **Sub-Class** | **Development Stage** | +| --- | --- | --- | +| service | task service | Stage 1 | +| | extend service | Stage 3 | +| | image service | Stage 3 | +| runtime handler | Virt-Container | Stage 1 | +| | Wasm-Container | Stage 3 | +| | Linux-Container | Stage 3 | +| Endpoint | Veth Endpoint | Stage 1 | +| | Physical Endpoint | Stage 2 | +| | Tap Endpoint | Stage 2 | +| | Tuntap Endpoint | Stage 2 | +| | IPVlan Endpoint | Stage 3 | +| | MacVlan Endpoint | Stage 3 | +| | MacVtap Endpoint | Stage 3 | +| | VhostUserEndpoint | Stage 3 | +| Network Interworking Model | Tc filter | Stage 1 | +| | Route | Stage 1 | +| | MacVtap | Stage 3 | +| Storage | virtiofs | Stage 1 | +| | nydus | Stage 2 | +| hypervisor | Dragonball | Stage 1 | +| | Qemu | Stage 2 | +| | Acrn | Stage 3 | +| | CloudHypervisor | Stage 3 | +| | Firecracker | Stage 3 | diff --git a/docs/design/architecture_3.0/images/architecture.png b/docs/design/architecture_3.0/images/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..5825f1eb4e73b0ecd91deeeb6a63192f9551d2dd GIT binary patch literal 102890 zcmb5W2NVVESr+(HlqMYn1xz6Pi0dXh;aKRgf zJ`2V^i}*d@C*AjT;w0g@m|jE#3?*KJ;$ZhDpC#*}zKF2kcU@Eo_!5(->W)j|Ay{8i zbl`NDc+KfGN5k2C=o)a607KAgpx2y83V#iLH0a&^6JQb?fx*68kOo8Xo1S*uA(?{g z_ZQJeY&H2(lq)z~SQ8hgrG^oXyb&ifqVr*5HQO<(F^J2B1$bJuC&6Bx^TRFW(2V zpJ9bxI?;gON8D4JVBWPDG1`!rR&sYj$+fK|DvRj0mcGJ7{gf_G<|<>Ql~rPB~vPQAmZT z+uem;dQP>xY#BQuH!DFq>&nT!Y!OsUEH8kY$eD*;ZIuh^-YD<7xtUeUZ&_qrz(G%- zjB46K>dCYV>8>L`rs3IMYEp(s`5|K%)8ZI1j!el>>watNO$dmE_ob{4vx@Qw1ie7` zv*vglwvG-NtZf(&MFn8Kg|ysrUCP03$GlQc2nn3~dAo~X$C!*GdNywtwUE=z_C&*C zg56>y6}_A)H+I)KD?-xe9Z^L zGAIgl82WSe5T=@|ZKBr3_Pk5s=*8f(Ei|+p8F1SBHt9em&m7 zx+2Jl)j)@YJU(@y$7j<>by_!>VPRbbeqXKYSWR@SbS!j*o$5FhGspN|l|Y7=ZWT3I zYA+EK3(4%)OD*FzT7o0_UZ!V36j=yY17lp<#&HU(DG96=GoV_GiD9%*NZ@{J+6;Pi zK7MsBOZ#T&auVy?Tmu}7?f}SI{6bA+ACh}6B2PLp?NplJ+|3o}zrs+v0x>bGT zV)bp_L2IUSKp7<5W05aGr}MsH*XF|7d=#$kCBymXI`OCWko5`6`B}Aa>FxehmqOrp zt-!=Uds-|QAqJ9DlCzdP%+A=cZ7`PmOfJF|X?bva742mjgP(E%7kZ~!l46TuIXZ%5 zS!aVNl*gsnIq5D8yFk?xFqh1H8_)@rYVUysXyL(UjU_u^eucQUO7*e_`~zJdhj2e1 zc>o$LpyTACh$j-7M_c@SHt_Iaq1F9rG%QTmNF~Ju6P_xc00xrb%L=G-h`>$6S{plR z7Py6^FLWN9%DNksd&ZiTF85-@vR7e6=!%a_>!uU;^2!m$RJ90cKO;IA9KnMh(4ut( zrS47SL{F`RH#sja*D9%WQwW;jPQ_Bt zVHnmcJ(Cz~bprcHbYM^OFyC0>-U<_NdyLwzdp8fuW6p#NvZ7P1i2EGi0hY;~>B16% zR&GOuwxL0U0_+Jxs~3`rG)}Chi9moCWz_Uqds;?ah= zPLn`*YN@Ea z>=UY7AHi_{>8U+)OgFt-IxC*-N4m;Ve^09mQ<{MZHyCIb@|@GPli(+BJx+6HuVIwm z-8*`=iTt^rud6LTV+L*|Vgf3~NOUX%l?R?%NPSFEZ}(@P*mq4=k}4X_I?7uLn}BZ< zp0LFTGJSX|f%35)Pf~Clm@EeO_0lO)n5$5-E-z0-qUVceod)rIQ6TG5*rsxw+%~d8 zv{Em9?IL)1OU77i;8=```hG5S%+z3W$RTdtT*OK>y%cbCx|4M{qFrnmo(+UJhkK5- z6wy3I;*|8vqsNHs#@Ns2b;rE%GB&xLni*&!gW`gRODL0sZv9L|Z&}rHNT*xkJv`nj ziAUKQ{q0627v^!A^mxKFBXH0Jzm4&EDoFnB!3B}EPb}B_Knkur-bO)JnU@=W&wF#= zMx;}0A-vC9i%Fb3yLD)p8jHJAn6QuD9$e#6)P?HtS8=DrEC2&%YnltT%&<90ELHeh zvd`8rqE+;?pWKSF!PMT)lZB}&Eb{{8JtkExk%x&MQ^% z4tVr#bVCnOUQ7Up&cjR43FQg@8OGRn;^jjY%PaVp4&R1uqBiwSgYP$gY=U&S$tP`u zX=XFy{FyGuSr)g|rjyn@bD^PMJ@3gi4JCBfAz}2aO7`u%QY^k6O-C4cXbc*?<9Fs7 z0wlr!*9cq}r!cZt`JN)=+Vi65B!8%Vrd%4}9~j3|B2kHEg((S!MO8CPB-y>nH!~;> zCm^Ubi9B7Dc<{xVDMXHB7fK~CMqT*l1CJFQoRi~4?nmH<6A`Jpa@g~}nhD7X_(_qC zE7DbKUREu1mWtH0sJ3=>+B@%BfyyZd&?M6Yt?UE@?{b5xpkRu()Vs_1Y}vN26I;BZ z=|koX@C5X5W|9t2wa`}JO7Oj7_wmuYD^Gd=V++CTD;KVm=-XVf)0KI2C&H0f^_kE` zO2rj4C!mN40g zlrFy&i+j>-clE+*JskJS?2{<~>40y{=q1tikVbrB#!8*X`ogQFkU5#1g!F@Yo&?cM zrXtvm8F;owNAi=E--%B35GrIIQ3MlyRcS#c+R5yR*;D8I{B#j}mW#H%<}6bX(>UQ4 zUBZkLO*}&sV=5|bXL2;)pvNwGH)xrA@<~OU=i>oVI>VT=sH0Q4#!y}b1Gw%i&88wN zY)fKfn(nPLPF@YuUS_|Om%zQ)XYEQEzO0f`%8d6z?7D}%OrDp9Sh<+mbJa_?3@Hsp z6xnu4)v0vI@KxF(tF!t5{S)^|gCf)=h;*qdE`{lf|Ac!4KMN=J)aG!r&on%(*Ct$Y zN1~B%=QA7bAuUp?>m06EsFv>vS5Odl0K&spnh&Zj-QA(X(@IGsb7~Mb7|Gz2W~tQx zkq~lM9)t+CL(J(n*O=5hc7H&K_Hr#JFcP`MDV>!pLv&{e9#6fJnWo$J|^_QL3R&}vV2ZJN0MgI^S*vC z&+yJ>2r1ar1*_B|#4h^X+8H_RsP4S;=?=>#Ik;-ok(g3qy36*si@Ts#_}crkr*(o! zO#1W6D66Wkq(BeFbeR#L0Q{`U`hi5jD_5LAI@f0&o-bgi5+w~hj9_&h|@9 z@Wc9=gDOIIZP8mSChUFRabz=%`~Gaqgj$2gv%1*5N=)W#TSdK$3M5e`xXzoL`4NI& zjNA4%Pu&107JLye#Rih8kqRvpn^h#vp~GIBb`;&&@tG-kkt{|!Bm_EuphXZC=y5-H zDWBHe&R~*pnB`nR22COJfKp|p|*%u}%5^x-&Ro}%Z1 zflhkjQtHN-CDUb+6?)qwdk{34yF2p9;~c7)fM7QuV)Fy4iT$UUo(>5A|tRQSWeLHgXXXxSFP?Yxn_>HWz7|Z96<_r z-lovQG9EX+QMWj*JpEw@POOF@SCht2vjS@-L!*uo155=&z}awb+dEdx9bx;FpiseO zl5<5uoA6S`r*UiRhM1dWa8MBc;uKQsx5U}conh4)%o2u^CMly})rBPYrNS0xK9nlh zH^|3t3_Wv6#@xdAVv&<0Tlu=$E7Yg5BpsoQB$xD{PGUn+0KMTlzLF%;w*%fD0JmYm zwK3)%_Co_Vg>v7I$+#oo>e6DQBNT_!61!mqwH|hTi(xJB`w7W&8e&lzt4PVlrsGiE zngzK-LcY7eIsx1+;Adhv%`Ru)Z8ckmE~+jBp-EC*t0W1p{dpWym5Qha0oX1<{mv=c zI9bj7>YVHA!k>vgg=i3c3I0sXkn~Ww(q#DXhKi1Z(B~P90F|jo&dwLJR#t48n?Mi4 zbh+Swf&(ryoMs<58*2d$l(A&*G?8exclAacfIJ|cDIE$UB2^U}P?F?O`njpx;I2|7 z@82H_MBo|tWFXJ8)hRpX+)i`4fc0`fHE@49iL1||gG6rRyz(5KCRk4U4 zS`iI4oY4c%1Adg+(kj8L#se5l7&xyjcKvSX%#flTdS#5^fzUB(-lW5QUT@EvXrco| zwI{)7i1E51Cj;j@m3@NC`Aa%QMvXS~Rh2vn?7869Xj3qh_&6qEihef2vl@AdujLC< z=C(iS+fvSq_mE(4e*y&MFj_T12W z>LCb>%nI4GI$^b3_e#8M%R4q=kB2qEq~GmhVxT6MF)=Wun4b z&$-<+mJvj0*?@(&Q*1E|F#-_z3|s-tq%&ZVp&r(UKK*YA6g~@p;79B_2n=GI1dLkA zvj5LOp_%Im7)WSLhL>qpC}@Faf`+-Bii7_>NLIq0`~<`f0w71zgArqddYatZZ9+E+4n$3IQrH;u#hPak{qn zc!MZ&V$5*glwEZ2h-*3_PpiOrdBx<+fz1ae8IbzCidQ=?{fpk+d!Zl%{%&`W!(~bb zd7Z|~stY@vPCa^qi}>5E3oypo{iLHt{1H0Po$d)GGSwIzx1cL-?Pl87K-}m%;`Z?o zuspW5S+8YN6PWnaa7kS+!YSwzcWPZ~V+hbSTnx$>#YPmZh}%wL5wsn)hFRrmC~|tTPELJL5Z~x|zLyS63W${1czAk~UwdM+sA7 z>s|BoVmj&*+O17S?ER78s>k$V2%`?56I*ZHtRC2nC;*szgp&J-kG=CCN~hH>O`jx5 z0uqd3lF!<92Q=u(<+;&>9>^efAVM9AB}@*ZfV#Qt^x_OJK>5~$=W%RT$2k)tmtLku zdYn?w9j%3H(k-AK@eZ%C_?W)b&v0XM;L;ydoC9n<1>BNpc_^-8b`58J>H_#2@RK=` zd_q>^piCHIb{x#=hw<3YBd=RUX6BrI_mC@1C+jsT?dWRnuunh3lM>yjfpwM3+HVYQ zD3%E;Yy4O|n(OB%b=FH9v&~2)aBsF)IKXj@aS|*$mEfTZTida|2Zw5VC+Nu=IhIQB zRigH!Je#Bu=K||IZzo8E^Gelbe!uFblFB4;L_z^P?SUXRRKs@Y&6{lqVr3>Hh463+ zp!xar0{EQ?xHg)EIdt_n#EfJMb&m27ph&?Jw0)tMu4xqb&fixTL$(0da+@~Led&bc`4oQz}} z(Znjn8Yn4}4wP=-h|%Q=vgj&vCa+sz5WTI52YPjGrEr^9gJJ7^2k4o3wyOmUF>_|F zXkJ{LwnlWz32D)uOk8D<*ake7Zzd(M7Te=w@{B=OgKhTb3w|*+=KRFdV60DhHD?DK z!>Gz6u>0WWq@X+-XW^|O$ClXG*RvUNgTL9&jIwTETt*iGqPNt7E>QPNIoy`VKwVJ7 z^Z3QwAN#T&Q6JnvGu!I*V?Vsev}AWNI!hEl#$=flP*s%!*bh!@xhoYHx|Oo1rZF2< z(7B(CKGYR=(m<~s|CE(LF4pv1JUuwL7Or#^=bBB(vn;KWH20S+ zt$~R8!`gGS7VXK}wunDol#^Yq!v+M~JpydE!hrQ+(kpRL5GMV!a!>P3XV=x{1h{K} zzlD?MRWW+A)oc!siwZoZFgka>Mlstn%w6q+`rK?lhSa+)4p2TBt?F=rn4=U@W1_iX zF+_}!B*~B4v*SknD$nRvp0WrlUXrC}q?qI-FzxlBJ7Ed?7AZ@f~6m3lAl(M`-p9&;VQ~sev@eTCtj^9Z7sN?gJTtr=e(hO2lE-W;$wnzaW(5()`4Fl! zYPszx8P7XO25g-(~p^u1v zIB0C&BVx<~35HQB1dVqc^ayz2mW)IkT+@nV2|ZU70Cpvg(ToUqdIDAxT8jyUEYMO6Q_-bBL0*DQ zeji5%Y?2RTSRNKLq)pRP^QG-5-6urr5r$#hJRw04Mymvzt4AE-kQO#bCuuiU4VlOv ztIMf|d0K^9BkgdVX2c8Zz|SVK5K1nUGvkrYK!=&9%h=@X1o7O^VYs=O>foNwWlnn^ zIMJZkwaFBGRi*Qpf_oXHl>y)p29RE@&R*Z`X0VsQ)S*{Qz8^CCka*nm^IW=UB5-Mk zQDQw1ZCHumAG38*K|1B3GEZOfYrq9{KFK!vHYgvU zIMRUgvPuDvE6oAhG`^0Wb(4^caw%kG^Rr&LwT05FxAR_dY?=gz{EL^wdx?|w9EZ?J z^wM2={Hb!O+WBx3XlEa)PsK4Tc351ckU_Rn^irUFyIi{75z=dMd7Q4zO$-LRa--O^ z{q>+2;>@l6q^jIJpFtFH&Q{Ln%|U7bA5&w7VRR*io_GOQ24SVeXMR$>&`c^||25%I zrO2%MYu%hikiMMs2$f~LkMx$Cvx9g+S%vARduYOIgWreA&N?U4x3p^uf&rd{F`js9 zvaLr<;#{86NP21@tt>z6QOZXTxsiZl357IavG(EY>6#S)J5!hC9E%UbTrWk{X&Bp` z$tTF>VM!E|l0dsvf>2r@<;ZqFMj59J>u_T7sXenLZNR@0$3MN z!yxC9Erw5`H8b?;%fgnAIqnOD8Y8UKXK331jbb?QimL^Exr~Ew+3YF3=~u9Oiq3p^ zulgt3p1Cmvs>R^?)xRPa4^wG_iQWB5UdtfZadCZG zPT9@n^XfT$Dzxt8=Yy9MM@IrsmjNBZ7fDPjH3emB7iFQy@v2_54uJz^74uM%;Uj-h z&FcP)P=o4jfGoD}sOq*p>XePI2*TqmrW+`m*x8^9tnC`(;>n)TTN-G=QnFc1co)Fw z7a01`cHJ~i*#l(^AWUPfs(`h4>~l)QcL}j@4CIl5x(2epT-q>}VylA?Ub+YZoh1Rl;b`9rc(VLHGx z${sGX!(dKzdjL{HzL{B%tQyz~K}And!bqKEMM4gY{A7>_)5;RqN#^#@Jwx#&rOb8R zFU6-ZCfT9;O;f|g>8#(ZfeP=>u-3NA-PZjXjvJ?%&1%S;6X3ojIS`BNQ##ZLt21{1QtJ;3SZ-!X zW~*7+6kY^{9|d$zQ)iIoSc5|&^#}QGokh%@ENea& z+!8DAAZ;kA1u{A=W+c=M^j1)TpEB0;znHIlcQ*1rbCti@N4~pc>r9@2>#}41DezT1 zT2D&^a`qcuG{i5z?nSK|7J??47ul7%KVij}cFpj+Sr<-~v| z1I_JOUg^(GBR(3ABfTSu$0Vdt66Sg`4q8o|7*Ht^))PDC5WO9cLK>t>HnWN&l?_PT z&AEe&kS8=vp*PUwFkV1F1e<9*9kT$UL12)oop+r4H^id!X}A&Cjj09_at^xmnkJu|#<4VQ8adL;=iQoqQ4Vnx2Y zjlo@5m+RF>VVH5x!w#xZLjTDhg;i&J@UQz_0IWfG!+cP%78T)kjig{9P*0XZe-b#| z5c)NVl&FO~V2+XtuH$uDu01&G1PubU2==DSbD?%E zKxhI3fkzt8*m~ra8RJ~}h;u())?3r)*Yp4_bXU$Ye4V#7F^}Qev*;M{v-j*JDFx9x z9dsnXa*WNmb#kG*Qh*O)j0dk*w`1r2D3%HWwGO6! zt)ZMPU|t?08<-pcCGuRcIpG0W)vKhVRkX10`3)0>jl5~ZP&?bhp%)% zRKsnoU>Ap<)pfjCwGzQyRBji*VqEBQ^qxDP`;P$xIhZANcz-4CFjMby^SB5POg@V` zNY$dN2!T`qBdlB8_mO3G3PqD3!(B_Ucn0aU_P&(8@lv#<__+$0@uPoHbAM9t{&aTV zIhSH&(|m!b&+*y2T6>y^W9-ml6vAd0Cnt^gNMAC9EG}1ya^yposF$X?yV9`;jLJnXSahDFk+7bWtVS`1 za5RVlUEHGOf>r=g$n=~>Lz22PH`~#p8o=TU-t8BIF?br%@aB?2E@NsJZZ9&wQvhqO zG6Q}Bc^*Fv3~C1Ulx;u~2s@#J#It2_4JBn5;gjtrE6zpjt-Ec*6U5qA57!K`CN&RM z;)ku5+NG+T3&A5R#T+gHB#D78N0F^h@MP&o5pagB0cZMzo*{b-ZJk?thv@U*?so;A z9m}QiN9VV_-E|XpdBo`g8L9pVMZ4|i#SXpBN0!2)5{1Ak+Tt?XuH(bBLb|l;fES`O z9QaaDrxnDw&DD^*43NTyl&NjAlr+G+3B?%GdC~K;G~a~fQ7MO~vd-IM2zKpMnB-1` zX`VtT2(!TqI@`rUsmNV#>M=z-I$_JAgj?(%!l>G{|8zqeSqx)y4oq6#fbC=qd=Vh} zM&v_=c%g<~&CH_i4~Hl?WxQfdxJr!;aa)#byw1VsC?k57XAT6Hd+5#&7vE=3V7JBx z-k=XijH-{c67xv0U0l_6>DjVH!<~9`fFx%SU^5sQh0kY*7XnO6Y*ydQ&iCvmGCy=p zz|J1=yqMJnGD_W}%uL+WatL+^JKqUV=78UU48V4iaCfrRA7=nQ7}Ljdx7-!$fJ~t4 zpgiTjvNi|nK4GIvAL`p#x@)%k$lYD(Gjn6-De#z zAjOwuEsf+tagwOSq;p#FKxc)jHKG|(diEakkXL9y4waXB8I;W-M@-HQBQSZaU&jXI zl%IV)izbY2^2K^A96D#W6q)a!*f?)4Q8!u;+Mi|M-kGj<&kK~2TE#DhM&sjtzIEam zijwYB>gNqqbcn10O8$V!Fy|}Kz4ZptAuVsAD4wDfYNW{sYIn{k`~=RLArYQJ{KR!t z1p|dm`?93>7&Je2N#F_u(mg)35KjygC+KIKg>>sEl(*d)@8KpjC{<%rT9t8jQ%7az z#VKz|EME-;FYXE+7npfnvgD?KAnVThkr*Lstck#c@cqrP98_WMKFvbI!bGWrx94c! zE4r*nrw?R#^ZHYT*GlpLnJ;D(M)0(S9&Q)Ji_~o}AD{zI%Mw}^6?n;Y?sn2Gl7Bzz zeq9YEcZxgLeuC<_zTs?9(X~={0V^+^m(#M+3Y3lRydFkUiA~4~?6urWNwssVyB%?? zF1jeo#48uj>548MshS9HpvsfyDvF*W1VMxT3lN^jqJo-1dv5&<0K?OeiAkVxyh%`T1}e%+hh5xErf864;Pi-m3q*3+XOiYu)&4mOQ$EexlTRITbsm40p!!KrKmoDcCYk03Hyq+bNZ9b{xdT z**i24B*3aOqW9(Hvhp^&Vi8riO&fudl4U;GG1}U-5XO|~XILA8&VeRjPaRYU>{1Lu z*5YxytIJLu6eBkyb6QO5Bw23 zxrMvPb#&@9Cj;-snpdX|L@NQ)dYN2!tXL44#_+B?+kw;-JOd<>xm^3cGE=}{xgbH4 zvI@M@391|(ggngone$f!jsa!R6;LQ0eP2w~QS|1_bz8Nq5-2#9J!hOM0*IdA=O)So zA`oxo_M<7md%5;#1I~{F)`#3h|yzeSvNgy=4HQe-=1806BR}JhHTo63C zXD1G?*z~|c3?Kz@U~7;XHiG5>2vVxw{q51(MtEJO85jy?2Dpd8j|VH|nbew#qCBg0 zYO2%W6WuOBmYwMKfLALsvTO|rS|r; zb4}MFX~Ff|>aNZN@_~pm1p>r#nZd#G9V{ui9^JiM47tr|P$b4RL{gsSdD$#~gYg-} zgUp4ym)ba;(s6#!@xw!>u>ssKYsORJHj#GSyAa9YOR!Ar14x;GUF;J}l~(|V?@dtj zVDbfY1{t7Zj(bRNV~#OOkknW7;j;TQvF9Ts-RPQ7m`1k^Zc4fBNzKSHKmtwwS5(Bl zJHmXoPy2Nb_U_2Cwm9G3SN!Z~sYU0VR&7AF|C5gcU6lc1f?z{c%3Delmz&6kLEMkz z*l;f3GiV12wp|o53CFi77vuR>?*$Dq6(FD#$D{1H3Q*KA;FyP{_?8ZyryO;m!s75n z!3`k2W~LxZ2`ZWc;#32o(QoNV-?a;$_l;81-gUXAUAH-70c@zIp05`WN$L$5WKU00 zblOJN(=9GGeMW?9>=64R@}lftGuAjXvy2X>WZjPFq{D24BQ3pen|KWzy);1Zz&IfE zJVz%WNkCyC{GpKRbI?m9=nXk<2!q8>*p6S)O-rBaHa-D?2ChPnse=;1g!jFV~2sErN?d-1ONg#KX1OgtxF7w%m!Un zBKZ#Ltz`bRV5(1CZ(^uvduNFzI{{RCOC_qlJU8`x6ZHN$qb9vIDHHT2+Vr==$O;rB z5aF0ffTvXO@kX(oXOc>jv+;1bJMk3X?el}0^7Opjm~#u0(+I(5P~q|1M}D{f{J8V( z^V#V@aRjc|&3nk44tj?syc%V$G%;Vm!nneUoPYqyZU<9ZX436Mp|QHI=2f@@Cz?jATDk5TgNXal)uWEAfTt6zJ$;I}dVf{~ zB#x?aA!%#PMluEtNl-_@1lbi2xzAaM(2(OZcTcn^;Dcu@r5L`ZdCDDj{aW&%fa?Zo z+yp$c&5IJ!^R3N91^7}x!ss^%FsXn_5kjJFooAqBxS8}0h{d5hXfs9+Px^G)_~;}t zJC|Jqpcrme&#MjiW%FbO)rdRSBX!_k0CLkyu%rkS>DJ^3n6}XL2h9Q($sqsH5@+z3 zXkZcvh_kSht@N~^my%x_SRe^(hBsC+w=`$pRU->jW`ncVez?2rEjNT+Mz{Xa9k^uL zoQlp5ZK(TciLE*#2F?hyn}}>MaOB{C9@_z}AdkZk?h5RJ#RXm3-0E0-4j#px(ERXh zZl5%J;_&e$HsGmZD?ePCC@R23M0_&~5*o&MW9gPM zb4qyf<7=de4o4(iU`raLaG~c^q?lYz3Zk)Dl!i>+qF$u~6pQm5B)i}#!U@us!6o9o zg`vmtOeIhN*VohDj*i%^R?uG#9}7<8NW$ZMIpVKvR3f*6-ds5!V-^z5E4;>MTi?aZ zyB(1X%ntKoy-}9EryU$$pFMG(l$|EJ2tD_CKxmsXok5w6!Ha#&1j3|ZC!(Ia5?&F) z^J|EhyGK@2u5E^396g)WxE5VB5N%&yT14tgDwaTUjs~}FjD3_W_s3yRKqaJ=jA|wr zj_hQJLp|(pO=r@G%oIr;CW#5~O{6{5hH!4#`r4LL^kSfHfKVCp0L!@^XofilyFg5b z9by3kRGzSS>MqwO*ggaznZhPV-Ti#O41SPn*r|5z={dTqmpmjzIPGFguEoAR#6aB= z_h4XVu%eBlxO__&aF6Y{m&Vm-v;>Gz&XmnV2U|gE0-loa13zjqTnt99AY$a6oMVVQ zbBpzK02J~|v>=DN_ZB%nuMr-WFpDL3S9gMjh=oWXjbcYC|9JroIGAM(_~rkWvaVXe z)0j4E=ggb@EPJ*)f#@3f&ytfvGJ+LYFE5(~^;TV}$AT1I;%i=1|h%5Tucp-tj)_eXnZ<%YgE*JE(7_;R~4Oxh`+VxcDLo|Ic$ zQ6KcYP^T=$B2&!h)i}BN;yRwbG;qnOW=jop@8-tx6lOtc;mYkQv@KU zn|D}%0A#8(d-txhN-^5CIr10a8GD#DzNHk&y%MMSj2^9GXG6B)(xI#;I$|3^2V)a6sHqxHp`tde zIp2?hEDxM|c;i^Jf!)QlHdoAWat^n3DUFcDisNX&t@Xb6t+)W=CnRV3!b(1j+UJCB zLH*?IFiaN<)UiDj+kKY8(C&IllIg}`ha|Gv`8FSe3j5iCO`-CPb?R7LeJjTXu+Vk@ z$8sdUM9tof>Ivjf8e=qbvkyV++}lQ1tG;{=44@*b-Dc9|iR2)78CNPI=2#BVtGwXp za?bKSIzT4irA5k{VZO8P$rZO*!v^KW@>T#V4CpUS)D&3nk{FmRfo)0-zadW&EOXNp zXAJuTCyE7Jo_#N$v2JeM$GlV5xkk#npUmp{2-dLslbJoM0zIKFtPa9kCM_X`q1(-J zMChzfIZM=)&G49IW|;V?9 zmx^jWmP6ho|#W-ip+9Cs3 z>y2L_4LRF3*U4IQL%Ck(OIH$8w;@5&d`R4lUwjgi;(Z^OFfvWoaH6%j9y1P_A88sj zfEi}`09_^}2ya)>0gHn#dvW~p03$~y4%HifJY4|c)9r|*Xh>`oK+ZB}?Hy19z|Ch8 zlnH~AS+4GH1b5h2kpC?BWgwPDs}|N!b2;laF}?>lah;|?o7QY$FdnW63K0?c3d8hT z1ZM!GGpmbO>0ZU!bGmQg<1VqGCS-dMKjl+;x)aSsCOi*t2iH@&f<-r_l@z!cS7Siv zTrQ9&rm7kVMcrmSBl21n!jJ?u=)E#ac6(i)y9$^>#seAd0}Li&1-y!&jefqe*rl`N zKCc?b!)|>11a)ltxu8-=1Jf^=!=%g#1r|XAc&wA$H3Ta^3b;a*dMW0o02Uikn!yDo zs04TxGZ=+`L#K8X*khFZ%87z@*b1x;U z9mz_;N}~|bg0X1rjLT+1)Nm^VG55hf)a5pS@*6;n3pXD2xB!$$hZ%&7Y;X$;z)X0< zqb`fJZbuNJgJnH}z7_;)$iXuNEXJg0MXqTXaC!I3g8=3jR1H4Ohyj-9OdJo$^$J)T zCNRA#+HWgESrkT657?(l&36~LzhZGgQX3qIGwN17Si^H45&S{02!&mbc6I&+vad#b z3kPq~Wu0-bl{uaiU(kX^zz=W|cmWDvi><@Y+`#I68E1OvairkGir=50)Lw^KJ2C7Y z1y{H-<*3n7zzUMmAU7gG&{hEp&-kE{HJ{q~Skf}%A|V1zHROj%b{pjxc%wTTUP0`;z7(V{K77T1}_M0#Mcr*>7JBrJCV zEOX9-CU}(~5f6Ga>~h1czXN%kdR=l z_U2tFuVW&#J~YR?Sit!RU~hXgc(2dc>5k4asjhH2Z)D0Z z;15ak<*XB;No3uC)w>Q>%-J|DvK?ZBX zSR=$h22vJbe7!tTrYj}zRhy=2hDASg;Ahy&i_^-_9UHRMTz8-}19X1%>K+i-{J zn9sW~Kix&FXU>O8N^8-?*Z9WmP<^40WqP`$w|Y}om+@B3w4RHIdaK2-h^Xj(p%W19 zku`=@@}dwuxd z{`SA~FTU^<|AD4Gw-4U+U9WllpIf(gyqWsDzvFAa=j;A@mcISXAM@S+;eCEw`rP;a z&9Cpjyz+~`Aph6e{RiIqx{vxZFTCHE{-@XH{{9dD@ppX3D?Z{M{OWhV=SP3)Z$9wG z_x|Ls$AmY2&?mn3J^$0|SGCXm1pjT$3vd73@9|-;{EgRJzVZeBZ@lqMKmNMok-zqF zult|Izxp$;d)w!zUw8WVZ==5NPs=Yg<+UIBBm7G@g1_K$U-AM!z4NPX0{5Pu_!r*my}$ame@ghj{pv^l^1u1TSN+q!k$?X0f9t=0 zmG;#n`3Jx6eZJ%0z5Vg8|J}v?puE9+dk}@WOT`OALhoBkvEZ+`x#Ug3Ps>%aGhQ}SDX z`7`N%`@`u+|K$47e=Yf;FV22Vhkvv^eeu`+OX1hP`&Z!ME%`^h$uZvN8{hIRfA#^+t8Q;pKlw9% z=p#PvpH9MC|NOuG>38BE^{$`#o;UpYH+&5Ig9-i(lq&qQe|3M$Uq8R%3%{0p|1bI% z1plsY{lEU@&%FJ8e)jkN=@0+CAN9Zdz(07q_{x9sfBw+hzu_-_i}!xk>#y(iBOmiO zKkrZd#P9p)-}w#S_vOF%58v{ZuYb+;FH5icU0<`m{Y~)y@a8`Nz0XH|@n^7q@@Id~ z^m!lh)?+SS_m%qRe!{1`-mlK8sYp=9~Y}AO1gnsFfT{`@ZV8dchz6OW*Th zZ+`Jle#Q6yj~|?U?2j*>_<7&++V}X6|KgS35q-n={|E-UI^|>GXj#q!>zyGTLo4B_Oifh}#026|Q;1Jy1 z-Q5~@cL)x_HMj;37BpyR9D+k|3+@oy-Q6ALaNm9J=1xu3OwIfl>Qq;Ev(NUk*0;X3 z*6z^6AD376&I@a>YEFHfRSKCB{W|Kehl`(?*SwmsjnPFU%~MQ89S&#rjY zpBt_&bTLU=E7mS&=92+haV`Cc;HxAB&LQfF*58u zu}=afAE)02m1#DC{U0eZM=FF;!k<_T zJm^9Y!jKzu+nz{AB&Ol?-u<12WGEqj z5kq?Z<+r(_KzZz60?Z-_kUXyLzj6MQTLj48L#G0z%tHSswSYHs8P($d$TG6; zFu-Gd`Qa-z$)C5o0`p^)L|O8m7XBkBcV7Dwp_IT>uz>!NdM^(djz8BcpNE@MQ7dT6 z(0>VBBUXPdhc#fuIYyX_U)KM>*C@~HaElVsT;YGdBJ!UB#469v5vKM3-%4snuzD?S zGcXXGc2(Uoo*d49qSwwV+#y3|)Yr`!O2dH~*cLAZYh^oGMCHErpKggbSV_fUa(wrByt9}rLSO4zU)F(7X;IYUcRL|M#N+a-Fz)HTP<$$Uem|z&#)!Q$ z?#TdlP6xjq(OxN}r2xqiF0TfzvA<&>EuoD=-RT*?y*WLF>Z&Rok}-L>y@HBnP!L|T zM)(lZXlz1PVfP;S=IRi)%jv!SqXvbb_uc2C#l~#u1e%?hSYJDobeQm+(64Y)c7f{z zCM;aj7Qxbr58?9zE&*d=u=)We`gwTCa(3Ci~77&{9mStPxTSK?QcPa(r;*^m8CSqM9zz8q471yyg@do_glZ zQ}j%wN;c zHNg5)D{jdCx0`y6t63rmk0Ty6Q2aPEY)8-Y6OmnpS8ob;YBx%g_cb(>%y#>2wni+A zm~`a5#Ltc%1;W@`H$=RJ)ktftN{K1m`0M=8D9275gp;dExUcy%SxT8F`c2m-u+_UZ zk(Aq?LTY8cG}|sXYb?}1BS)C+P9>#!MP9!-9TyvV=$`4j3yCVzZ^oh5uD%H&5b_?2 z@KP}vPUD_T3cR_!HEa)ndY;HXfoGsht+pJ8R>4w8r8xN-PWY;9ZCB3{n&O>SsPKVfA=Hl{3>t*m-`l^EF6aT8Yqm^ilP5r z1th_36bT*L9~s1G(Ar*u`$;L!jX2UPJPdQWlSsoR8RC9iz!>z@V@a|Kp4`?syavPSUTl>h&rb;k<5HO zZf%|=GAo_TyZjpW=Z}tQI|HTWuh`-U_%uHkwlbJ2zc$Yc)S5XcH06FFJt%&& zKpCU{n%|aV$>Fy549`k2B=T_c429zUxW#TZoE|(h&q?A^G=at3`_4@5W6^TiY~4FJ ze%;fj4o|ky<@b-Ch>&ZY!t3gnA1ucRN{{6G%Yc@QE>2F1qK_l3u#DJW*`6nU-)>{D zFaM;>Y`xr!u3|pFI+V&;V31ENjzAFk85V8r0E%#CF1iP3OQ~K2{R_5jpel4>HIs45 zH$WeK(NHzOPawIQs2@n)Sc$rXJ6x?a;rFsz?5f&kta$FFuC`EoV;YhA>*JX&`q~$V zzz??S!jIORJ@_)yhiw2XnY6mfhAD7bXVhy9^2t6tz1elrIJ z#b6jZM5o*6j#*O_-#&O6Xqdcrtq`0ycRqqJLaw4)hN=uL54U0v-R9E630{Q6hs z=H`rk=SU@UIgqfJ_CtR9^yx(< z_P>8mul-&?K%ifJ_jm3jW<(nACk9BUK4)GoE~>^z9QJ_hY*GLfqu6?W?W-v-_sh-A zt)w+Rn5lU41_n{}sm!2Ne-W1qk&==Uz?#tfo>!nu0q=>+2n_mPgWp(Bj~01NjW2e_ zLJ0+1f?2x;o}pg$qhhoziueAoT=nr1Gyt!JbRrPl?hatJKiT^Ctslm+1ikh^Jb2ew zp`ujjX8FEE@zuFmT~TFZm`YE!Wj02+#hi55PL%{h$CxR`&%?QZ%Yj;wY^B~REwQ(r z`23%<9wpZNrhR0clHqT|osQS_Etazpx}#Ix#^Ke+Guciqpn5l4iXx5=N~W-3H+(t> z`|+&`{Cpx%=W&_l zwu|-~STuFGK&OfzZE(Z`Zs=_BNM#7E&qZF|H3oVppVJT{%}e*DI!?)7I_qW$H zj0D4S6^>3DuV6F(Q_aBH@p4ZMJu~#i>-Ma$Y~^UKhPL|)Hd+PjcsZuR_9*fUV-g^K zqcZpK7fH0c;q`|d)~8&We9qsA$wsW6u1iRRG>WZ*w<75khkrm?|F9nWp(NAf@%S!N z&{t%R%`zeMr}Zo%?$6SmC>KwWk`e-b2eRQb-bm~?yqI}1B>CGkQ2RsBsJT7Ti@B+q zx=Zskxv%kINkrmc-b*peqA!ZmZUWW6W`tmPKrz>dJ(<1WOX*=_w!iy25lXYiqjcIC zHwsylShb+nMo8`PlF@%n%*BoNtRi=)G#)oZ%kiA?Uq6DDA;p|;w8RDylrz;l9GV5M72UQ0bmVZ1yV!CJ4t$;;@(^wmuO%KK8EJbZG+P)~wovAy2)qs@x zIYfZhkkzRLb*uzuQe1Dq$IW*fP70MWAl|)>L9eW{xzp|b#>`|OP04#TSZP%|*_;zX zZm3vnR@KSaFg|P2vIE#5A|={2OoqQ8H}fa>q6q~Fl&kKoxI^BWFfuZ_EQX?xPOt|; zBZ!2dlH1>3*_aF@9&UTEv;1muKaXM1;~@tFG7J$dE@s)4rl-+#C`B#I3FVMczp1S! znov!htLdt#xml^j%Y9Mi5mi1U^F*TFwa){ltsC8^ z`=R_r+-p*W(kAWXnv>M;p@@RT7WFJRVi$`r(E%Jwn}ws0aV2*}@Kq+6a>Y6;>o#_3 zN5td=GE=q^XC_kb1KJnqV@a(j22)yf!!#@nT75cvwWMiL_@}+u@fJ0ngZBfrY2FW} zvhKLt&fN|N+N@M~6jv@*n7)~b+zVVq!~b>AYB?=T{xfMPz2IP>&YQ5lpR@aMe4e>~ z7zKMgU(G=>PY>jL`&Epo!<4$zy##4h;k)1Sq*Q(%=~%YjI92M$@Et1A;_={IFOhy0 z91iP(hxb`@2Am8=HySA3*GGD45X5r*yexJg>-O-6BQ|p^^TE&hfenR&6>s(X6wTRK zx_7H6nLEY^u)lmUuAHB{_cfU~!9ICojxn??<%-{KL*pTW!Q z`n3kNSM&!AST?M3XW1&ILATrO>Hzy}d#DUly*<^@`|WXiyd~Zr9%oh|%p>zWf0bpP zo=TK{6HnRB`}7QIFmt`MRPfX3kN!M^I!A*4P%a~!SHS*WnY*;2(k*}cbbsvYGOwN7 z(wg!;^obU1(M;4Jzl|JA?a~F87p;C87Mhr{B3@Jc8F`G9l6!6D8=U4Yn^c09HWZTS)JP@g)~NAS>=?! z9LVbzBy0@Rs_%^F&S#pTWUeO4qg#H&s@8a8NX#*#V%0@gi}^vk#~+6dX4fog(*W}# zmaJj5^v2)_UjnfEPfq$+f6Q-%@o`sy)r$q(ylg*>cKFXEV3Fezs72;GfzW3|XCq6; zxpN+}w5rT@78|ceuGJ_ht>>y?c|RRJ$bn!4(@7OU*E?gPrj@x2+Mt5!fh0y+2U@Mw za?k~!s?^+rCfH@fwd9{;GIMi8(L9y)N!Fe&_ooZ=8h;jjLSz7wqb*fis;E`b%kAal z#)IvaTM7W{O)*&8D;6tfikAmRVLKGhG<>J`^!~O-zyxZLS8XyN_av3cP9Lo#wN@+@ z!{&u1GT|5oSJsnA>Nr~j4BIq<)h~JWw$t~oi)1T9EoMLLFuB-{c8gHgQ>m0-S@*~) zZQ{4G4zoybMs#Yq&o=6vGl}A{A%YIg=$BPlaM8mvNpN zig9YC%^I<{21*e!F?EV(OiMSZ}sfHB;j^s`BjXzagMt8@tvurfreq7D?{R#?i z>#`x>OKe=Ii4Rub2eV?9f~Ci8 zu|{jy=h#^_usWCGnm1*d6Ara5>oO@av^Cs9qIPn}iXXk+$pp5a3ntvcT~boz zJ5QeGrRY7zC6n8Otk=T@_INkE&|H97IpHGVV}s_0k@oe1Y1>x}M+W%SQq0m@sen}k zX$6WD*T{3cI+)e`X^pL3s$0l`B8|}?6-NQEBYvpyiw!O+H6yJA4wzJ6Jv}`&n<|sQ z`^)_h0J|2Pg+*X87O_MCTjLw6xniy~dV>3KENMt3qn1gZvz3to2O|cpx~w5aY$mgD zH!_KK<$UF;Jy4_D^ArC&47viAQJ@0#%bEFFJB>AiHQ6+-ilTJ0ac>B~=uXb|B?jjY zSE^7t_U5&3^r?LB@yo;|hgZP{f6U!0dT-Qo*KF_nt%L(ny6>}B8r!04V!twsAf?Ar z0JpufOWSIC&xhnY-^*kP1w17P&T>$W{vmV`;vHZQ?JTuc&3&MftL%`k!f2z?^B@Ob z+ntLVZQlr9#Xp1vvR0RJdC*boas%+1@ zbu_H%kkQ)|?NT>0w)jDvFi(0O7)zQ`Y zX@OhyutqFeN!XPl@}Pu+O58F#G)lQC1?R;Yn$_rg$Pn~>+UB9tLe;6YRlEH4bu$4W zgoH!v{c}nf262PUX`-w6eRuMus`Q3$=bX zF_95(W-^kYXF8G#H39-iruQy;6aF_eMp`Vtm7gE?CTSD0t}ZU%!*z^e>DW_`&;aZ6 z7v3Wr3cc-sxbUrUJPZ8n;K|0Uu}OcjmW`^%(^`E4cJJ{6fq+n3(kqjZyYo<5xsvfk7YZ})x2sKC!qm*qWQ z`p3CRa0=6EA5JO+D@}*LKRxBh^T?;w0gF>j^L%slhzmkCr#2)|~?b3LY8?{l;b zYmHKaIaP;5z^7=WX9RD3=0mgY`}CldPJ}vpJ$}+nH+|}iC};8LL;tr5l?GY0yQI8@ zc^$rEFBCp31GYEL;^>Hp{0vpZCQ%56Qfe_VtSedgEaEO#ZZCd;r{&`?!W z>?AC}HMtn!?0n7uIP9ne&0Br5DNyMIU_EHs|q%+?&3ADACI~4gxHFe z2rivCNA(V|zw}!P0Cbt90C<`z!43zQng;a343Cb;ARhM9LDjtp=TU6@7Xm&A){r{{ zgJ`cDHr!${1@Gkrsl*dtyj*f(nI%#1u{{oO=Ueur%k6d8Sk|>Qbb$rs~f$_Acv9?6TlgO+y%KIDR|r9d<%i~GxWu8gH?;%%)xae zOc4h!nMuBJTcph-hxVM@7U%ioX1|Vtb6t2E&A)mg4EYyb3QLB8>UQi4e{5hN((vxw z$j})+4AMEm&#O4wD=X4VhfJsQts+R`i%zKiwD-~4MMeabUOrV7Fu4HHO9tod6@nB# zO%Ay zn2fvO;c?jB0ti0b<#g9m=u9qFT>QZ^VuuG91XaDkox9ytdvh^T>^K|kZ>;>h+_xR8Fr6B9Y^Avee9TOj|AE z&BH1pY99S-EEAZu&ijyI`<>CZl6hx4W7z;0;FznmBP>}i=)oRic`z+e^bf+aJw)lw14ux<;q2(EkZ_8$NsI~@y(7e z&Zlb*MFL8B6+EpWj^fPQ23se|j9dcLJPxrG)So;}b2}-Gjl87>txuwTBg7!6Lz9jz3yF(&DGz^$>KA>Sp&$$0}6?o(K|E9Ma-h9FT zjRp9pn|}Xad;b4?foV6>a7OWL)r=;x8_o4ZZ8 zwdd<)!RtW8mw~snk_n+aGVVHAzf?KuC!`dos#&r7N}un#XGPz};bKh`!Q=Vyx1Zn0 zOp`759{a~IP*~jOxz+I=B&MMZhz!b7QkBF5U(SBozr4SRFP`UIw1INqlM$i65K20; z3dTaiY6MA_j1_zDbeaBllO^X99#5p*FD%L+s|1Ef!0Xq^0d%**MjRUnc8A)3BvUZU zrGv&|w3>aY)L!DZN#B_HsKpPR`nNqzSJ|>bMdhhAuYgCtyDC50(25_qrx}UF?zIYh zO~#jppl~Y3(kPV**DeyTyOY-#hYjFuKc!GKO8Mln7twfh4T+J}W$VzpBtY#Uy$h-z{9b@sRE2_RMOt!{Hf} zDsyqf>y94MW+m~2up?d*I#+A`NMLTVuZd{cENAuWeU`S#D543A+*E&Pqh(v?br2lq`NA!u0L44r&h-qr_>_W$bgXM@;F*5l4hV7N{iN+E;G8C za41;LLC+h~!`r%{R>u0A+_jhMM5=>`#%8q$OK^^FzxBBmb*z1W8)G;X@wnrj$$kTb zceN1qEv{!}t@*kfpZ_f_pDt7P@M~_zt@fQP{;^US!s_EWB=&)AiCxEpvEP+&KW%k` zMDJZc%(tI&IodPwXiX@sHCcCkEKuTLESh4{fN*bbvSgBtlZSOc8Jl(7>W)$R$>qB7 zDzRZm&A7$=@!jS->};EDfdai&>Mh>6^~FX`&6d+wX=ZZDEn($HbLLq6@nzs`+Qd~r z26Z|EyO!A-dRGvTLL|t5p?VSiYN5`II{5PNRVFT@n!0vph2_}S0@Y^3lnJ_^M@qXk zE+-E17_pnk)V0v#<<@Lp4LCaeMi}w|)lP)v0ZE%=jC^@K*vm^uz`1b~8(3hugvFR4 z7`T2BR_8SaL(561jJ;qzD>quvBeZG2IRGu0L6Up-T;5$~xt8-fZ?r zCh#CI7;r_1vr9ivfP12MBC5=z48sE(QBO#!;gT_e7VEOy?2kDxN@q%~rSm*CjWp=A zS`{93>pk!j43`_7Sc9)Dqv1X+QwRHR;%P8uN+eBP+CN?4su9xvRHxVaUO7wx#w=M1 za!e>%KTMsb%9uyyZDv@F*_N;49_nmotxd3Trz%t<7K}>s^{W|JEO+zIzT>uCsMD=^ z(JIL60MhBXp*DF1d};wVSL+pzqfk(iofT3U5KMh>3Xcy~xPqpo z{o2tuGw8VLjOxD{3&mjID=NEuVNrwc8sg0dQ@N!~!1`GNn za^OW$OpNSq^!>HcgAJpWi@dLiYNzNvbv)FMm*4lAUZ%u{Y_Q7Tq=x!DoVAF_}Y6@MBT|ZEJ21HGEkh{C?>HUw+>iA&3Yu}=~g;vjfP89lY&YVeF z!s{?R7&qmpz1(#M#Yy#rKQ78__Wi43Dlx-+@0!SU%l{fQEua~D!YxgZflsX&I4Fss z4BFr?G+HMuD-y^$hrjR)Dwq7__Uh)@i~VVl z1PXDCT46pZ)qsYVLDds-=EGm{84Uz9L|g(UUmb9Hp+sN~`s0b2^LD6x(`~jUHk*s= zE-DEzRHs17EjR2&xXKL$bo<`j5WCvwL~oVw263g6)&Y#L z3qRuVa*!!5-(<2ui&y()agyp{O?;pt==}Ia6>+?POu4E)O-Z3>*v}du9oN0E-?x!W zUW4ze_kUvo!&S@0P)0lOwkV^KO5SaP@B5nn*);BBflBK)n)A1ZE+~axnU~gf0|B!2 zyNs37!?gj#k0n=-X=#~Wz29sp?aAJxOpT=i$Agwa8ur#r{n-XmN;~80*Hj6D{Ls-U zp=KpKMan3%7dKXN)e|1uLw4ayZ%P1Jt)muI;V?q|Lz~9{UxQL><$3Kn#AlH%hTIA< zAvZC_p4-2Js3r^bx;TkJ(G>T!0fZvSC!8)3X(f|~tSMKE-<9b!NVh=fT(^P)yb9=E z(6fjDInid}S%qsPJ<9axd_r@LSFD=g*`LK3H8ki&8n5QCkfDxBK&+yS)t_O2GTWzS zS}fV2o$c^HOMU)$?L)fqd;klGjmVy8f&xcG?%ZFdr{W1p2#1#=nF3jA9W+G*+9{?z z`XA%KX?(;1vku*$ntBC}4bvgR0P9&N!?S}R!>H%|CVH)%0+xZ_;R!VP@_m~1w?rBD zf^a;P+8kF>-I@;x>b1g73Eqj0QQ7+UYI#~I6H<+IOP{NES-*SE2XmEQo0m9;*gjU6 zKE6|1rpv{^K%go7jf~Rb7-?M^PXX54D`hnu#wiM2T62eXKTD;lI6UH0xs0vb^asg{ z$Hqz9>bJi0ltA*<&G_;SSbvNccn}oV9|ORHC;652kn1|%V~#H@Sfpb8s>1jia1uw$ z%_Z{3lBvTxWn}~1ot~d;NmRD1iw;CM<{p!XUNdA~3!lY9_ondY(?DS_m7zIq)B6oK zL(`@jMm`sBGaB5`rYbkmrCy3y9WS*g<%gN{C8CAJk*Otxh9-^+Y-}K-++s&oecrj- zS@p)k%2A&lIQ2VYJaI zGyNr)j2dzS@(cBPNVaILkK6fH&72pNDM7mo2bd*M2!(&9pMxeO%jPdcr32^)58g}o zG+^{Ju#)(f?cTf-T9ec&IN!BP?t0_O6jc$ifBl@Wr<6<%-DZ}+bwGTH$FzF^);8VP zjDG-Yp&USFO5hM9{DW8lV2uj^YwuKt?mvMwvL%Et3D56Di@)MPUx3+P*F~sM8lj^& z8txyJ0vJ%J4yZQ-7oCyp4+Kf%p9yrK9`%1Wzq}C=1xAAdvA+2t&HlMihS>l4@bV8F zV40_ou^9ghStSDcS^8%K0NjiLjE~?pH|~G+M-k-ouMx)h1w*dQSnqZ->Wm@d{HxAU zI|ykIC;hBKD>akHpO_f+Ev3H;CB6^}zPx7Bmwy=&pI?cwO~3wEix!dqa2Ng11gsIT zEus96ms^Sr*SjNd$vb?XpYR1+o`srjPQjf5kuRo`kTqb<z<8MIG-JuVwmF=Rzs z`OPHaYlQ_M7XhsN2B17NL7fh;s7&364y5OHAr`9z-4>mr z5Kp%A&B@an9*#rEeADxlPa{YXX?!2PQc_OQ>A`hW2bce>5=p7H94&M==c~Ja#lANw zG~J@N!ZVLvwP%gApu$JhS#f}-K;$Ld^#*Gg{d_QK?;q$_u-0=;;!nw-qsoUwgaoVG1vo@E^*KxpsQ47rE&qg%%B;TW*d2IpKTGAALjbAQt$;9uL+)VzEor^n#9_UuwB}T zKw;%{2tQJ`#C`^rDK9Fw%2QCJO8VO%j0mbpfkg>9cqiQQYa7j1H|*Q zOT|iEsY$#0ZhWq$VwZcPp>?>P4!Jm$oMDH309fLYdVB*WP3H0uq#S71JN!XAM{tEtN2BOQH zWz)IuHtzkPWSD*_4pBpTdZ8@cYI_1l7z#K#5`{ZE+%8cc5x@H^3yH+CMVQZDR!Vgmo zE@ZY|P-K^XmVcb|4UE|7$k5l^sD67te4zI8c%kmi`MXQ|lZV$obXo{xFzM6;2i`#a zCC>hogWummhVRgaaFR-t;sNa^G3tXwp2CH{?X2KJk7V#ga9q+m?4EWBhVO)26B>OD zEWuHz?0QoIU1VK$oUm`9TT_ABAXmiRm7e-ZWH8*MN9;R{9Z7%Syi+|f%m)A%>hL_mz6bo@7M2Yhvy zagnV3wuc|O~hy~B0g#GFsYfwPN|DhWRCxfBHC}3Y84paW-Emtm6WfoG(rMh)2!oez|7)7Nyx` zQQhrO5F#2OVuIbE)vN2Wcv`7kQn+r&DL-^;83izM^k10m{ay1DvhS^s5UgYVBzFK< zrAjY`&)L%a=y&+zP_<9w1CZinbg{A|T2AmG@{Rd*G0%3};Bi^L|;zk{X!JApaKDR?IWUzk$xmTx$GADgsIckx5c48cQ@22DeNk#CqE8Lohj%o{w)8-eqrDIHrT}7oIWiKz8m|A2Fz$s_vPEU`S(xrOUTTO zC3>|#XBvEP`_*ZRIb?##!I}K-!NWt1RK#?0^6#W#sEG4;72q7?FD4wWPVBD~r+d45 zEr+kD^s?B<(0umq=w)52$EV3V>j05{k@kvp%%kNMmH?n^Bq5Zii0QS$O;*K810Kfn z6=>q{VkxxjZ+|B;X0{Sp4gdIvmYFBVft|-D8P+L%TpcQOuGApqh_+IAt%&*$d$z1{;{?B(WkOqJx+8w{`ro`zkm#x44mzV{I8-w9JaX7 zu<{+T1YO0S(cge(eMtAilaNMv{3PEL@JEY&kO6=Z<;46a26{mVK;jH!_rRYW29$b= z)qi{cpJA{8WLF6>lm5S%wfAGXZ~j0t0Qw>ZunL0z>C5K%=lUMHh4bfUqSpwppLIaT zTplp<(TiiFe1Aiew-Ie={%Y0G`NxBAw5!BKU#k9I21$d-EM8Sq{|miEoaFBI{+LRS z%Bl|joaCAfF;@eYbMYmzS{z5Rm5&(iZJ>1vmjKoAz z*60+6xB~&8C(|so0>Ct#kU*u(hKUyoJ?I1sCp>j9PK#4ukKKjx^rHY4^rAg(^x(3; zA*lnQl&KE?280w2;)04tAoV~SS_dI?@|L-7Grs=0=`e5rx1$Uz=p^k-zSP+Woi)4|IkCu!u#Z#%grp0Vr>st?r;Y2Nad-An~!I+RzahrqrPE3#|I%>hlFN4u@6X0Oqh!r<uRGFg4?2@m@n+4DVLmeG&zzX5L|L8gm*0v+5{!bJmJA|hDifHs{PvM zbJnY{xJ@y`drDBU=_2g(cNU0;RS7_!#M9_tv+r^IBAK3bdp@GqO|tEVEPh?*yb2$z zJ-gce$cr_X&)+n>BpMzM{lYH4Fmzb+vojU$2piB|t4F-#btNGeLakZGr<*db^5pPx z51o{c7Vsgq_zec#axg@m*s~i_{QcAfcf^OW@2JCRC&A6xF#0VgJvq#Q6()}Y zz=FUI?ZvAdzt;0a>Kg0Fu>ZyimGAXfF=#_zCQlF;u4VdGg%`pA+CmMudn4 zr2-hTPU-ZQAWDq(=2uf1qmzO#$#^bU+vSTOfbNvuF=ZP5@w8ay3KjV=zHP0>Q!xj~ zK=06RvMp;+3g2P)_*FVgVtdwA(li-^v}d-M*z?n!RtOM$L9VmlJHvIU{alJ68N)=3 z!)6&dTr#xkw4U2fB7+9R8#4hBk$T#x>vxj_ZVx-JUETH6wa!-PPoNJg%xe3!?R2YD zn`a^C_Qy(N!}F(wXsze#-X#DiCKdPkXyFHBL#Qui2sM|hiD07L!!0?+SAXA|6^6VK z9aIV!lvWoS^j~2E3MU2ukVQv|OCLUWsHUz0TruKLR^0^tzL-XyIpgct0cnsq+i*Z% z6{e%%9xR|I_`g}uf4)6B#C<=Iz_as2gIg(<#!+DrJ%|2|YK%_H(lXGqM(k@&-v?7x z{z1>Pbe(C@c&CS50xVsSMuDyh|AmZ|cHxcyq8#L@IjBj z!g#qFv8IH%aHR0G$YF@-_V%d^rl68Oy8$jx)5#nFrvp_4m63;G#O}()PTnJdE zy*a=+2q?|lJ8C9FwqUMJj_~NAd-?1a`p6mV+MKs{8FZevK3JG7OycnnUoUL;iUBds zVQq(aqG0?jzzMcriDCvM%TR!*43GEqp`kKQ8A294Ro-w^bE}*HW8N zO|zQ=@seRX1c$imK>QMW=$RkUFMW`6vpBt)S&Ez>py~H}_^e2!sZm(VtcwIL;}jYp zbXN8$OZohTX+$q810UUVTUN3KEaN)GEv`30e6B@9cLW#T1V&g;kHn!0Sf=YticO|m zsu~F$<+pOwm&x2_#`{RFtxjVk@2v1^X2Q|$P2ZZCzvMA{oDt9K81~O zG<{rddl-ZLTR{MKjK$=I_cfnMaTX~F^xG3cT$)_aoSUD-{-a!Kl~5(xAoBR%rE9Br95p%QE!~eO~E@iI!Rvv6e%h8;qHd$je*; z;24*~3ZaU{ooTWi-vx(5gQV7{#z>e%`{OMHYY$+l89vdd(#Nu@A}XgCvDbOVECZ&D z$nCFh384gIt#LH5u!9>-K9{H?8T?A@+W;-{Q;~hc`c4=aoZyxgu9A=2GoqLyPCU%h3){{uBuI?NoF(B zp$a(`jQk=f0hOECa?5Li#V#PyDv@fe(c+SU+`6N|knnOYdT2kVr(&;FWQInW_5z&~JFPZP%)li;)$!gDo$k zlqVFd$c<$_9FAjpUQ~%zSeqfx_YRmKT9jt@MMvx9CTO@s-~eHoGbUnO+#puKvUqG) z0_`uSNl+M33&%wU0%=Sg8Th!&R%e5ye$C!>~w$r&~4H_~KpsNG`+%W$pNx9zX{eg2e84p0H zyiPao0!S0qENi^j=sf}NyHLYy%51g!qZmiHLIeyHs$bkM@jl*bZ=>{2;4Xvdu~F+! zYwh$^7G-n8m=uz!V&~8qYNmHtO;jn3C5xAH68hf_$RFh%oqu4h%*2sd!UHu;|CoQ9 z5V0+kq#rou8B@d4o-N-M@UcqfLon#zvjk|nUkAAvMk@=Tgg^E)vQeYYR{8|-?27g) z`=F0u4reI)gmei@v*xsn&Kg$IxEIcUuSXrOBvuA_WXcbzLF=W&_YG>*X-|qogeJ~x z`uqRy$)=U2Cf^?gCpKV8Q>RaMV$*yh&b6uZ1?eYITqEOpQhLGO;BE19tL?E_Sg1qv zso;D_n$890?(t|@Z5t$Wt84MiVj!w{fE^QC2HICI$_|yV(qt%$gG*e^^WsDR9$uN$;cL#G1=Vd5Nl-gs2wbFS z03kj)eWv>HuXJ9AC*~-=OPtFKBev($)J+?EhHc420(K_Y@1mzuAm>du?+%6Cj8D2rE80nFa0f6nk zHa|i5-%k4&A=ww?Ic^^F1o%aFUQ&9Rlg*%|0Fo*2#pU(A4_pkG zL3c8MF!J5bhIhc!5!P8@&FK`G@r>WDFQeIHblfe|7F}ekS|&8R!NSKB_mN+o*ZOJH zwY0o4pbSCYC$?<@N$Pxjr;NUXxy#Pw^WgRc00X`9Cydu$IkVcjr+vQ?!4Rnxsk+^q z5TlYyzugsvO*F(L~&afrE}oT)H=Zn$DY9eX;?Ajx#tZN=Gy0)`$-YTtFt^_xs4 zUkxSlc-fEAoOBz@a~}an?T14iVal8WRr|q>q<&p`W%eB?&Lbs6aM$743J#s>qs~$J zd*1yJI|(@jVp_`8@w6J83oDH+Eep{-*+gD^0S!XHL|~fv3&gZBLV`s=_n8C{Zm$Vr zXi}lvJuC>#zW{km zq3NDPI1L0Cz-%GR-dHw?PMyve7lL=N7stzlKo${@*6MTNAxJX7ec=O({NGK21{N3#_V z>w_#Haq!X1=@V~%bVUa7rIRL>AOwp7EWitMK^)_Uh#LA1Wylc`Kg~kVRVso+JE}06f%craWE=9bAvMuTveB|(Y%9`4ktn_q zciRGcvcl8HSj2fj6AR6Z`sxmdCiL7^pC-)?ow~#Z`;whtLMRb~wT9bG^IssVoW-fO z&C8bOP@02d1ZbU;uWK%3-?Lwq2M903Pq)T*kxAXeOKGCsr9x~F1A0yg$yRSrLBSJD z9}Wazt@LG6zK~1+69&NLXCqBWXHGuI3n|b*iqFXowSX_8P@=53P;3vSnH0wpRXYl@c9K~oWx8qq5sa|2%T^%sse&K z_vUx;Uh;0@C>3J3oy^F9EbKEUVK_2mDG1%TSV%1KPe7cVllQ|3a}g$MM^o;(;_MIQ zn9G4}#%%W?410s)s3&_l=yABY&$+|AfLP3V z4BL>Wo=|C@$_))9(nOBd6~$qQId`Uqx{UwMApX4F`lrWmVDr6@fOgzt4ss#2P=fpSX@)!7SyeO9j>RYdhhW`zaNe!Yvp^e znRR;mrdzgWC_B|BRZ^iWu3!mFxnTCi-*mj;1`pYfUFQlsPK0})y&9U5FaWYR3z-@G z)uWKf{OR|J;MH}#8-lncLM#w_{WkJEUmn0EVgWx)?Mi478e>pto<`1Fug{{bOTMSWJ46VIzEx_J9ORJZ?uS6nY4K(2vmNxf(0uHow>9 z6%}ElGde%5l^)*&X#n|8hBh|%WJRK^H2duX190Th@i*D!VKJg(6V!Q=ARt~I?d3Fp z04TWk#9A;U$zoj5xSaCM>NOwLHipx285kJGM3{M|a!l$F5D-?kw?n>Qvj$^N1DQ7t zhjZ-l)Myp=9@VezpM^DtQPQhLv8@h&*Cg`{6X+))kqQSu?uBdTT4n2xd+Z2&#|2z; z@9y1IPSIPkdESXiTwHl$Q5tP?*uBA4HGjZXj>4ZLZObAc$vh9Y#cZ-*&8eNS!%Qm=}|w2z)zqISv`@?W}l8U@<=ru_)Y*ac6sPNqFA_!$;6U}Bt~7AMi>Qnew| z>?r)cv(WGaf@mWQM`FL84aT_#I6$csE*~aa(LPomL~fp2S8T5^FSI-(`$T8Z5PL3c zlh`g^imIG{)a3R3_;%-O^Kcq-GpcX!0 z#33EBl%4n8w&turzv4X&`%%D=O{!7w0E^e%12*ZstE$#V%db+Hh76BZYSh_YOtJ~e zmzbFxM|^%)L;G2JPUqj5Qk)6G5qPh%HM5#DH?2c}lMI4!Lf*Qgbmc~1bpCeXwYOak zC&K+Z?RbwkJG<-oNk7NB14vGGwooDj%8MoBE!tn6@ND#bV=p+)wW?jIG{HX*zZCvW z7Die^BYlWl%@O~EP)G`!ll$BNnEH-}X0XEsoDLr5CxRp%{w};c=+_N$LX@|roh22Ud^b8h_qHS^#z1DW|U91a`=M{%~&N2)ncM^Tu zjl|_70fZ73vk_#2>tWHu+aX5mk-ZZUz94au&_StjQ$_A zy#-WNUAQ)?hyo(9*$7CpNl8(ap&gy z&N=@$|M~B@cZ_?-V4%QWYt1#|jpuphJ2RNGPPV3aW7%|wz!oG`twXVMi1rJ}CgCvr zF!u4QUl%1eKCnYMj=7 zNsLEQ2>#OL7Zjug#VM@xS7>bJ8hzY%=L58r_8$i!jtaEO^7t?R4kJ4YWcEDH4u3Pf z{B&z|AU(?Y_s`s>=)>Q`A(}BjdD;Z?FXBWDrJMC%ek+zRdqMUiZ$>cNkuXq z_#c}<#Q-@L+vdl4q>|36(Ky*N|5C5^`rCn_Bh1am-7Z{3lO%u$efj9m;)LAmx?`Rb z(Z3-kg=H(f!I2FDuq=mr&K2vy6Sie)k6-M1uBkA5;p}!bNmwD=$Iz~GrdaNeC!0BQ zYm2@kU1D;jvME$~XN>(kQ!b4CNxlvR5rwwlc6p6ehLT%ia937#j_7uVaYzp7hE-c$ z*4TWS?D#+u7dC^3d@TM)N8HC`xx*!q`v%^X-zb6r*K0Jh{};ta&o+33$ObWIDXj72 zXf@rNCZE~ep7COxUC;Qo{z1WRwvr#EA}HX$Iu52%EC8qVs3O)8ADuAWuZfG_SAKd_ z$>AcFYPDhy$~YvbSH1jl|HGwxQW=_eR6)m|KCig%F^+F<>uLX3`^gQq&7ui@w#j7& zkG2QN^`^Mm{Y!_{eEybm-cgNkSzTNuc~*igOU!~$c6CywR#NOG7sXWRr!l2vBO6t{ zTI-L#>tr30`wbDq!m|K&^Md{Cxc^l-K;Ub?PB^dK+|pq1DRUV6#G&m+g(si0=^gK~ z41DPIKE0P(^3#LBN}U~h*osXeLL#uno+gS#_J@r< z7lF7t`c{ozix|ZqfWWLjRWQr6=OeQO>%?~utH1vHXGF8wo<_v?BIjGshZ^i&y3p?+ zL?|S{g}px7nJaF684Z(+)qEI0P>-}Z<R0%``hrhA`{=`nA@kRCUz?eFW^-Oo5v%N(&=gkW790hmF_?0iEdi&Q2WlD z9G5#G6(Rta%ctCDRxij|m8L9!+Y9pZTiPguO1;P6+>;nhy71|I1NNu3ZcdcF8~xm> zp0CbEAGu;Ec7bK1(EZNjHGhc78n<4|zCR6K(tZ^Sp4UF_Bee*aKwhWS>nght-xqO{ zw|Jge&kGGp#_3z{>P9J1iC!{f-r2}%VO|6J7c%*058F3`%3hmMdfMd1XA{>}Z3jSk zBlEQ_En)Q?6H$`PEx%;p>Gd9HG9MTMbOm#!Uc1=rI^sUMUXG5j0zynd1-JINFpU!D zgAYJLdM>~l%WNP!V&222(@;5Il1)s+&(_@d%xgr$aJI$)WA)*mh56GvB%^Pt34Z^O z)8+b}>lQB+goxbyO&%eDt4=mbBu^EAEF;SMiQSbVYsOWLaMi3Pi0a=~<()3+=Knl; z+{F5&)b@ueSFR4*uhSV&Y`|smu!U?+B2QgA3DYhq-U7uItLvMjI%6k93#9^H$_?&a5A;b!SxIQ_+R_?EK=X z_$qn0S~yM!9xa|MGTLcmO_C<<6IU*7qFBW2;`R(BJT*on&1bTvm|L2|yo!Uo&S43k#1(Z@^xHf+-nR z*~Fl-%2x!@Pz;+64?UeEQy4-fh0i3|K%_rjz7#Wja59^hu`+>K@cu` z7`z()i#tTEpJpuVMGQ_jOa)#o^HoLUDIR;KBng#(^ZMFccaAdiyT=uaF!|-Cj~_q& zaod`#c!j3>?GTo^5u(6juE}og7y<;StMM4rjL>N<@P0}&)}^bFudaZlecCFq z%I_Z5n?#rnhxg1O3X~xOpg@v;!O}dvRMSVNzzT*htxVO3QD=ov(PnyqKNs9L31sNi zKAbCxlJA~0yH`f8)-x{45fTbd*AoT!arq_~$#T&VxE%irg#%I&Av1f-C`tStX=z8J zj-PjXN=^r_t(54!bb-ERxjPxV#S^DxAxzXR(_AS{GWqOd_2G#T*xt;0J3f(}`5oIL zpHJ{%X9B-*4ou*ry%SC9ZmI8_K!IK*ZqLleYbn*IyRM@#Bf(u}ANw*9?we2ZK_U)! zTKmC+s&XOuru;c$*gcZzmVI7`&&2UZg0i1is!+BSYEW##IM1aCKZdbc4PH67WRCyb z_xJZd*Ty@-pT00v9&cSDjl|o?pRDB@iT7eX3*|TJPjHe7NT+_ALjvOJa7-9eCB78n;xWw3KDPgz4os5v`g`MoZrj3LYvar@UKZm5Um|QoR@_ZYUntf9>d+_P+ zZ_)g#92)m|r5)!l6yS8>pckPd;y+*Jk$GvgIB(AO?ck0gjNQ-WS(6W0f7U%bviA>2bzeIF# zB%dRk%Vw=jtSjo40hV@^t&K4#utKS8jb?WZ0+rMgU6W|ZY2x=vqL;q}TsPOPvFt>D zZy8hUfuHe?9 z{!v$;bP6NS9CV8%6G_%ymdx^@^!G#UsxlUK{uyo7is}l%iwEadshp$B*As{0Z>)`@ z=B7G=T=Cr^G*VRM;m~+_VGdE>_`;?mQg+3YS;Ht|!Xc_iGFuk5>!u*iXawbcCzHll z0>)WQ6Ld;GL2~2onB(lqk=K@x4yKfee+|s@&zb?z76!-nHaHh+QgKZo#jr02u`j_F zJdrf3eA~u&pNpR@M=ep$tS>2WT{-Z)s%&1xul1ixk6-J}40u#y+k~vSWT6ud>Hodk z4@Rdwrs}6G%cWg`6PzoPjv%>JoK_Wo2th4E3qAVb5@@o#Q~6wk^3cG0FDQ&Gf}oDO zgok0DhxQwiik?RE^nu@>+-w1eUXETHP$!;mF4ZrU$0#kizu$9)Hl^$!8Cq%Yze2*f z&~WjVb{AUo3V5*`Lfn!Ze^XHobC0D6KVaE?@{o;PuFeUb`h8fQTwvh=L~wBsi@sUL zr3Ht#di?0wfY31J_52Fb-E7l;+!&pggWbJ&&ig>Y+$)@sL)x34e<2V$t?}*!oVFZk z!@-bAW*jq4Um2$G-W$&4YS7DYAn9Dgb9%a#E(h26vnnSiOq@D8L`iz_zGAXGE)1Jt zgfY|ng0F=~&*z!fdLo7%D>zNzxX{feee1X)rk}qHSERN9+QoZV`y&|7V?51kL+i~` zY3S62s1D!rhMgbhlVJmm7UoobkrAmEnGJ`Fftj$gKei1vlYZF~YL`XM*mse344df1 ze9iI5!&%7j1xrfSd&JezhMk`LF$ErO$;XYV@~QJ#+v6iD3uAqH~1?`GrE9Q zwd**KGh;JTp(y8D9G-4V*!kEw``KPx2^h@dm#HSkz@NNg1_uV-D_rYr2tz$leCFzN0U zhO|KcH6%^7Wt6&>;-olMv=k*e+s5$H5fotrxVxjj^E&^7a|T;d141Q@|LiuGvKDmjG~QO~1Vaihf)< zWxLVHGoLC&c%X(}%yQJ;LofzOQZ7Q+hATMY5kiMp)g!DwTtUVn))`wrCnz*C@}^|z zCNERw#xiOzkVhoAOX#01CJ?8()3FOo_V5!71xl2Z>;_6G7GSS^vK4=#@AG@)?bLMX zJLxg(+L0}2!{+@S?Ew*C>&wxNIy1aY(ZO0h_rx$H19{e)og(&dlg+ZF(9m^rDUmmf z6roQ<$0G%|GzF~(TDH#Zn%q_@MDrSEJTD8`i_S96Lj*bY*}XOwjcN}hxtf#pB*){> zITf`{N+mbDlRYOGbZi%ih_)*SY$GOz1(wG@Hx!hITDUfgsV?4*JaUYLbLkKKz!Npy zslP6rak;#9oj%#@Db`z;KDe9wFnp@vJU}}2Mfef2f9_6=-SrPw*$x(6c`gszy4o0Q zT0>5~Mn=-TPF!kCH=%LN18gP1?#jsxO=3I+fz!@A1=8uaT@M}-KbqBco#%TUMWH9} z)`*aDr$cVd2rEeJqJK+7fpz!^YU66(PE+dX#24zVau3t2TA2()imDgyGp1p!brKI5_+3_bJ#K+U>_#rm zN{rKOF)?I%EOb|NSq#y-wzJa&S$~MHJcBzwPNqZ0QjEh~dyi0DjKtGiiyHPTze}9- za4tcIE9^n`m0KQw_=VU5-*vI`F7&N_Cw9I4*!i_whS?R@65|^Iw^zqq5WD$oDnBc> zk8cbWR;?FZr(egvujz3oDPw{ZbA@!3+-^`gMPI}piojW8Y9hX_L!;>wZ?bTB?aN=2 zy$5NIwi^y)mA~9%sltA7CGx9n5-z_x?KH4bxzzF%D?rns#x#Z3Ti{4!OxF}k9?oYJ z*J84oca9qdvY5^O!VOKf+EXn zG0)N?CU$t{aD94OX7QoQ>lm|3)2*(qtn9{ff0%e`RVNV7y2BRDwR$Dj?`05Q98>vS zEQr6zNmjb3S)da!rESOBnqQ$|dmILlts*H=%kX=xDX0(6Db#2V)IZjU|!{2 zbR!h*23SQQ74MpGXHwq|a_W2nr(Obge0?AAGnc<4b)8)(dzDL*BkO`1kySiU{Pz;{ZN*x$D}C->;c|MXddY z79hJ^i>8IY+k@!~VMr1^8tf%4dUs+mAyki6D#O++%pdMt*`Cj=LuPN70U}^oY~^hD zr&Vyzv3e{_F;c8nn#_m&(e2TS4#|LSBs^A8kb8W4t3Uc@fO44`Ej5Jg9X{wb(7|s& z|Kd7_CKEnQp>3N{9afPQ;~aWSkw9wkv6wKMd3my_u!tpjk^pLn5hM&nYH| zUW0M+QJM@q+b8gs2!!eRKW~|+_y6QA|NnS1*R=%mG>^@br6@aR?4CVVp4z7mU%o=G zmprU+GqA-IKJu@yoXh)qE4K{x9z7sx_M~@I$JR;kauEBS+Vx+#wTY@2Ppz1*eKr+7 zvyLlQf6wubt_CY+{5EzLcR9THF3w*c@q*%{IyMy}x*qvOwRsKKSLfSJf889IOz;l+ z1&j5Z`nZ3&uY3f#l5v z-(Gyzfg5=K{&F;C;HPzHKVW1&Vg~8F53VinGddhovj9Q#8m68t`#K+|VStn_p6+p@z>C7xrEpjbP|8IU%>qzs)0sW!AZFS@~V; zg+Ry(!Kd*#z?JT&dCn00+o&%K*4uVUECevedY^Z&<5wIC~b`_RKGgU zkJ#M&C7>67muLte{#EIIR}=ro$AaF$`r`>f1b_xTc&YaJ-v*)6Q9~&3zlIV7yhH~T zK#fxcT=H19!JTre*6I9|zS(}F)AIY^(lK+)dd#+68RpZnmQ~%s(7(ULs>2ha z%CZT}KHKSwW1^vW1K*my1kRR!Tao^K%C5;~&iA}9A3;{ps5-Y3QBzs!!syd*rax_! zm#8r96E;<}Wb2pIqvSNTNO`QXGWIlC_9&I(NBt8Q0H`m&(}&g4!A$=d=6}BD+y%W- z@ud4t*1xZCV~<~gUp{xpd-88*p*|X*O^qLh*dEFXQAih=K}4v7@%XPB1z6GA|Nas` z3C7BFh+XUN-vL1eZCLGOFZ}Y~fw=i}Cq1xMUjKC&>b_60T49-9N1K*&e%B2P(f`|h zfx)t&&Aa+%#Q*u;sz?omCHdQ=M-&)mf4PsEE!4Nsdi_!b-HI&wlH6EAb>fizn5qb| z&eQTYV?{?DRDL8zwW|C2nj``#wAz0)e}}wyOns(@n!Sq*?|)syt2Q_ssGfr5R3S%w zSWgs&SS5Feq?VEod66MVk*UBef%+)(bL!kvj$BWcAF3{(CBGN~1`m?p&Nh?Lo@{8a z_YkxUMxQxbMzvzQ^H?R+1>N-cp87Eeb~p4`QEqgEMADn08kJ#6ZaDfPHNNN1V;f_Y zzh{+7r>Hmhq>GlZR-p-Z1_i=5H;S8hngzr^52oS{F~mi)1{nCBwymc7`JV5#;MshM zQ>+;Q)e?dXVmP0Uwi(UR*lM?drva{IO^qF!Q zzjh+%-}6AgrG8Vuy4jK0lt=gR&Ur^{r48TV@hNNiW(qCtiZG};P5DJ zuPEIOcmXd{>jNctw}7pxkYlfDjTG8{A-Mbfj!nFtYiU-bKzD@;0H=5WV|6_BJ>X6% z#(CrUc3H)!c8_iCZw4LX2<|`of466*e9wL}3LXwh=y}xh+g47Pxh^6ZC*w5qEd@7A z-f>(0Y%OX$vvTXxv1?fQ<8iVOVsm|YT;;XXF!oW;RjVrJ=xEB$jq(pJg@7iI1?Xu3 z@jw+eNByA4<>9L-+q$g+rF0Q?u;=!2D=SLm8_*t1q&W3kg4c9n=F5k4K{17YpC2*h zdiqRdd&e=pv`91Z?MvrPYzf98GaJ)$OKsTiW~m(0_o}I}{c)GL+-=sQLs8&2-SIK^ znC`G%T^o_kco;|HI_+xk5z9gvPqUwC&v&L&Ll zu#EbEDF$X11B9)vn-271apTz^*VC2wDNq4whAtwZJ?GR^+gAk@xGq(kR)|DQirEEXVX{RD{`LI(W>CNyEi{kKpt096ci> z{pP5SoeM=vSUB$^rDrfwya#EUIkVDKKEEbGb_3tjGEZIUj({J&DQ+;sAh-8F02LN; z<1xDbjv4H_3B)Skj9Nn z`=*dB8%B z@AE~OJpH+&#|A4{PvXIgde5+M*}*;(*ZyHX{u@~i7HG|2cy_w)B+@|460dog@t}keCqi}1e&UT@%a#k?4T)Xq12WujuvR*Vv@K|O zv-i0n#oPyUSR6;ddb|Q9OoD)x?YLrCNpxn#6n8a$F*iF+yF?h&n$8nq9Z8V=#}jbb zee(V&MH2Y03!>FR`vP~SdueI9EGY$tx;|&R2fe;atos^Npmq#9E#X`|KEc`Hq>T1h zOC1Z+S`M=~%GL+A!3IPkiulhGrMI5LU)|#zZQa=OA9tOyN$m)1dv?LwwB%zLKtuV6 z=;yL%%7-uclf&?T{K#;tzD;7P*p*P*Sfls>L;w9%fs}jRi4t(=L9da-s-n2?skZbu z2KJBB#SWo};l)JrG>42wy+eE?_dPQAtiTJ-9DTooS8ZSKGOE?#(YjNA(EoPJkR5n< z%o~qls*y8g zOy4ZgGzHQAhGJ{+$Bw&@W+w^TlU^K-=YeG-Grw|QrBY`LW113;lY8&}D5AKai0&_< zY&|Y5!MEeh)RRF+d9p*F-wZbnx_BRbe=XPOfI;7IG-%rI`jTBL(cOGB&bsk*wY{Iz zJJlGs74lrO9Uy2nU?j732oWW#ysM@uG>G*J&%`EVVH%%9W>1*3fOd|a1!cCdEQ}g5 zQ&UzThF*`gixt%+F!={}uY1(FRjlmIy^!){_Qj=PSlGolU#g}#(QPg+<$L*UkbXI~ zieJ`k-bWMW+Y8U9D=YP_m=-2$*dKEyt2R_T`!+H(e735vIt+rN8Fi4dObCRXw0HT( z3-}rMG_MnBnLvn%G?B%fw8JKNM%goInt8x?x%n6*D2l)WJ9&`kbe#HZ*R%5*hfH{E+S@67u|Skq~#k|LX`SDrk#D znw>v7TBPR@_zx40%6eX6p~>|5$&Bb=ea!>G*(V;9KX^)$IaO|H3=}ZAeGrXekhEB( z-SXOYH1Vd7S)Gynsjdeujf3Gkiu45{a4%P1<_4$;qwY4S`On>^H1%usQIT6HDtKyx zW%B36T+(1|U`C6(3BZ$8gHchk>RU0D0|S-6>Th%Zpx#jdELU#i`eZrnUu_VH(xQti zoi)xV-Wb4i)CIAqm~u2r8V?i+uci4aN52ykv=5IL&z(c1Ro|mfezaJk^cKPgK_s}<|e28x1MGrg7|DzjMXlE+VKgWVMINnu_68j`L4)rp5+MmUaGP;C`hh`3VstY&;x=dJJ54qq zR>2(7jf436x6|Hg^_R2|LThF%%aQH%3rdPrZ7^L_c12B&T?XK{@;xBJpu~OM$zs@6 zesSYBl$*tYgWS8oX{~>|>30A1*x6ih)0OA>*+%hJPFCc%ICbr=snOEo`Tr(=N&JlcC{UlIaz|r$z`a3>8oPupQ)6j-YSqMPU?GYt)wY~u+@b1 z0T<fcc zm5pzNWV1%xW6e~%$iq8k+~#az7!>aN`k|IWO{?+_2;FS~YhDH-tY4rbNOW9A)O?a( z#hiYXPc*8UaZN5Ol&17(1tCVmCvq;zm4$FGq3$;kj=X_j=@gnt9-?A$&;(P7_D^4n zJvK`_tjqd%%2xZ5F;+mF$uSN{MM!zCQP3seb8L?jP}csVmRX`^i+j$Se*U%aLIa10 zknN~o94+!C2uL!vw8;O0&>IMH<}xHO=FFJV6KA{+`bL}l{K$B1bzf4FMg=c~zY!uG zCDl{I1u4zsLk(R+RG#_oRQ0y<1(PCTd>v&%Al6lLzI9$L8nS6(U)t|H@5vQMWw{%N z-7Lk{U^c|t0Q(tiKkYW>6EToXL!f{Dkxge3L_pBY>d-jl+`Sylkf-{}>!$<6((9h- z?u%G;hd2RGRb9ovVnub`oXF3K=91 zL8i3hzSDGl)i@o;5h7+mThQu$ad9v(3d*D&-+epKrF^y$Z!n#iN*g8g^Q%(2K4a$f z*#^gP$y(VAxXd<$+z|nt-c%X-sz6r76#DXgy6;&RU`+(cIE@0?8c!A#KRM}vNi;oo z+6t{=6bM^V?&DQm-^0{P<2WDu8_YgXMK$t)gR5z9MGf10&KHr+k?XvIEo@tguu#ui z8u`!QH*e+XH$KW}jf5vzMq!Dl@L2%j)5BIZK4G4|h0aOy2n2XX{A(#oi$6p!b|tFh zy6{9VUW2Tuq%^-1WCMmQMb(+9AO;`JQ)3Z4o_3mv#LAm zUOQRccndiBtaf$AbED97gh#vB`qB2FsPSoJaY36-B zyY+qVb7i(u@dIo<{R;b|t&o$`53(AKgG;0N8XQ%Z9RDQs39d?qASKl^z~h_uSMeF$ zxr)*bR?AQYrVkD-a2#=(NB{z{kD`CqvN17yQ7N_#PNkG=vS zhMV4<@wxcjNRXF}v++a_5fL2MgY`Di#hAXiKMNxLqd^J0iBHep>GCUUb^@ztMP^%_ z0Ubg(T&25P{siq_I%eBHxx;fIYVc zlwVFyvr*aYJNM8J7K7)Logc-K;sx%?Fs7QgH-dGpaSr+j)p1S>6qs|<_PCJ+(&%$w z-L*efbI&-QFsBl{JRG*k2qO9x#$MYQWVL&39fv?%0RhFdk$5YQ!>{m5kejNFz(^s_ z0GIuci$>i1qqbFUEpg=Mt?;0%`dc)$2;%|SykD2m1GLT!G(9a-eqfc3HyqDyBsHA< z$)7@zn~nEMHQldH*NSzj@H_aRr0N#=B8m;*SRNa4wOae9D$Yr^#w=J``pe;biwN53 z12);fAK(Or0eyhCdL|26k3Z1?Z(|durfz9nWmcEwGgTVH)XHwMKyw;o9p9aoF1GFv zgNW=}#J6};P#TQ#1Fp?jjN`z5Zz)O%?X^Nvt3Jqt%)q`Sh?5zv?}u`lrdw2U_eSVV z0#h^-Nr7p4gp|@P!y!g{pJ1W*iMGOaW-^tevXwxty1_OJ;}I;S1P{*jHX1(zgVO!G zo&*!-TLD`oXA9qzSYj+9eR-Zqi>!%3j;4!!MT^~IBk{TqM^s1)o^}JEPBF@_CHEDH z01Hzobs0;~u5&+9^va#sw@nNEp!Zo6_VA)UoQrczT^t124}Ee&6ZXX^;|oKUi9Nk( zAqq7ELI>ewcL)sDPO@VWt8`MXGWQ#dKf=p*X7mM5#pk3o^gX^zu@iHGFtP)lJpHgQnhSni53KQAC@8~o-%iigHP?CSlR7YZlKxCh2MXG(hujFHprjV$RX*TwJ{ zw5*0g5YSr@d*0dr_Hb4~jIRFLdM#L*p@?u2bIHoQ)DhyFqS;C0^4EiIVl~oRA(_X; z0Jr*fX6v1lKn(#=2x;>>3S~WH?6Nx?mPGvi0?OzIaONfYyLL3Ly+i@6X-)?7kC@HQ z>o=VN-W4>+S~+gC-F6WBO3{I_l9RH&#Qr|rF=k!%OUf~O(I?TR4fAAn^$SHy^oXW1 zvDk4Jp&^9DRp0jGde#O0B|C35l4oY&g4&6NSiIw}?;JNL&uyz1lX?mPdzW`Pw3D%c zqeW@-@OHA7J%QJE5SyHBQ{B)7(;C?*u$sMQnV_l;7fCz?*oE%C8o{mdn7`{oEkW-H zzj0W<<4-JP#p=r>x5iWAa#gb!Fk3bLv&uqrBzIc3(^$uGlzA$?A0`KfvUd;*Z;P(> zc|-INjF*eP9nn9ksb$BHoP!>!+Y8b`aFM*przTT2RW1iu#@!liV6@ssb)6J=rFunA zk6AWpmAE#va78cd-Aai3-dNukri`$ODmtdc_nZc}iM*Lz!{3ikcN&#-#Sn{Exlea3 z2=3s7U6fxQP1PxBOm||kHKQS#+9X1W4}tq|NmCR`6QWDd#h-S6j`fTh?jIs-cO4t> zWUcHG^4k0-R$Jps(K0^DFYPQnO^p1M(FiHsi#5^f+!h3W+ZE(l{R5c#p5opF#a?I* zli4}Kc>e)8-(UmvqM?c0bIP=O*8Oi?axk=@CuEc*^$!->K)jMXz2;2$ot0XWkEWo; z&0UN(t;PVqT_^|0a3tOQXtqzcKtkyh_QK;th1ZC>Y6ies)-tcKgefz6SSA9l<7rSf z!&n6V3Gy^ZnA(cpCzf!6uCiT@ND(sy8qIb!? zuGHozUUa*9X}aQ9huq1m*N$F@69BheVW5awXEd#z2MSMPiTSpg=4ESec=sS^nK&M? zDzf6c5Jb54vIx=6qKjAyz@S{eQe?txPKejo*1C#}dAs%q4HYN28n`CPopj4}k}q`> zy$~YKx>M%3qoE(^&fu7C%*aZ@=={0V7?|s<2NJ~blGKMjsxh!yg1YA=N>B77#j~Ag z5r{g$D z9-=V9*~z6tIB$R8wT}yZJFRnaeoeh&dsd@;k-KjskF_wnw45L){V5vlLpsf5`MfXdYr!a&^ln${NN`eME$ zU8d7k7(ek;pThVfmoei0AJ{V}%(wN~>>f&MhE>!4_2tDPen1T@spPT#TP(|#b28*Q z=+00J{qS`bgRC^~eVOCta5vKDk^ZYI3Goz-uF}Z8@BBdr%*VbF^>fQhL72&6_8=$i zpap5OlqFh52S`@?S}-y+m|`=WiI<4AGH8hYygJ^VnYlwr6H)b+jB0>1=KeJ})@H~< zIf2AokfD?fYB$MhB=H3Q{E^4=1pnQ))cruE^L(rIAo?U#Nt0j*r<6Z1b9fsq=9CH z^(f2khaO_8Z5a`$v{8yDQ;ZP7Q0lvBl9h2apNCQgoA|cMm>8#66+fV%!prQey$^>1 z$>xLF?;%p#y<`$z(m?u1V{IxYzHWzM8VHv_0=0c$N&G3p^2jpjX{Wc3k7CKTf8ifX zJ(BnjEx@OY6E)zDwM!?h@@vjRXY8v7{wyW3WG+bM5OEB7>&;U#px?3M;$R9Fgr)Ai zzPw`)iFr=#99-H@JQV0$dhgvDvq1;GFfiMDn1C8mP6F}CQW&#=w9jU@mth9Hyap>4 z=`Z8o!s{+*zGEnI8-@>#C0m>YB|T-8wYzS{jDY=?y*%@zaC$C~?iM8}NBRBuC`-0GHkKy$16;n2$NL1`-p z>wjr&_rxkCW}N-#Q7+%2TM99U)KT}R5z`AL068azy!VQjDraTj-I80j;3&|3hv+v3 z_3MUC);b+M4;{t>L-C)g0L5nXk9hH-j;TIzhfS!Rm9YwwZyf&6FUXcrYZ(qik4h;` zXLscd8uC4BBjLgmm~Y~Vt9b3M$g`^_E54>Qw8t!q3412wKo(hE@}8KqJ-F1|n=~0% zEaR!HjY~`r9D{7rH-{`e49kDOQ@&T(2sZzW3-*@ngZuZThqQY&NGwtg($WEdKL>?a zClh&OBC%Vs`|?a?2R%xtnKkXB=^-=tKU=FytHY7JvO+lt;aD-x;c5bsZ@;CSOyJu* z5Plpx{M90TUA)d|dH?%6VjC)wID_7!JqDO9iBc;zrKLXeX|04V%K{nQHVyg{EQtJR zAYykSEyUG?OsEwrY!O>wTVyCuTA3k@ZPBVmBdc|8!h^=RF z8rvV1hktLaq3<2I(_$knTA>Hc07qO|#U$A48NW?w*}Mqc9*K5$xKU9lZ3>n5!(1hJ z+52dIAuD2o7nvNPjhomj*ftn6Jw149pDw4by`Qhhu^t=y2x(Dwp3&%Q?}|acb~;7B z*`0-C21zFp8xsuhikrOd4ZF`FwFyo0Yg)B)=n$Zvd|1dWJM7N&m92I(UF=0&wmQV;U5@A~yF)C>goJj<%_OJ@>VG*SL(-@Aj^xgn#+SVy8Vru80X9JT8x zvwy^E;~>QtdX6`*a_-pW>|DrBbJ-ZDi4Ov+2C}|^?5MwctugYN_l}E{#9u#?#{UVm zD*dF?pqxf`5;rWe!X^*A2Y?Xw?(L2aWLKiVlL|8Uk`(>n_8pu@Q7SM}j=NJdrECwe zl<3kO_sr(E)_zEB(iz+<(GX%#ZoKd@^spxw=*V+eOhZhVW=1tq?;@i;5fd$8;^FiL zcXyNUZQmtGY(i<3P~@kZz9QdZX|tE?mH7Fuvv5_?_Ll6_iVuvajP=^P0|Y}EJqKIT z70cLt9AJ7hPgGGYuE{a$or*`13HaySZgtLGrhulp-XyQ~XAQn+IFCSH64vR6ck-Pj zfVvIZ_s2#gUn+e|)4Kg?8nFTnQ>|0GXjLMBy~7AIZVKBn?#`B~)Lk5=@k&?W#o&PJ z_|oI{7>c&DZ}W(8 zJvgThwu&uXCbEZ~h+u~oEzJ{)(-1&90M$d$g(6rnDX}x&>l~m8KuQc%hePdtHGRd z=Z7q%df<|!*^01^F{Jqi8WB1-lFv;18Ggm)@=FT?&H8S{`m{|R<{Y&z#h!ctU0vxe zAl?`hqvDpxtNk&OFMjnjA!{PUpJxVnSl0wG3c9)7GDx<-Y&z@41*nW@9=(BuFBsqz z^4eKoV~?`5_k&i6YwN8RHmC6R4(Q{GyfZogmgQU<%;i;xWfO=)Gf$go$;jb^u&DnT zC`+9=fd*6)5kfy$xr{3OGJFiRZiOPuRzyO$-{aR09a*|JP4w^`+e3($OSz%f`0dZb zc|wFr{0TX@^Dks-1Pk%)Qtv@#-tP@^mth7rii*dt396&U@W0xYPt@Odi7#rP^1GYZ zbDA!5n7c6%)Q_OGev$2iqO~52?!#y^vFfM^@BHYc%tVn|*mu@7KvS^FfR=Me|BH2T zAhj7wfi6Vhv{kbV5&dv&)37&F!l?Sx&euGdm579Zz1gVc!pF*|=Ch%Y_mnm^nA|su zocjN6_LD*8W5XKR$HsI3EN_Xo_8t1-0kao7OPrGbR`Qcz3#woHatW8A!Qs+AFafu8 z3nBuTQP$EJWN0V@t5va#{CA+(lhpJ>Atb+=Zt zgE{N%sF!9NT#-dzC@xZMD#l0NwcOo5etxu)6uh@<^M-918{{oOJ;I5p$FU$^vM?vR z0-X;^erI~(hWHGi(6;{-3I)hblkepw5KB0ajiYdXVWU<|HyEkD_?Km@R(?wIMsaXV z{0PNj9IuqorwQ(NEtZbQ@iyy?B|%Kzs#rswQ>XB=QuCHpaq4ec1eok+J;}f85GkE> zmKaxwvKHqQwfd1ZYlL;g`tCofu)5+Y0Z!AyR-bA_AfE)A88uID+TSVJpy z_!1JXyJxAF%XM4)(Rf;f9+T!RX%cv6YfHc$wLU*QiOceX&PB(`YGvHG1v;0`h$%lO zOQz~cJ_w6&t+U0=Yl56=P22p&EF9D@u;UPw8dhP=f?)N#5@IU&O8$ONwA2Cx6?Wc>IpP%2TnZEvaHB*=* zyo{y!WN+zTJ^)22qEuO8^q6$q*Ys+3^a**m;0cp}*3+*84W9XmiTJJ3g^Ep4x`Z3e z)dh+;ytLwWd&`&$Xr9J+rW>B@GYDn@dnho{QrHe?uwMcw)`?@_zw=_i)E-rsq-zQN z{0F><=>}vHet<0g3#%)+yjj^NY82oNDr9ctHhRgfpY;KUWdi6GpjMgsf8QySKy71A z<-Hs5FCBv7KN|b5lA%*^4iL z?O69<1DY$q-;k6J{qM~V%bEpxsu?f|%|j?Xb10m8IV=C<<*TC^nr)!wqF3ZBicasO zjT#5wIdM4$Y1F>Gl~DK6L$~dln}c54R#ry|*kFU0|BM{qqP`~ET1iv5#Ja2x{-}D? zZEgniPi231=H}WX$cSS@+OrJ#xG8a#h{n`FDtXiOMapV5sGH#X-*lHu#)c2jWo`bH za3OB6B}fT)8o++0g)hO?TjF-9j2N5{PafaMu7ey=~{fu3nhR8Xo|&!POdi^ur?WN)$?!LnV8bAM|#3Q)T z+ii)XK=QSzD61~w8$dj$%uWKqt!zV#yr3wKJRGcYyI1LzI~J|bT!pYKN%>#mY5n}C!+3EY(%Z1 zqKt|IdO8l*()}g@m%6;^nrQm5lz%mWmY%MKs|D3BbLu1Ipy-xXqnS!#OxcA_RtoH; z;0x+ZR5p|uuBXdQrDKPU2fw|-XzdU#Ksq{|w-bXe7(ykKdrt6x@ zI$TiBD1vvk1n4}npPTY5XQHSf>2yV*J$T0-;H44^!%}Kbz{HpYTaffiHo|8Bgz`ox zrtHHFvC|6K>s4g6jw1O2K0ASuOiclPvg~c0iJ;Hv=_)u*`ML*aPaBsa1+5R)hk~V1 zT(jlm=a{5!vP4kIPE{YL8|W`zBE-lK((!(y7_;I8gw$a_z%q^oM6dZW>cFLwrtx}a z#Cp!rN>$f!Ye3vq`K+I>MdW1u&N*mR0Z<6q7H`O6zelwlgD>qI0IR8c^_r$}YUDk? zhqHA+xgs1rdmuz4CcjhJI%88cGfjlhl0kJm$Ks0`-f8=HIFFvTR!%$09cMOQes_PO zD92g~SARb?rD)*Wc(3=ulIX6 zwqsmkD`!35=l~sh>hTF6Xi8%g1-d+BPjz-?8zLk~U!dZBuP1(Izq1uF_wC_aJAj8d z0X^cx68Y=DTd4A7#3_i#v;;Q}G7FDYl$iwrr|_*77nBWu82Sf|{=5X_LqR4`u^|!w|)P;1g?YmOV*O_X#I8F z_wJ$4JhEq3{o{UnZN%)?37p2$%D+LOk#ZcKaq9JwKoEfn31C;5^(FDMrx23TZ$nNf zJ(i;@!4OXlW=N!#1kZr|lEzCNQEp53bi~Dguy9p89)HQbV(2!YTkFWMiQxl-;DFPE69e@ zFbr=aL7#sKuDzP`QMzZP%T@~{8n2;m!3nvXyS^Gj-bR07^cqMcUGePT+j;p# zR%|+z&ia;VcYp@8@SxoXXaFjKHxrC{_O=)$DQ>|NtrkKo13_;)N|H(Z;roTOhkGVa z9(a%mxtFlj?>4&vQO^`anp~NY(uebacvBp)uTjG{g`)C-$xc`j1|rB~wOaw<2^AcmA#q=i20^sWDH5Xal3czFU2fn~{pc!HsNf1^9zzZX`Z9H+oK5nJ)qr{ZMrcYqyb z=Pk4pu%K<{^?mU}PB~MA!AB`Mn?8Yn&)ZcfTg2g~OACqEZG$RU-0la3_Jn`=C^GO6 zKi>ru;)q`8_i?+X%M^Ep2Md4li(Q*&f<3+iuCE8oo%Dx2*4Soqy^Th=aIuD(5oaD?o>@Xh8>I_aNon@qT%>FJxD4v z1e#Vk<1`?=&8siq;V}XJH*=kJfa%tZV#{fFT$hpU4qZJ=$iXNEg`{e)$dbvIT{kYq zRR*;qsu8HkBVJknl~XA6#UEd6RPr2(T)#-LKqu9gHR6E~c#p2pM20v5n%GfLin`Lk zYm2+;Bs9=GNmIGr94Kf_oWfLcLN@H zOLD=&Q(=fOlx{nfL_!s{4cB!(0HgNF10qqIx~S6p&oDkLi ztoE_$w1DrVB!A!rwFINj1m)d8T7lWdpW_LUk{!WDBcojQw-XN=2Ys{M47Vm{HcS46 z5)ekK1_{=^-cRL8tAT!nHk0Z+ZaJ@VaaQi^L5cY@^h2^q2Qn;@mIMh*XAj~>Po}Y5 zfvk3l;|?{oobxxiBI-K|X;qgCKaF&0t9$&p3CCnB9c}BYl2(!avLrj4SFv8f$8k)` z+0|ylK*S@XrxQGRl<#lQ%g2v9QFJ-0^}(IJ zYV%jyMXnrBS1e{@Hftal8jN74ywfIphie0bYhG)IC$+gUrRIk)g6pJ+Py|JU+-+|e zepTWNyCi^2F!<^8g-6z-GSzF@ms5%7R3*76oU4SYb79V#=9JEQ zVV+o!T^MxQ#_f7>D9`}7H(Ew(JdkB7B<}gV-Zi{P8Hg&<&xzr}aqh*Frg%ZBgwgly z=k7a6m}5HZ6YI<(Z7HWmiuM>x5OSm|6V31VN*INz-y0--*pof=%$z{ z^N1rIW;^g=TrE`F+y*vc??dVEEN&g=oMCydih*S;rj?9F?R@x05DSd0)>(Gj{C+1r zFV$Simp`J-QoGUMj$u@!7N=sy^P6pQjK%0F*FL@@_t+96kMn7uU;5NUwJ+i{^=tdB zdXI)c)>*>)EaRZ93(wNB?cwgt^UFe^Q$R=dGqsus8F7ku36l_84-^2>KfSNd&4nJq zBEt$*e=^d`Wkwm^|zGMFOJKL+jO*QvpQ@+vA9m?;c zGd>{mnW4w~UFMD5ZRa79cwOueLXnr{oK3FHDmmKaK70qt3Gz6rfD_*v`#ZiYctkw3N z^0cMnu_O?tp80Bl_P>}d?!r8KopiEy-IAP}`vOp6zm7eHsMVNY9V2js4x>y(;#=3q zeSUxS!0OH^1S4L%Zys%Quw=O4P1Y}grdM*~9ilffU#v34eE7~_T5;SadV81F&E@ii z$}!*JT`%~}=)Aq-2~JRQOp9%}fp1VzhpKFPk2?|vL7K6+N~`d}^(x)2L*OsMV;+&7 z5|UUUjR_d?nJ#xefD?52Xz%mZRA6NF7hQG2}OjH8Q@P8*b1Kp;2HpmX-+)^Rz{#v@nUA8)W&uCGA? zY7XR*M3?15nb`OIHB~ON*WBfF3$%+=gcD*9dXm`yhtxm=<7q^yhPp9Z2L))9^>}2jylP^7ecNIPXr^xp7e$(5Vc(;%jglGI?-jf5Tbr}t< z+d=>o0?_2YfkJIO_d|m1&%Oi#Cgc4RqBYgv0f$P{%Rk>R4*9@?<~Gduu1?^#S@A1DQ4jkjs`KcYQs4e0tPwEH!MRmrtvSaLQ` z=lE*hGQ5Pi_*&Z^Iwc%0T6qNS!9KT{E|3&x{I@HC(o$$|q;tv%QzjA|_y76RB{#M< z%D>AYaK=m3y-%T7&GwXV*UFzoKS7Og+m3DkYDiFz)}{YO{MWsUy~-t=VCPvhy(V?+ zx1$p7A1=TXmy#J#Ox*Mdb_k&yj z&i-Elmj5VBBN}O=@XR9iNtjO{6?eB}1SS%>4NjR1{-3;k8--QIhs6j2+gNy`_wW@#2(S8tAA z&nZ-3nFAc~P~6jVbK}{o0Mb6(mXC;sm^1mtYWFB0(faGVptg|JTJbg06d_4WQb{0j z#XKwiPtb?k$8SdH+$PCYwy(q&eG1$nWYE*S9P-VLI&pqzd&m%t+X;x`nxabFM1>oN zq?!)chw_2KRe9(3jbGa7Dl7|15*5;Y({@nUm}QY@u^o>@-#;n&{pOvPMTE>JN_OEB z>^e@L-dEM^!%i&4kz5pPBPZ%PF4KOzB89}j${tUYjukWE20;+{rB{i3@ zuhLXU+BA2ky`K%jx$Bq3>;LCNr_`YRwiGPfmsg6x)daM`R*!i8y#($-9Jyo7mZu+R zKlg9PSK|cePaabA^D9txtRD0HH4{LhXBzl6k8F91I9!DM(L8!x+kYCSWCi#c*mnDX z?riQ2twz9!z)1S65$ivn03{E|?Ts$lFU`GaeYc0`5G(R-V)8UZ>L7%uBJD^n#t*ZvQ?z4cnP(uXp%C zo?rh4$42~GHhz#zAY!m+d z)CO25Ja$qwbLuFz>3jzA%s)>bOJj`YQ+gP4NecPx5oQS?2YeePv2G6IcXU8eqRKv$0p%{xblS)L=!i z!}p_(vG-^QqU5K7<^ZLzMMxP=_+p5O@?PvK$&~^sIOw%&dO=TYhaj?#m#%F6XRKrQ zkv;pvc-B(2C&}7}s4Y;cz!-SW7czdYu_PhjhKmUxxViEV0MPB}-oKI9;tyLX@NVsV zGCQT7*{1*d7Issz5S3z>9lRfsL{L51~IZb_3SfS;>zoU)W>6M)7@8fgcpR@n2qh zAq>~u$@|s|Aypma2kNK-rjo6$#xP6KI6OXuDs9_;pv8w5)sMfNFO^-@u`GAR`OM}2 z4BaCNXYsWsnpp!oZ=v6cu0Zpiv-Za`*{fTeBrzc1VsCa1d8 zO8kGFoYliiiu_RL%rk{AN2zhC4pZkS{5#s*Ad^fd)+$$UBUIClOd|K>f0pMhio;w0 z^)rJo1}UCHm9E;*1^(Hr`BMa~1tPTiz=M))-3Fo(@X@KxYU_CGxXApV z4?6#Ovso0%HHY7Lf#|)TR$bB*t$6UI$e5`+X%U>Qs=BRhLl=D*b?e8)-3mXzE5H^Z z`E_E;N$)8-MNU5<$pcBNJsU1|S&sybe^cgZ#j>!)3;*ZU!UZC5?qeczcER)Kyc;k| zEuV&WeKICJ6z&UNdZx3XgW}k>QLOFxC($& zFVex-ATtC|rb!*y;|}u?m8!K%N!}eUWf$}Cpg4+n4j6XMOd`X-yXl#TTQ6?p2wOt2 zE+*3lm9?P+wXf>;##~g7b@np)9V7=BUEq)cvj?y!*c5xQa93U02C>^qY&ze7O~Kw& z5r>q`bZUlG@864+U+w3fzbu81^ooG%!X~7r&r$NE*9=m&ewc(dK=BZC3i(@OumAxO}-^YCl#h1+zwdb|Qbp4Au|ux&`Q*JKh(p?#eP>v?>@iPYR4s?+jv%82Q-)x} z2~OM(@wsQ_CvN=Aq?%*ld<c9o%WM6_?H&V)F$FgWNBF zNU|o()ZHGp3l6~v4UK_&MP=tff{Ee;_K8vy!{n*@{((v_r!@&Lk&^FFjwIWLZ5MkNjJVCf724AVk9ly2f zpffi8+g-DTJNScc-=7}-ds|P);kL90r4Mrs;)MFe#{W6H!!R8U$HoxTnk zo{phVk6Lz-jaXg^ALff@h!God`CX~!2b;Gzq}mnzKPvyOfQP|o1q4DsLaX)qlhwZI zB&LcN7`i^L6g{ibzZO{g0Hj4RNqo9m zYX8M7fwEslnKpU}>Vf@WZ*C7Ne~$>7c9{DWZwe{yRPnMy2Ll^Tb@z#_H*;HHWmG;F z10q=P_PF_aeQO7o#JFUw)_;yiNwVsV>`$*7If?7>F{C}eEGpL9QnD5~ zV?-E5)9{}~MduQS{!cz>ODaViL!h3uKw5?Op=&7`(a6$+@|$ZXSbi9_AVk5?%cfv8 z-k_AEo+h1a>`E+-TXBZ%3|ES9?&CibJUp}%@h=(FMJNx6RC-dMlEW((k1OdjX*7 z6KP*s9y1-2f3WA@(=Gmj%tm$))_0=m4nj9Tz3|phQE$p9_dfwUfj0I=PCkASB;=S+ znL-h_A~in!a|_s5Vf36z_F(KN0*X8HKvP6EjpDkB-53`AAC6+uNy^Qp>QC`sA{#+U zIDa*8F$}`ZzCUI%ouehE1nUI9UbXl)&QeZ;Q#?fB2khI9!5%07ha>vpC74@q1bcotP$uonPn)GYt;9ib+4IpX~g1NGxb#Z`{H2 zbzZNtO2mX(J*D~gx5qeOu#&!_RP|RqSv?9@;8{Ch7e5!`Ht~P^dGLi1raOA-e|TQf zi{$ygyvSo`-|)Tqq2)vT4zbSrD!T*{Hw*3iEG}rg!mc&Nvhp#DTdN9T8Aciu|EHMe zq6?-~mWP**7ZrL`aP*4FyV)d>@DqA}WZTOEzReVfs0eBhCa~bH-;_P`G#091eKyvw z3pYRDIQwHt!tm3@ALNb@iMEBR*TxzM zZN_rp*1ab8PGi#@4mmu|^vm%%C)DAdkm$MjJu_kV)>voE2Or39P`U(;XHZ`D$x7JG z7RmWtgLtyI_Gy(nhQfhA^ZiBo(A=#or~U2ol;^ceih93!j=89b_(Ug!Mu+{)j}~2bbY!9##481#RZ?1gqX+^ z_QH@T!W(#ot*@gkK^Wco){KUz%efw*rHLOVyJyRu3>VQVR~lu%3OG>m7$otX(i<~u zsbDST@Ry^Lm8oE{QDJ*@n0F?TJ(8T1WB+yWh{y9q3Q{TR9XRL5E&u83C`B;3XvkcS zO0!^KalcMqf1<>y_h28mMxm(tbj1wOI#DOhb-#^C>`ORd8%}?A_FGcoTNIV=wYTe@rfu{JREkO2 z@2Z02A3qJJCe4A)*MuG9A04mQIGI!*VM0m?o-Sm}uq6a`6b7EX77GxB7D83Cg)jlB z>mWpzPbw}hX@LB&(R1RCQXh~+W^vnY4G_4${| zXEZ9;hWzS7-I{2PyiXQg3)W92dU()j7n@>azUM+;FOa~d8x~Q$ z9&vOw|3EJh)Rhh^g*u9LpL@Z3_#G#L99PykfqWRbi?NI zUPf;L>;{+sxG2E_9c_spD3BEvWw1sv1c*=J3swA*c_Zfr0RgC5- zckAEx^1J6ZWbLJpE>y-wA>slY?dFX3`GH1|Nw2~&TSgttU;VT0o}ueIUj_HY7$7_a zr~6DLdxB2;a1`Mlmw0(4FVzMIrFa2h23iL(FxzY|E1Qg#C9PjjzCiGZ_>+cDOyJlf z0&U*Jn-K5nz|k|W!BDCEX7W%F6`wVQqqO|;u;whBEvaSDc*2e=-91jVbpQ6-vv60? z&rOo)TR63_z;m6Ufq%TJ40td``@aKA)nPp9Tq&CH>uXTB@En>QXdud%J;-uC=qY?n zmemHu@l-Sy=4WefLhrJVD6gqNbI1`|I=TM2&TZiWg*Vb9r={sQ@-SmJo9g%08(_-8 zz)Y>{4d}>p=^F++utZ<9!Cxy?H{;2MM6<&sG(02M9m1k<>QUZRp=o#SK=RPaY z8?mdq4;=-6+G7?4hM?yQXwol}8yJ-Q-lFYA+0Q2+JI++=md;NEnv=schph75q@Vhl z)xglI_=UHQwr(7k)uS<{G28Ois{#zhvs>rNF^dsRQ^B`>Ob)dTK}nEH^!sQcuZ=Zl zn~x6(1{c~WWBrFFI=Qg-U9I<8WB@~P6_UX|AdJp4AGBv|+~;My@7s?SKw$FD+o)!q zI0*YO+u4$w<-fBo6C=+{Tlou;+44bA*Ox`|m%r=8_;`XTpNCpZ{XvPA5HCr@2isE- zkCz7Sgb~S_jqnvjtSzg9UM4TOzJi$XvJaarE=vEn3BkH%C)d6E(>EY7!2JlJH=e=a zKk@=@7gZDr)Q$$#{X%y=Zj60DK@=S}2n0XwC~J$^jkL#&6RUP9vPWkIi8Xd=!+VKtaX#Q>9D12|og%g!ivNDFknOG!(PMyG13qoM6y7ps{5$^g&&gR^=!NSRoEe1J1%|;V@^Kqu>+zJv%YRQ>2W8xqSYOz~TF?3A z;fFxoIf}1do!A1(yv-yUH#ww^KPi^e(&5u zy-Iz}=F)^7e^{lds1hpD?uX!jbgt<*K73mA=ggtg6_*C|mht0G%%yuthQ4<4eH%tkf>*iKE0s(KE*E<+2;KxB-P-Ls7*E z!qm4N`65i$*1xHbb zU6L%pxzlJ9XP+sO*<^ekNnc&n$+~p8P`&$1wf+bGgyWr0_{LUy1$eDR_sWHes!8A6 zKDf7MM{n0nRK{dn8E;i6!^#j&Pkjw3l?ksCLl%f*Yt&6g&yO;I$p1Bsqim!Q)i3-N zOUx*|!g?WWdY1iJksZmfaSTu@P20bfmKE*e2d)GeY`PJHH zx77;Ef`gwG0#19lajy7T`%xMhYZ+d4s-V~o1>K`8SFdXSotzfnX1mk_oLwKt7ox1u zNqbI0pxORJ*jq24XHvPIz!VlVVIxEZ+II5)>V&?fzV9s$&7mGLK6O6h=u8}}flR-C z3j_h>-nIviu(;oTKBqh}{x$f{gyU!Whin4$H0Y?NcU#l(SNwYqk^9|jQ}zGe!_Edy zV(-$9m}1!H;6>S%^>$u&<7HJs-< zYlY@os&*SylWlX*jSE4~aqhcwtZFGbf9L0nIPcC4r>zhrOQenlg70glG8&)E+d6R? zvZOUkX{uSm%?3Lb_K{Z32IX;=22Aq8b-EuHX`Fj6*a6%FX)5J(EZ4?xJ>&?goAsQ8 zRGbGL7b)y`TtK^%qABnCUhD@^GMJrFbEfrqia58rGbniRGiCK#E4;Q(;JHwQ3|*wg zAx3?BF><;7i!;m3Um})u`FHl}tVBQe)jfv3o4_jIclz&P-Btf6XpiP@oe&f(>+Cf1 zVBGFI8IB?)@gR%VUsC#BS^Nr&xn zxz>(czED7Yr5Zrqb5wdkr3G{gr%uWo(fv+d2vOfr_Jelj^|v039#0$a{G@LMXuA^* zB0t|Ab{9s=mu;k1&l}4mkkaFta;Wt;;w=1tmi`1e2|s4q**Jh(mJvNSIhEjk2;)ZmvC6j8me4|i6pyI$RT zq$7Lz0`7{Vz0O^V%T$3V8PS8zF=)@tr&Cbfed)pM(|st9dcV(^TnIG8;>v0EgwJFG zt%NlE49|*0PNCDC}? zBq1X}8D3AxDYLk;^{1KJ@8{VWWU*Lc+ue*3C3<297;e#+WU zVKPm3%WQRU^?p&ppPVxp5Zr|Pb`g?ZZa%`M{sUz}K!=W506x0fbwrELxUpg5)fM0< zs+8kT(=n;{av4e|GAt0~67Gz#lmIX#{13En$Ounmc_By(~K0sS#8ZTI<#rbI1E?wqygth0ijLcnp%n+Qle>jLJPOTUeLt+P% zCQyaXY`_42-Y|OfJ3?k1fk|hPu1j>PxKmAAREgQ`nCbetyu2r;$x3*M$G$)&(t}Jj zUa~16o3fD_K(`(LSk*nFGjx_k_@M*%uREI9{ALPq!)@Xco4GJX25a%YJY=lfH4}iR za!9qDoyRht$&$}c!=6SpG7d?P#{CBD;bRh_lXCQBI=$A z9f8_K+VTKDebbO!ZesiN!lWi&paZma`Pyn?3K=s91nO$pYpbk6sXYJC{hHtqZ?Ddq zy&A3R2n+&rBJ)avBFXz2(*c{l>tOB9?0tC&!xq6Mf*)tC@Z)Kj3Oo{&-yE9->l(sj4k(N5B&5e5 z(ojt0s%)Nlozt6x@HX$0ijhZ60h-tDagK(9XK#8{qplasy(S8a16YmoI|)qs%EwW_ zBhD=nt*L$`nN{8M64B}-f1`rXUh{(2`N;QD5eNQw1gO5n;fn2L$o(lF8qNP)m0$m; z03pYb5+|rgsL+d+a8d+9yV$-{xzEPh+M8_S@b}O5Bv!w;p>I+wa5gsJj;nQLf9F>Z9_gJBQ5*l`CB}Qx2H;I&w6rLOVV(TIt`2rzp z?qLU+B1zE5_29xim?!sTgeC2T-h4sZzKZ-NfD80q6TikH_#HuWpl5&kCpyClv&ABN`t?3vkqET)?zy~q)S}=ffHP%% zZ~NHketuVNyM$O&TW~h60=u>!GVQO`#}6?}mhruZw1VVFc)F-X5SfmnTAO9%4LI>% zA}bS=vBpTPCM{IV}7I~ zMW*NqFZ^KbH~_R$0vO82%6XORk#_z(9ji|cen}j|RyYk;zv0`)RiWzex2*VC96d(Y z#?eOwe@KsU4pln9l-#~Kp4-us(FVY=PgOo{NR~7Y(w!o35P9?a_oh=NEG&n~ge#>; z28PTd>&+!S=k-XHpR)1jRZ!yeA9rgp1?;E@IlKD|QP-omuLK&H>SDH_Z^~ z4(lfnc$T&xFEn;1y3GOn19wHXH{)#%5?HE;)!)LJXr;{G?B^mPvH`M3Ma54f%_xtJ zb(QK!H1^FwH2%`sJ&a}f)VL+USmx@^qe{t*`g&x|D$c)wY{G++_A`lheKNTjFfHIT zSW$wKd*q>03H8+lm~$mA<8Qisi2P+EWSdY0o2^s)d2{1d+0^b^0jZQ?<&ZcIcu;r; z`yJgs@-`m_!6nQOPGMo-=YV#=lj^C(y|ietjPPhP>%T3z}j zNa+%BF7-nkw3x2Yh(y`&Rw)RS#A8Is#P?0J{2-4JU|zm-Zm=ul4omgs+Sjkf2Z_$n zX4nltg^H2cKjl2AxhfJP0II`bD;mg2MTqY-Z?{416m>R8*Pp@0gMuQ}sfHGA!Y%&5 zST2`FZchKrEdOqlRUPr#r08Z*H3P%5e5iR1z=GvT1zo10H4adqbb7Bl3UoP)-of*K z+J}5(E^694_?hf(-2cQ7w_9NE;a%Z76@tv7@t7lSGW?vXpT8Xwuu?b-aK!YpB!L}4 z;h2WkI_$@Oe%AgS(3WN@*DBV1wo}UK`DDew+3s!~^@YV7kVZZ=tq<)*A~n2oof!Q4 zESFZQu-#NMfyQ_M^0)S$L31!m&)M{Yt__iYzMZn-lpLNX3-abd$~wLuw3+@;%&*Xh zIEj|)#_Gb{Y}#oeruJ7cyaMBM$*z$gbm3o6cudFdxJvE*yfc)uT-dNrVxfLX71R%Ao5vx?t z&>%$1ESuB-@SVx$C0@Y^zYymeAXBl7AhZ=zk40lfoT}F!JqOl{$N5nO%Ox|*@?S}> zB%s&PUeF9m_Qk(l_3}LHjNV>VMb6(JqYt=OOT(zw!DlnU~5t0&QV) zpHngvAWOmd3q??7;UI@xN2IZr&Q8zSFs(vBMd}`27?Lfhbr~_SmXsfAo#QxnK1w z_3S^gF@B<=s&C;~iR4|Gl71br*|5Zye9qpRj$3wVFKMsIb=dhh5-lYo_k{@(Far|JTZdwO-K2`Sxa*F(YQmPY0U>sW%M z3eUdWxqs_NeAZ3*&tjET>@$gF$0knQPo}Ah4ML={z59=jltyX7{Jx~qWKEK|IEqL~F3^6lWBJ^Qh!?toefMXW`$>B#>PDKWb6f&Y?& zxp|%vt|($Z^B1#i2LQ+T3y`@>&NuGYuL;|RoaytjPjSCP)A;GD8{J)N6!wU$P&TJU zOGQaYe{(kGamw9eOB&-V(LC9iH}5_$8Jn$~ z?}$c^Rr2{oN(=tXp|Ny=1LyAwfiFOpa3AFfcFw;dyf?1_SW`h;>t|GhZY13Ut&41l}f&)MY%by z!BVQapP?;q8-?PQS6LyCyB2KY-DBZ4`4>=^rm4^trhVdB`E+$d$8DH1)6r&T7)DB^ zhYYee>u<{R7a@D2=_uFatfk;t$)|mM&q@6G2dBo!(wrK%I9&)%>|y0P+G^%hNux3p zf5M0ECvlXQD~)vgwp8}yQ87>}%m3k&t+`w`54A6Q#(GNmW!$;>kL*WHXTr8%C3yOI z6t11@rvc|CY9Bznk&x~{Vg4FdgmjO0O&Y-=7GJpuWt0I2F>K=L`#nEuXb4iOVo&dd zoat~W*OU?g_1w6u<+o<(7n7=)3n9zyW|m@p`JRhMman&d=(@g^NzgU%>9)TqT^#qo z#J>+Cu5OP_X>DJB!qZKw9E1rytYClaFw=YA53+lYaj9^~;@Gsr0iVfPLlJdt3 zT;*A=!w`XmCkdHl9GPU-eeAYON62#O9V5i=!L(9ZsbHz;ch>FC6?nIpTX3cBbI#R; z_Q%;U$D|i1(v`5{LLFq29JLhk;nrje`qz9B>xsaXVGVG;gHh8SX3ND8mH`le54_WE zF&rUSb_R&lP0QRu{43;#EZK2~!t zfEhI45ElRSYx1CC^1NHymn&&;3JhQ$+#mY~UY=)$hb~@w7zId(@8rebZ1w$SbU|Zo zH6n2oyk(vv7bAGq25wyvb;<7OdRTGdqfZK-x4DW9@+_1a92xiM`BqTL z3jCbUKn6N`d>Pk%P?}QlsMg^HkkkSpFQVLBhaOr6O!q1rzS|RdKkJIvP(oC*7-64g zU!lcC1KXVJNZKONM=Q|m!-e@8PN**SZVq({z7zECm7O`%`8>~y<7lhXszP2jNJPvi zke>i)gkvTTt77@@8&DDxnR$kAtb0K4Tsz_Ln(-BM9qDdV0Pl=i5y)e?TErGnM+oa>AD&gId>1h?3Di zl$H>S_X8tKZ*3D%wvhT6dtBWe`k5Ira_^l&ghC*5wk9OE#>}Op$n{;+o+Hj}7uUyF zW68H@)Za%yG-~JlFamk-tKJ+x0!NnoL)eovh!jpdWFp}qiu0~NW`T3VOOpf1uB%+k z-O~hmdEEFri^`-XPwJ->Ezo9HPAt$C;`4kIxS!rYHYZHhBCT)jpz^J507(*5E&1Fs zoF$tnE=U%Y|Bn9Fs)jdimdB5up0?JX4Wd@UjbmbT$FcX8_CPHET}`x8mzl1C@Aieo zOT%Y1@D0d@aXdO(VPJX3vQu6yZm3U^Hk_6|+2X{?U9@lS4d-+Hn?5bZdzwX{qT_&2x1l&ILZFF?gR1(>fAN9R)Vz?@+@l^PTKU zJz6KA(Q3dR&i$Oq!3iIJE90z|sg~?R3lph5G>>i5Dwe2~gKiKWPljAf!1J-Kz}|OK z6GuTl)G_GU--=PZMiVNlDp*ke%6CPOjv1@_Hs-`vIZ^74Lz3TaeOLWn!%lv-h@l*2 zd&gr9rksqJJ(vJG*ujs>e|Za;OeOcK@goPU!ZYs(H=IZy&}-r<_!Gk=jMA#$Wuo zcd0A+v5Q6c45}|Tik`j3A{pu&2W1P(Ix~6l^8``*J!fe;Ec`!s@E}r>@-fmVvRF+q zPMdsiK8V{fb|GUX)F_3VZ9glK(&IH%>MH|ss$bNh;8A(nVka}%A%6Z0ZJZF6##H$v zxj;h?Mll79g|#E~{Go!G_COM{9Yg#sm#zj5ZRQmjT+!>;&4~@a!yD8qbYniKA&0Nsd79%rv4Jx`lV;0p@DR%Ram3x`e6-OMksS^Og zfcBb%K+AqU=mVv|mDVotRdU*>%`o`FD?tUJNrg(cNa(=R2s<~!6f(S_63$ff&ZNA5 z2cOq4r?Yk#Xw!l2Xhd_!zx3dKAbBz_R7)xlhLr*nHNu`Dy6-cdVgNV`e2ZY_N( zJQ~w5^mcz)&(&z^Jvb5-@P_$KOLAc>Lr3e4ck%MrVpG^2+(-Ab<4$ILM&@;bi_kxb zUQX7VVw3~m+#5qRR(;%~99>ZX#4KEk$ktmqX)|KE)nS!7CsZw>Edpq-L~3IuYq1cr z;qng&2ox4C!p98?DNmU1kZGczI-x&!ky!1WlC?XjV=F6wXd1><1Dw~k`qq(Kf1+?g zei!N~S=E#&$(Y)uO;7gzl9Nnjo#((wZTA(y%lMPS`8j46@H-CEAzDK*ck&*vYQ`YF z%V4siiOf$U;`4XmSedmWdCp>_17Eio8YaG%!F&$B-2#3vAu-qQ)el>>rF}8WJgZ$@ zW0j~rvv-9>lD%&>S`sLYVy5i8(YRtOPBpRXwTRX|?MXd)e)%qH#P-s||FYmRahfi-f${36f5jkN z&ZcMInSIHu#u1`lMdRi<4n94ljVP~ABV=fPT;#xmFJf=&tzANg+dA?u-n|w~qtJLl zek^5)F099@{O6!bdo596Ruq&6^mKB+JPa?a($MQC@F3n5w7EkGMsZKxy{KJ~{jw0n z?jX93FWH4Nas;ftJvaGv{_GE0#-t^~xxMiz;FPAd4_DTZx?`ajRE|J2iwV5Z&d%a@ zM+))WWHu<-A%r5*qY&zC<08f15DK1(gSmcUNP9JIpjx{X9VV|6;Ta;F4CJOFF5%pW zh}7`E{`jOM0eZugedi75`09L65@Nlgan&G<%Hbn^`+xXx?(B`22i=hj`A2oqlf420 z{M0pkb--pbZF+YW>0VVNL*Ko6JkL=c@HmBZ=<~yK(GQmfw}jy>D-G(z!#PMjJO9h0 zC30OAArbIO6iAnqqDjS1iCTipWtNh?mV$$5QIJg?`zl`Uq7Ql~mSi;dPRZ(GDIAw9ROGT@1T`~q2keg36iWlZ}j@|GdB?1S~jXr$&o3S4}1)HH6h257q_ zyADZy*s6prVioEZ6*twB1sehWspcHMxEm(rWx$DNX1`{=`J{oZD41YyNmq1@y7lOy zZTPJiyfMIN&o3tOAt#z1Bd@{9pOVl`u@pdwbK%`c?Evl>4}n~NqqO-{uF%uSmI!EV zsLPcGV2u<04@7kRhy#)c&6?gN>D;1V4+WC2S;fK*m z1hK?t@tr5OBQQ-Rrb&%KV8t)GccDrkG>AKbI;|`~u2Blu;=nyUGsVfU{?gSvXAyT>{Hpgv_46S?OjtN;A= zn>| zP#sUPSAq3`QsmLmX2p1@H*Km_0YCF2^4;nWJ8{DrVbZka$q)qM=RtAsp?Yi80Df_E zKA2W5A7O5w!rIM6q+F5t=60e%IAedq1}G;;{XJ24!@M5g#SVEtE^MENR*vI!SC);u z5RVNwLKAza=hvdIacSqCi!aP zT_P+->{TH6hET?zBEqeHt!TV8{wB(e*#LKjNY7gJjx)9F&(A*;HVq_J0G;&sPDL3u zUpCAjpfhlguGmBIsu=O@e6p5?u+{2d_koblfuJv_m?!QrJAng#8g0g4TO1hWt^ili z3IdE8+GX$9^7CfLzys8~VGT$%TuH!EX%~<2WheIf6@&KcQ;FRbF4tp1h_2-GB5N!D z0M(u&`_DXK!=?x>ci-Q+XOis)lmPS$oNISmJcT_X5R`L!$@D`SM-GcE6nEAEpelR^ zpYiG?4DB~cjfAJ+FtiMsWfO51ueN7MMNQk)A2^rxT*^Bdli=;NG`9SajW_7SZaP{^ zK3?1{Y!dezWP^D;5loUP=bSR`c0+~pdBEnXM~wnlF5E$(Lq4YJeenU!;{=(i+fAeu?sy~*IFoGx_- za3dp1?0XBJ+RG(i0v{sLm+zse7w3m*tLaJEO_FsL52BI5MvUViMkFWQc;P{o9X^BT zX-wteiAR?%`kysDMDqM;pkZqXsuupFm&M0-EVG&K7_tfcPG%KuUDXZUXDXN=8AGC^ z3a_K1Ck*R%YI%F0#Cn=AHH^2T6SI7yfIixaqNMj1lPHZBAcU0*e0kiZ3Y0p%&>vAW zl`G}>8uiy?EQ4v{Q*-;3o1O<{ah=Q#->=N6XZD>) zfi-7M?AA60G~tT|sPI-mZHMY-NwEKM8s65}SNdT>)QHB|h^}K4qbP*E<@3i7>JW~Z zgUGi7CtNfBijy^8-#(F>sNY0Ssh-ENe^+hPH-+Ql3yoAQuf7R_T%{ScKTQP_AXi_a z{!G)>h2T3QOAxD>-4`EASNt#xWK?5pO4)+{=YsmCZ=>}_6jVX$P!V-rQ-qqlpve}2 zH4*=*lbdV_#E{WpcEa({#^{Z4EMy_}gQxD0`{Ud#>!IK?K9;=nFb*=}(-MfZMzM>K ze?an`LtnAWpZnAYO4*?EY5HvG+xojvN)mls4E!PnoeBgdsh)CkAU-b|qn@;zghoH! zqt{KZS%$9?0NkYBSJlINutB2b;PE?nNbxI;p6sRUV5Bt0Nk`E5h!wF>NO<8gtm_2J zq=!9FGFyW&zhq72q3B%nea~&Y|4IxU9mE(-Zxs{rpyt}4GbDTIi7{+&V7$&prw+M= z6f9Wsi{076vMgP1EkXXRCd3CoLe!Wdui(;K-j;fU^kh92yr=@pIpH6Hg*Sll6E~Oq zRh(Q}E=k_zn?GB0Z}#m!__u5!&u(Uo%wjQ51jsUn>=GI{x8hQzBidaC$blw`Z#!V1 zsQ2#5t;KY5GMk@0m*c)epP`B92T`k8kNCVl@`IV0itfcVy1*39SP*>txR|UU#M7XE z?a6;)f{)({a;~3fM{%$#lFj5X@IJYI$7>=a@A{y-6XuBynT_|03pEq;9lZ8j&$9HE zhthXJm-8RW0RJfuZCdzZu`X!JUP-RM*ule@{C|T@k7ALKhRe z9aU)#?IT>Jq2GwntqPt*QDN%r~=C8I|DGTQ}4hyBpW5r+A zvdiMW=NHA9T!30`{;*3iec@N(_2ukQ^UljsN|Ag1;R5VIJqTHh&^=s)6ezfxg{c1l zI^F3(P53u(prdx}oFF*jg#g=;n(yuRJ#zNomT@Y%==^oykwXgAx_BiQVme9K*!*Nv ztmy!X_cV;dxu0w|JGL~ddxV90W1PC8Y9C9-X%zZB*oopO6#bcc77(n}MzGK1xAB1k zhbry!{Qn~NN^)MP=Xc=1zxTCOkbRP?cF&>dSlBKlAZP^OlV^#gdsa#wadT`-Ou8K5 z3$v^+Xjf6#?<+}%*@6R74}1(K32n0-kk2tNuc}BYm-x1mfa$9X^d$yxOTH8jM_-}n zlEAFAT?N-uDwuecPmLJ_nX!3bW?Y@v^<7^$wx8rj+?DqbmBz`j9QZ0sm>KxQ2}|(t z8A`>Z(6CpLMpwVTB+*rJ*ALQwQZ^9G$m;!0qfnCx@Q~j?y?buep0SMMNz0-UdG2@b zj?7GqJaCKfaxY0woJN`r_%??DByUT1uA_uN5PE3Ts|8^mjhcS;tNLHw5i1dxlHS1L z=Y=LpY(zh3?EzAXZCup92Q`X=-pl&CX!WiI?zTs4Wf0Hx+=KR14JmK`2^Q0VKj8^7 zy{-~-C}@j7>6BsN7o6?1{nEqXVa4=|UksSn-3uQ%@e7~6-%g01jd>)%H)*?~#9chr zKG%PO^-y+rjZV=pD=EANslBokrQ!^anSZ_lb_0s7rBNb%O-9QQtn?yW4SJHsO9AUe zy796_1}cD|W;T1sk~cHY0|f(xn3dw~DZ;s@=9mk$SQoM{V_KCiaM^{%(gpwr5lhY2&tXLaEdO?Ic6MC-Q+w1X_F zBMC05tBP1OAMS_x{eajH?|cZ}ZlGR)sAE|H(j4;D{BlL7_yXK_iTpvahOGd^S^Vsn zU%Ll(fET?`l$p;AV|JZ{!eQ>>OJIh+&4W&oRxxmTJ?8;Ct-O4s8%05)0d&)v2gY`3 z;I6%N&(tlQuta`PGayua9tDEP40THq1pABt_@ZWZ9@q_NWmXE#F<6F z&aOYeOFf3nY%krH9=t?}V(`2)6tM9@d6y9wUk=dSD9;GSxvcibv!XWFNyL^!?vnjL zl(HeAm`P`9eOyf-o|_veFG`2BghVrkklPBSMji`&-9}0Q^4YPgACF?vCt|doNaKr+ z5C2|irMEJ&%d|IXt6sMa)K7ExF*%>g-&P?~G7kXYqtXI_&sk@G{@!z7S_bC@;V;FU z2B|o-e?1?d7)*rJNV38;>|snd()vn4C|(MU&P59%PvU|X5d(@HM7Kml#n#W&eLj!6 zaiWGh2c>>!AjO0Ou){rSf1$*Kwg^nrJcudO5z1-j0W%40s?(0dPh8G~(tQtr;daoY zwE#Kqp`za=1g6h!{`u|5%DeJj=RQb=%LBRDh%a8PUga9gIaI?I3@ozoI*DlM4eQTC zpiHlj>U^gz0gVc|7{%S6LA+VbWI)IZ)Zo zT1b`bA!4q4f6?e&5#(`VE~-PQJtYwI(-TjOUh03cD=6KsV@ml+n74?0AE>AB%#;S9 zPJcG0s0=Fmcv=&SoT~fSj+Qo(%=exf^?ZbFv#zw~g%H8|yqc^lDnUrkmzxpUV)4e0 z#$WbYGIB{`9zF!2O`DGf^nJ9MgXTQ6Nk~z=1v|OgwXsV>aQ8-40xoaat2%+iP(m9E z!d1B~(3g5)Js_=iG3B@;s@RhUp^7Kq-_y?yF^{$cpPb6@ZpBRgYP+?=N#y2qgBl>D zZo#}{Ymh#=Alof7*lLmfa%C3)c?Ri;Ow8VQy?RXdBJb4E^0%oi#Te%As9qcm7Ux%( z`d%Lv|5F3`Wb~m-!GTIZ;2j^x`QT>|pilWbmiyr=Wn9>N@)?Wu)fZ0HXG)%`a!Y0* zIj8-cWV1dansCkfUibZ$T3G}t?*&i1?>%QC{89{UIKQYntantmH-e(P4mEZTD4zU# zdM%r_M-rkXJ9_r?pBTHu5sxbU&c>rHhfg=NrR|L2qZ}o0(RC1x!bFUj-j}D23z3wBlL}HtFg_x|!<+|s>^Or-et6C>b3KFf;YR=-#uNKgZ zif1l7xU9zeZPHKPTF(>98|lom2a0A#D!5Fq>`(J)lK@!Db|q+X4c6w1uJB9g;@t;+ zyB%nKd0T5o#pN=3*$;Yrv)F7S$p{8$^icF0VpC*?K;kfChQ9z@s{18Q z!$Y8L$WL3*2iYydQP~LsXYh3p?|4Gn^&!Y_hd9+U47rFgqGJ^FCjwv(6J?G3pwOj= zO`(nheA4L zsqwz4efY5uD2RELwgh+VXAo-Y)ynj5{7PdIeU+}G7J|`0s7cksjy)Nn4%kmz4kr#&KTYL+sG%d%Zios~0(u0d))PFq% zOHA&4-~G0+ulC$t);3l1Ceqw(stDoZfwG!=YCtoi~mL81> zupJNb9zn_z!gxVE{j_kadrW7jJ1BIkRH~lnr@e#>y#iW1^71?5*iN({wLyp_ zOB+GEjRU%*(<=D`%fAoKw#BN8-VIK0GNi4rdWxU52)Fh;A$xpW;7w=rOUY#%+fU3p z3SV86nN^rwn1nro8s`I?__tevUdB=5JaO27Y$6?AY72Ym14xqV>|+x}Tf;rE950~H>2 z^r4}j@2{<+lE89pM&pq8?c`*0&Nzj!Us4hsPV*zU6HANJgNnx02fT@FnDiHt!LE7# z4|Q)IO=bJ{4I>$%?2XKcZLG{>o^3-2r3@L$tTIN15HfAaR3wF>2$dm437H9zQW-O4 zDDyl;@AuesUH9*~pXYwpyVkqjKi+j&>$=ue_CC+!JkH}dzMt=B*gcwx+l0m6yPc(~ zNvPLNcM-jMhb~8N^}@D@I1>j24o)Rpi}KRJcf={cxq8ti9xs)Z!0eQS5f=JhcLdKN zSxY{A%(jf0oXpxL{19349qYvfES|Bbgrc~q!i?Hi)ngXyJ=#B2XQ`6&zV$Rz_9Hms z=|s=f9@c@d-7tp7R2GXLXW^(a5PdUn<=ylNALe0|oOSA3)$HwIctx3+0gwX?3}TOk z-%Y^0?uw5m`*vP2qX)+}+~&rus1o={M<~ZWkxM-`e$PQ!+gHQtfl|L~BIyF9l3Omn zNTp+@lUX}0+oYmIFw?64D8oyE`xWI10$6y$tjm~N?N2Fe@2MS2qGLbjNW%Tz)^w-c zg{m|u6=OpXr*QF`Uq7IrS(H*9!qhirO(nO<+2{)OLZ{N?av+2Z=z>P+^`*+ z^IONzq*52hIPDR=y!rws>RW;HbC2O`8G4wST4oe8W4!Q=Z@BsKTDz8z$;B7?}M{ML`ALl`n0&h@{y5u(>2+ zvW2me_OQRSPUbI* z!Bx3F>je>~m`OopdC9ae-&0BPRp%1w^x2}FV$<*Vl`*h2e97UZh!pxzyX(N115;=y zw}UqBZclCC*lmOWqQoy4ZO%$I#a+n9!(t=Xv!!p3bZ%$`^^QV~y z#h+9k$2T#pO25k(-EhzOn&_Oe)`8Uzt0NW}?p2nG^Y+}~R6*RQ!cs}`<~g!)OfFh9 zCzjyhSGKvifYH>fNt-&Rr7lw;u^{55?7+UmFT5S zSczo&?iafJCC))%DQcH;d)tIcwc@ZJ>w;8?CKqR#ZuZ}y3ADtV^Q)bocQ5U6Y!B;W zff9^;j+tAA*GI#~lx0=WiQ-!Qja?-a`VkB!)r?1n74-vn9_d-B>M3D4{cMkc)Yzj$ zf|hB^9RjJ|U^Im10vVwy^gOMFYi z>gndsXFJaw!EfyW3tu?uWd@8r|Jtkp0)Tabb8Xwa2t$rGu0HP7`EYSbw~9|X zV(o!(>+ElyiHYI*MHlhL233nJMdN40+K>8GV<-CAu?NzH`jV8F{0(8rSf5-zM{WX?2{DGmm1Y(}s+Fd%i{@IKQ+H+Y&bdF>g zqc%*D(H>DL@w7HZdx3CP&(`{dr4ti&2Zg1fMK0_gz1FnUX0Tf*&NhXqROJQ}6YhJ@ zs@Rg$`^N5E^jgK{X{;V?v|mI}3*yI@<4sTYfudo8_P3CxUse z=tS#-Ih07q*|*WsT6QjNC(dW$oMUJLmy3Ywuz_dqIe<;lW;#qup~ICY0M$iBU@PHN zS-u$X_=vBFx*x%A%r3quHtA4S33C=bPfXJ-Gef3QIC$j++h#=@^A9)RjT}S`Vt$6M zYmjTv&~60e(Ks*k7whT05Rl+J%2e(!+phMSaFIN!BYz<w{cjDEetHC4+OfPhwFVA{26GIG3_s zk2@=3;_>WP29e`vePs32-Wl3IYc9<@Dnq7)v`*#z6FvHCC1$aC!KD_h-#n`y?~658 z)f1%3#8GX%x{02%=jRf$*Y#8M*Apx6lzku@Q+F7jDzsg3{yB#p=+T$TuM4yvji2di z8=@?f%98KvyKl8Glat_G?^ZRDXXO8y=|)qh30hi4a_w>@fTgJZ@@3K`UdV9yTwn2L z(jK4Yx3VVt=fTn6&3YS^LO&mFGMZUMi|(n)5kY9D^2PZuAGxJD5>Bz3So8yO z@QlUuzcE6IGifnVXCfUvL_LA*#g{Fp)Ls0nnC?qlq3#~ygYNh>5TFsK;ivhclw!S$ zJ0^c$!J~s}$H|HJ!0^O)DFxLfrc0{#sH)MY@S6yHX@wE7-JS3l!b%MhcscyL4zl;0r|P}xu)bWlSr_j> zu>2X1VXR9Y?mIZGOaD`ewP1CJ5AJKdjNj5A?&gGL$WVs!sve*8A9~b02)a+DM613d zmyH(1FYfPr{t5G9yH)VZ+n;p7x4AFntEd~%nrapA%m)=lJUOv`6JY^m<$lp%hXDur zZ{^-UfLq^iR?DoPEBRq5^`l`s<36R_FVMiZM+oHcO};WKo~$2dQTu)l*;e&~faHG$ zqe}?ZP!#D3_Ot# zx5B+}$Wm)k6iV%7r^SMdiu}bcqf{Le6zsShltP`F0);HY9l0eDqnzT zk*8{dF#X(kBFo^GD%-bq|^vXv0Ak%OlEEkUfi!}X`LXF|@dvL=W=8&j(r*)P3zXjx2fNf}P2xa4{jY%p76fh+ znyv`=JDxEqG%lfDX{Px=42@`46Tti6Vra!KttzYSVJL;W0>IbN|LRY1VTIWdxj~rh zyvIJBx8F;900QjWC-*&}Xqbb(bI(NXGt)Gx(QEG)WtEL*LX}1AuIdWFx;<^EFOdLDaZf-sdq{!M4Jb3ZG!p5rh$G1-qzju4T zJRNQm8uU_%9~;0P6Gl5b*+jzjVKR}&5~O)!^E5dZsE7kwV-}HJ&`$yjbpEv+F>$(o_;ch)_c<@92zhC&?gzKTz{B`&V9XBPxT#n z_JRC}I740+!)gYf#4LGbn=?+;IbcD!L+0$O+g}`y5f^#>29IaNMc(p#?d@ggi?jbe zdhv^pOHHUAIzfqgtXyMC0lMppnv4iARWk*W19;2+x1J91M%5GjfGh0+5Ko7HNEfuK z2NDax72?+Z7Uq1B6LP6*d#!v0XaWCA9q^lE%bKyf0!g`ASFQ zxTyBv)}{88Il#1^1Bax`4FvCv(j`J)!EdW811!oATS2$#tlVbinT+_G!&1Z7;O6kg7 zQ3rfHEae`>cq7&a#2=jpuF`|!&}s0r0zyWACI!6CRjxqU@*Uxt?u0`@b?dlTxa4@4`aIV5_gIcO6(3h{=Q zS4)5$)&rf0*=aG;3m1ZudVeRk-ADX3WiT&7Ksh1Sgm`XR4my;3Rue+R{*)ittMs-T zAhPr^e)ot|>B_6$AoJTU_t;mzo70F~IBN<>O(m$O7wzgeoK+MLYh5F9Dix47vATSE zZ4Ggrq9fX*kjpZNvihe7P*5GxNbWtT+)wQtj_)gVok1#5UTDx611E|4;0|E%>W4Q`&<0nYq0Wp>c2EzT1`gtiWIszE;hjaDFJDs zwSib&Y@pa#n6(Go>*t~C@u!IWO7sM(GLdIpiq$27qL%J}_UM!z%?{-^+cwC(aj4Cfsy6EwW}5 z!(PNsFtghk31GK__thMFoOUw5Cr4goE|?;+2DNUT%GjePTa@(t?#+uWs>jl64MNgd zSq$yL7^CQT%)vnGuB7+cR{-SCyk3}pDNyBSRq`qh97RT;#?bo^wC6 z>re3}U^)W>p*t3USR_J{PjQl*UZ`(FhNFM)r{GUH6lqIC(T3~VHXI1N zGDH>g31f&bh=h$i8T;I&zMY*`#9^2CFLVq9Hv5SlAV% z2#$WsJwJbA#3~3Zw$l%+kH@(1o*lH)J`S4;dZgTQ%6EiccAb=lEyZcdZ|yfQ@+QtB zfrsVkVsIYK>E^X%ByFc9!?6`g&S)lS2_0h2JpP*)c?S(_`4esDet;tPvh7jH8JMpI z6_-Tmzq!*(l5YHb4I-5NK_+?3wL3^MGse9xcRaO+7O&jF8?DBc;iveWmVzb8#;Jmm zMYX8$2zn2t8kky$wcqPhkGEiIMaTM$#6OGO=REz54{O;*)OM~dO5o6RZ@wHK{lhfJ zV?lX=j4l}hd?+aa6N#iVVl)1`f9ADe4FmBDj}{i}4>Q}O9EvL$)deW2o}K0uk~Ev{ ztqAg4zz76(A0tTOfi8w2`vws;OVEg$Q(o-iRv5ae7OA`%_ zf$kD0P|}>F=$jG+?$%IdXq*Ja9|?hUA7=|u_vybtO`|H9NcCo@r^8a{dfELSfjR5P zmJbGn9Z$L|dUlY<yzt$p# zM1V0`O~*%FY)MEwAS{T1Li4vHzh7t5G6QvDYY6JgzPB(`{MwLly2ZhSviN=~rUCk_ z=jUa5>jK!gJ6tHEePvcdT*q~f&&20+tG_Nhe(mZZ8HQOP5tA7FAy^pU0!&tAce^>R|Ke@RUsA2B)RM-3ff z9s!b#fHQb0*Sd?%-U`LdfeD0;_e`CN#+|r2XhxksO(7T{u^=^g?V z?OdS?fuVDeS8`1y5)>!ikxW|*0zR+5{T^yzw4;KWxRqq6_$ z0qV7}X998YwjGC8Z283wr?6af ziP&MdBpH1gUowL+=KdoPtP+iQv~p1#8=HbDr1bRE7pA9cKBbhA8|ZmiNax-DgUA|6 z4=?jkHE;&beRlbwPTbeU<0>2he#&A=RI?5@=Y2$I7#oclI!DS;S}S$5Rz^}&yc2pt zHmZaJwR42w(i{JN7A>J(i9IQq7Rs6=hJiRiB zK!t{Whvcq?GQoS;pr1$Lf|ind*Rp#bi%!_$*!u9{wb5!cX)J+as&*uG)pRw6x!7-J zj>j;a-)$afzuL87HWaNi>JV4)mCC!~liz*wiYSIGkUNX$M`P%|757lPi*oY{S=)z1 zb2*uW{@VB%?I*hWRBH@3NqzK3SKh59Rb-UrmN4IK7ynv~+^7E9+1hPqh7#2m2j1B7 zAKv5ouE5uL>tWYUq5TdV7j@@rO#J95SMVrFlJx=JyH4czHSA^;<6)bp$6r4#Bse;S z!>ZHg|9IYPeq=2B?vaAHlmM8qISSBrwsha z$~4#DMU)jQ)u=3ZyjhvxX-mk$U*e}|v(`^lB(#xpM`D7%YrN>BY~$pL!?;mhRKC>n zP8%=jRjsckjy*7T}l8kB%Fnw83zp5)*4vbe! z@>IkZsW_%dMV_8eeK?fasK@y^C19bw>5W04atB+7l;^sTPs2;hnT z3-AP;-Q!Tt^xdRlI#p!cC{nOE#JO&-jCTfI&c-?ykEKp$Z5C1K2*#jko|iX9c7cL= z(4lH|;$Gpx(L*!3;glI=&KDHR*+{jfC5CtD(=g<8-od*l;Ue#xtk+Ch94HfC)5+4Q zKB4i7B9lFpIUQ4~a_A+2FP?oyLkV14?OU+g_GMBHOEP!uEZn1Y! zVNQ=iFGaLL}FwfF#2pLsD=^daAz#jJH5ANzBp45F7!&QDx_VwbOM8VK`o^!^pJ zCVpMNJE{8z-^>4t?-lfN<+BFjx!L@(%_p@huH;b*t2%S+yDr#X*(H%LN8D6C_kSTy z@>_pZY|tUGv}!qZfX1|2=l%~R@)Z!{jzVDAd<21?|1OIhQ`(T64O?QjaSnI%*Vecv zc8SMfu9q60l+_;P6e3>8t22}NXO>^^Wqu8s`6hF=O1eaYaw`K2?4I>#uy6&U!s14fzyr(Z8(QCV&H=$;l48_fBv zWlBT~-{XlDbv(w%@ktm`_4MpS=5Y{(HApWW;Swi&$Lzd8;04G20~yXt?$}-1a8{S6 zlYHZ4pJVBHIbvkj#hO)(Kjr&{Ou7-5hiH%X{PL2i1(nK$Om`Z3w1@!h(~HKf24InT zft}r}67Ox7dvx*cpWABmHssARw<-8;2!_Aww~8Ao?h?~1XI()2T2u$BM@d1RFmFyaBR$K+O1);ZmrT}S~>Zb5+dFN%2I~x>b(|_ z6)l(iIM*fx;G6$|Cy*!cg}1o_0A_)vYVX`j^nJ^I5dUMG)ja_8eq1@d7O)}X|FR+f z7yjohLi<1&!Fkmve&2;7%d~z?cs=|?;~gRdU%>3Y@yDJxIX-+AKr1fb7Qb}d1>Dj& z?D45@uVRk^*S+!Pr^}ypxn4}}wX%MB$+!~un-`)GI*6d}BEkQq3-;yLGostug{m-e zL|7g9hk}aNMy{=}@Wumff?R^ZJ{J@|NHv9K%|C4-upkDk{6ZO^nk)a}TS}0)p!M_5 zUv!xMc=zpTqO;$7-vkn*y-EA$S|-6Z6gZOLD9)(&jfQ3ckFre~$u$0*o6liV6D@_0 z9HbtA#45nLKS@{_1=jOU&5397;gm_-e-6dXp!>2LzI0c=rjZ_ zxC6CU;Dkjang0A<-0PPY<>pWfzclyXe~BolkZE5V0ORjODZL&lE7agKUCla4LdP zT*<|I^XXfz>C=-A#ve~2+IXE@qBs5eR|?0f@N2$~=F2t$4#4ZK{aFOJrhpkvnG;la zwReL;!D*$+BHC2yqt}1P7xb6M@tf}0)h|Kw}4&BpRO9N{PYu(KQmlH-8_tn zk$-5bc;UUkxBu3sV(82vuI7d3Uq-K8_>m%UeG<~7%)qLeFRKJRPhVN2Ucc%R zxTN+SnM#3)_i2vygTcRS-rtic^i$H!Bd)F4{crO85f$3B9-yI~t}J@rprfl%ba-hV z7L}_N3Qo8^dlz39^e*wfRFQbC3y3-qUj#oL_Ame3aqrp7`De5Ld|77(r0RUIhR_tC z&w9Okj}#X(04ct3&a`~D``2$83?)Di_mQ1|CUb$0cMoA*lFSQ(61tM=Rz$Sg)Kd$E>yRcdN=b%+8dpGv^Bx#$DlcWWhlp&vI z-`|M{phPB%C|N9DOnpMtohFc@0Ly04Cawfl|!{ z(23H{huVFhBe~edkAi>PxLV_QKl5h=ZSlP8NQ4NH$JFd7pLAfW3@H&ZM;a(D;F0AS@&sQ6S0f@G2ni+S!oZE3^_^Wq&f@_vQHZ2y zsmT@{ouQt_zQ#8Tk8T+}M|j$p#yJk83j~V11xU7LfZ^|J`w(uoJ>b3G-2kK=Y=4MI zirtXxLJV8M@zQr9H+2B;x^2 zl>~q7e5i7_hwl~AOOFK@!m9d+H!Db3P*s`L224$HpUDGNq$3uK`4Am_LeU&B!stq1 zTN>cYAUb6m=)X~~SxO=N?khry1K^RT`F#U^eOkgc>`|`pI+P(Q=XR7TKFw$HLk)^Z z6$nJ){fnP+Vy(dNwEaya7Yih4AbEhaeKSTz>HT09u=tU*Fq*7IQ;;WTLBcCov7KuLGV{8<@U zI{`#G52^%hrmW$iZym@JJK9lJV$v2E$fDfWs@4ZeNr3d>u;K;UDYHrb&Sd(y*?<2?to*j>Nhk(dwhJsR#M zwX%_0za;?KhHH)_Ee|4o2buh0>{0sW2R$rKMo70Q8QN` zv6v=5Iv7=$hqHlHn)Xj@K?%`$coyy~^K+4FG78w7&MZxuiApW|;dEX`GSA2;+R(OB zzrm4Rwl9ivA#$?Cz|`t;vV~wWIHYa(sYy;XWK|p*k-F8QaAg9^Qyj@jCm} zZJ_*h54Gc=J2kIB!;xN%Jn-z=tFzhg-jfr}WfeK!W5DZMM@W!Yd_Be|Kbj&6=E^OF zN5Z<@e3Gij2)3m2!j0(Fv5_t7CE-GP)JZ&C!^dPORUD`v1zNg_#59BQ!yU&=&&$1w zjc^HFUmF|WDt?y|Z@hG6<>%N)G`6EN|0;eNIfCWPM!WJQMy;P$p4RUu1V6TR zy<--#dGVMKaGA*=#Kg8}_Jgf$r3feM2`x+S@1OB)^qYySGWRNr#EkJJeq#9tSJb;0 z0%AM@e&#)916B}hAheGDfr(jI$A=J|Hp zoyoRKER4uIQ{Q>QA%xov`D;F+_lh3kqhJ`mLAedBR1fPjD8huQ69I;lN!*{Z;h=cl$%l>&ti`eLZlr`3x(WqaQxH zwNbCdR)dNBUFoE4v^2_a_ol;jm3 zsFE-lle!ibRaGGAwu$7*>6n-Q~z3K9VG8*2=4D^RRu0WmfM1ewB@s54p^nPtS|1Fc!e<5=zD4fi*^j1PYL5;Ij6wp z0SVKrq*E+mFiZspq2XgP!74x-HXt>BZNXFPmzE7fy~AQ>W1W8On;QZ*#p2-3zHINs z9*7_>A|x~z@#1A|$fgrQ1?|+%C1K{NhTD=1@uLxTq0j@H8TaY0zW2VJ)$j2_!ASwd zk$S_WouQpyJHUVLMoqoieYwHo%AK?vxiZuXO>vBbW4(IH;kd7o52}r=oYhj?gV(P| zzY>x>I|$C5jw~!Us!2p8D}o_^Pv*n9A~G7={+^Yv!6H@)$#7P&YMaChd}Xiobc5|z zy%;l+Iv9>yzbZM$^d_D@>|}=Kbmf_~=+zlID8!Wxa<{-xPUPAh&KMR!&VcJby<++ZOB$n)B_s-tVeF!X2 z?>c19TCd5#ra7razwadx9N}&K-~g)xBt0TdYV$* z!y@gcKVtK>vY@b&(?ycgL=P$^N~zr|_>$fH197BHun^0#C66LzEa}FYd)QtQNZIEFd27v-c=w|r*%du9mLnS0Oc5uZu1%JY*OR36p(nMwX+PDwJt zk1uJ>6x{aNKSlHN$&Pa~3BTvceZ%1Os;@0;_>`*xaB^z{ZeUrq)>&>I7b*EU8o8D0 zb@a0OIC&pdx{q)~wnu1&(rR~|f&33%nQBA;KrD?iq$E4=)C1@WeaPH_zBl6I@fekuFALGkEEUcx<&>KoXD)MJ=Sk}~(vJpXY-zB^#P^WyrXi`4$K=dR}oNR4`FNW+R z1aQk^GiIXOP49!l$denlS!PW#>w<5(1rImp^TJ#qGVzFfu~je_LTNv-R>)$gnQF8w z(gph-R|}PIE3&|u;8q5|vHrwm&}YIy5ci32oc*5eB`Yb1mym_D?d@Sej{;75#1~bjJ*g3f8s~NzOwE;)bbA(;I0^EXCP^F{mUE6jX&QfvadWI zecyM4?hz&SLKwqkAp6XIG2M^Q%0q-U&vZu9r&*sj1O9BW%{!oMBD(?!U9{#eT3Xap zR|QItxg60<2_0u`Wd4~wb#a5o|%TR+{DOjT|!^$#7pUHREcK*YY^iQAvcHS@M{3CajCL}cPxU`My= zSZSGQb(fzurwZSZg@`>Nj5h-^h7wf$i^d(`xJPjG-De<_Bv8&-al>e?=LME(+Y=&^ zdi|cDaOQVJU~^gTDs}*17Gzs=_`D4_4(0iHTMQi92D>n{T3H9(cSHLpEFdyLFAZP; zq5cO%4i`oa+9C`afA_4*kKGt{k@lRWH9GwWUMo%56#rNe7>sv~g6FkVg}~iy55$V_ z&yu`y^pDE`J>AV&*2s`)QCjU}9MEl*{`qu#yxxDes+Iqxy=uC|&qsnwkpFiyJTRSU zcztNXpE?Q2l#>w2SI*Fzw8IS?TE0iIe=zuujTSS=_=*#~>0o@-myV3DnjW!mYo9vk z|2^@t{a-lJilZn@KVT9BvxtLO0+RG-FSpd z?-hTxOb4+NLPQHS^v7K1A<*K|S~tWloL351z0m{+A+qlfJ72i>v+3VYz^f8VSi{gi zRB3pH{`@5AsBZGTC1z!7QpWGEJ#$ij8N!JjfZp?_R!)M@LZugY;lF}1p%d}w>|w5~ z$NH(Z|L5Ch+L-tMTZMsTaJ;=*Csc!7JqP6BS7d9&pa6 zLKd1X1OH@H=Z65PcfYr~{^@maDMJA@d*-6uI1lHl(r z`{;*UXgHiDV3^@T92p&aEB)X5c~FYJt?VsdR%>?|EH>fKEdHk6s_q99oq}ryguFe3 zNwoLPd^{nP{G!MMOxBAbVgLd^X!HMvD`$8@008U$p9vA|Y<}(UBmGM#Uqke@f%3~S z#QE^S1N>XIxbhtS&dr1vxaKP}IBBdBa1nc9>xbODNH8*byr!1Fa{GWsHK{c4Fy8=D znorgMVB`hFiFQR!696Gf<$i+o&k-3P1kKk_0&Mo=v}VxgQ}W^~aJ;+?u2Kea?kEQm z>d7S+z?#Y+4`C2ufzq#i-T;Ji8kGmGa1dPupmulR=KbKE^8k>Sop`9#1IGr;qhCNm z8eCbCXce*p4Q_s4+|NjMH>#7Rd$6IlU^o+l)t#M#bB824cknepFRmjFWD$3wf2=*C z?*^)<3+y^R0C}5gGN2$Abkwl40uYE@0CNl+D;}b_(AmM4xGW%A+OSJGC&aSLxRq87 zHwx`C+_ruU0&#>p$FBCU8fiaOS~Q0uX=Z_s;fAg2q12x1&K7fRE0ApHnVF zC*qbKxwnYSOW7st2@C_@x8S8rp9J@n$B62B>9V#II<=udrye*CxB&IF^Z1Kis1N7o_mzRO$>1G3*6p&a5GZ*!k4G1Q-uhIc6&X>A^nL*#5#9aq+c`Kp*mYni=)kq(9ybqJCkayJ&hyuzPva=K7sGx#(&YfJkJp;&{O}J=-X(T*4-lhztxZDC`4c;6fVqO@pj!f& zf{&UyIU>@Q!b4Fwp(E$jCLX1KX;Gi>kjLHnXzt5VD=gWN_LyUzlqi?+Xe)>r;Tw3E~2;GAfv~}q}MpB|D`T6 zs!}=#_Mi{N;N%I18W*sUuB1EptW3xP&*O->L=dcKr&0#CsvLd>R<1d63zWY>&|JpN zHP zJJ8$r`F18;;puaRT~riW_3DyE=CEo*ip~Mn zM+s4k)MgaaVBt;G(GX@%K*K%<;SFy{se?K~NVilg?2ZI>V>)ToC!5+l_U)+9`u4fX zG)ax33vZR&ZMbNEWvfOvrk0cYhoD zHKOK#IuE}=+9Wvxr!!sXw(3M$*M3@lu=)6{CJ!*auy^3|+vldy1e-e=X$$ZS7G^fnrC+==_j` z8H{{hBaf~%{V*q$TsYgBdbJZjULtdMg@6hqoJ_E4HcODG>EFcE`8Bbc&CgAf8yE0! zkhedAjM(=Pd|WSXfl5HZw0a`FnnZ=4svc|jUjK?hZ;svwUl zwzP+aWocXKAf3v-?X9mo0FbJPJi>^&Uli)R?r zk0okz{!pAmZ2c#D#G6rrfT6R(8Q>M`iM;F}Dra>Czq7b{1)u3Vrg$%cu$?bh-4f3=zNtb@u@CnwZD_;<7%6t@xo(|Xf#aP5x2~m(=U3VrI<(Sv z&W9W;c*}6}Rt%*OoYHdN{)p1Y27$l?9F}K7j!CUIPeeC5E+lCCG=}VAuisL-{Ijb< z@z&;z{BOxTpjMMx^&FIw#>Rf7EF>s)u2>0T50uG0`K5~$JdI~44{>7a;3JE9wtxwR z!_NzbuR>Dz5A+wRYw`6mIJOQkb$V|$<4}?Fimt&Bqb{BHS)6$=94T5A1>xoz`!f<9IO>7h^fCB3EN8g;~x@MlB zO`u+bQAmoRydym-9=RWvphmtcHW0T7tMFxY<$Kr+-->!aH*7~&>f$6bUYv8K4S!Hl z@SSCvj=qBUY#V>|O(Q0jYBG0`U9{>JusSNkYq|-ELq@@px$hV|yVBLD#5=lT?p*fa zvq{i8F#Mk6@Scefg>A|X;L(3wsQ*?3hO`@gCO=s}1>Mqv8TUHK^yzqosksM7KNJp1 z3Rj{hZO3y5Z|RbRUK3yHtfham@MvuT#9fRGG*)#FLz|T7GAa!Po zZKbnQoC_A%|5Ou)9OLcl4^e|o@wTi-#i!}s(4wZTxFLaY)+jQvB+^~Oyu1{oR40EL z>En5HqK`H{tZEf`({(gCK)hMCdYmDsjiM-w^4nG0W=Vz>y!}a2Gx{%N7KDQ5z6b#)@spQlj9dGTUT~mwMxCgX)JE_Ck3 z&&G3McP^_36NV$%SHlm8PjZ!}r=M|FQ6LzdrPxn(k%A+_kAH>rbmb9vW6Lj1k5+j- z1;(~^{fmz(u;xMi_aTAAT18!=`}$tB$e~(>8y|SP!elMU1=V8%Pq2_?rMzV@f!<&Y zzs*>cU+zMPv2yJ_`5``$qD}oF51m zjmd8kinv^2J@)RTr(%}6VpDgyzGJH;JEeteZkKz*p{U+MKG;Q|r4se?N zY7k*^(TGeb>SY*V5^wx+(eKW}?dLSR-}nkb@zTXlRIMuDEJO;J{CC)zIR;`-O${BF z+yB58xCAM(kD`ziTL!<@@CJ(1e@c15L;xbLVk;p7o~5`Gb|~@AXJ=kM2P&BQ{{UNn zr(A>UNiDMl5u@GzmiM~6fQztq*GDtBu6#C)Z(`VnqY69t%Ksnmlm8J=NHWKlxefF- z#Zd;Yh+M|?)&l_H6>-IF)Ad5DoP{15mEiavevtQ0td zaV-p6XAtj zFk2<4{(&ZlqbHb1@<%saT7KlV?)B(K1|`QfmeJtke@6xV2iI|!8=t!FU5NS5k29yz zzbo|)7&46C7%Cx5xqGtn;aIeYp3G1FzjKQC=*D<4PY9e~*vOsp|Ljw_vP{h#_W#8Y zhY>{)Ec@~lI~#I;`u@koht^})q54EgjS#hI*^yWKpWnKl2Xl|m==zLEeEs(i&hvc$ zm>b~-*QE;|dBkO9ANtP`f)#`WRuHyH`=clRYlQJ!LhwI}6i?s%&%57?Fp^LM6NZ2B zFc_&?`UJwhR^#ly>jM$i`Tyi2L*=nnyIfEIFJF%5VhIg~bis=zMITQk{KvKa|L3;; zYw#abkYC*%5wR$86}>m#80@n6%3|PsUqMj8OG`)JwK-}Yij=jvBhyY_vQGPRPrBiZ zw8)TR8kOL)aY(`od6S;AgD7KjFRR3(>EPc0f;@$0iv#Zii__O&h3Cti%7BTH$2b2e zYNBV^{eDSsp;YtT_mw&;FT66oNPo)dvlKSX&ydgK$0ewfK_}1?R|R!bDQxqPiXtwY z#6#gwN}S@P)PYOu3 z9DtqC0t~6prvpx(e#?lA(LE>f&%?GQ;YAhQLO2o&1nl3OS|wE92cG zr%M^%%p!3wC8u9?6>!pQ&5!IT;5OlkUVv5VsS1(gr>Ya}>8^nNyZ}H&`Su00^dS&$ z3Lp}puyM1)2Xc+>Pe8?vaoNv<_U2%N?e?ozru??n_C%Qq+|VKrJDEYMsmdw<{-iYr zfzG{OI{>J7v%zbCnsLnpfx0%03x%JiHFLmv3E@lUSL+!Jzpv;A%qt)ogZNi1TL<(ofcT5ZVa&poCDsR@scepMS zn2=`FTv=}`f{$C2PCP9B_EU)r_z z)o*_NJD7U*_ev;@L-89VEH30b_2YV<(EnC32I-xuH4re<)|P815554s*qefNP(b$> zK=ef!#ub#r+5OV))(5ee?^=iLrOC$j$N-0;V?8kAVD#kr`V)J&DmEUY83=(*_!r6 zuIL_m)@*Fzu1=bdkGYC+@!8C}VjpyzsmRi;9qljV9(z04Jm2^JlC;yT*X-+f-|zVW zHDQa26SF02@&SX}XMe8@P6YyV_ygFm`T)ajMYC0mMS*2-#4@0!k6M1p!y8 zSAdjUlCuGjq`}nJ!VcAYiH??FGhn{-3-J6LSedecHzfRZVI>tHITbxQOE09hmX>kr zzeDGgXch=GuT<>q#>YGq((m+j(k$(w?`(hS?^58`{Tl5B zFjs2uU|*H}yPM1KsIw0l=R9I}d&MnTf5hae4nThXvvhr6Te;OZEQ& z#sCJC`#5x#COSH*hUTp8Jl{k5>_^bkrvhg_ilfMnAIqTxP?21SXUEm5k5Lb`;~fX{ zI$Wk{m_^!l(v<>yW*=4s0zvf$j9uFza%&drpnl7I=bcnrx`K>Nb>XX#>+MS)BUFEa zMxP-;;(uST>k37H5JvhJD>u1lC%_?q0V;RVxKBEQ@55H zpzm8C8=e3RJ8mM*Nr9&#jifgP>bQtD}Jt4cp?XADWuJlPw4OkXgH>)eJ4~22)>>-PN*WUv< zCAVY3soC1}TzwbR(%zATulxXsLHksFP}})XPmB9@1Lbd5zDr(99`{w--8q`k@>}O- z5eW(P)G@r;sgP>LN`Oo_>-KHA(pQWmzWY>z*5~X z*((^U!LCXgncA~1?g>$}rwe5I^@neBv9(f??>THcTG~UV*Vs#kKMNM@sRx?M%TlY( zQHk3bDy^KVN-88dExJfjzYD+XX!ZVGCv~)br6=3fu+i)ZVQLfeB48#Tcz*Q}#VFk_ zBtO2qV5UJzy(=6#_P^RB$gD`SNG}D&DBJWX*?EynqS)d^NwM()>7QKXQ?k7U|5pG3 z3I6sA04X!UM=WGi0!j0*1kwHGXMkt`_?1_FdXwrWpL~+rh~TW*pQa|-oKg)(fE#Nd z)|mD&&85X0D+6(JYQlsGc`kMZ(QQ7^kBct4D9?R?n6)Bc^*tkC?bm0rDp@bVy|oZB zt19Oh#w@OFR1Z`SR1XZ|fe{7Kc?bg@EDUA~I|&RoO{EEJqD}RxtFB7Q&0&wiu+bP! zfC{hzNFAudl!jqP{oW?t2BI;_z;L-ylPAoaIWvQ3%(IOd5W|>-0hxd=W=m=qh?bs? zDi~7MH{Eno1`~7U%xMMD+Qb4%t7F)FM!OiWE+E>0MXBS`N2O2slJNuGbkpx-BbY&c z#wH6+N>cUIDMA325~Z+imT~^Cq8ST|so44}fTMQ6L`XJI6T# z_ZQT+^_g5Nq+`}qSOw98tjKz8^+5GN^}tDb;7qH|5fTlO0uqa+kwUCpyEe0mKsH9S zeE9`d8oLWP(enb!0FwC`+;Zxk?aG2jI@80l+&Kg7=wh70&1IPlLbnEK9Yt2 ze)PeM7A@)oqLsP3^Ugc7J{dD_yd)PSD_uae&uRm3Y1Y8TFas#`qBNfV4Td^Nt05+R9un%;e42UMj z-~*aptTzLo0hul!I<%RXFa6%pu&sKYwboyBRbdM&82IP(1-y9xgAAR&umW2$n*7qoU)eJ*!5+FLXy~Y*U z9YG48l_0v`{Okpy0bG)*E+AU4>l^_&kc>uHWJr3+Ip~u;;{dq69{_k+5J9lumb4@S zKT;n7u60&|=r$h!(apop{6bzOTh*`6A?IkG{iq;%6joC0L-j!Qz**ygQ3uh39xU|W zY-H*rOXWH|ht&a^fj}%GHc^UJ`rbx{4ORcxW)EaZr`pIcXT6x{t{__a+9vLxqYYo1 zQm^D10RTuHZ0NuwEjMsTuNyk#I4sx3L<6q@Fm0lxCK~%Jn0V)%cQT8sDn`_N1{MIP zes!V&2&_9TH=VGm9zM&T(%lYOF@6C8fK(L_R0ILC-4CNpdeIL=mt&DmmuiMgj^(QvGiGGgpIj$ij3b?{vIqZ=bo64P-A9JPTrx(7A>H@G4?jEs zth;<+oDJ1E0z_l#1wSg($m$3lLpo7{=zjB41_VGYndp8XIx;mRAoTMxQs%ujJYxZ+ zLwkU=IodSY6w(dz&Y@Rtm=r`=ll0OKm^#2`o%;SD2g0b+Czl|)%?F{)rW-fMotvQ5 zk6_jJQCpwKI?K@fF6cEcYjut;tF>NJJy1PRJ_Yq4fFErAxCMgaH+#9X4wKS4z3x8w)XRBY9YA2zT}&e0r;z#>YC0VVz}?QJ}OEr1qpEbQVd015C3 ze`r%66Nfe(I0I-(IfuYTKgzjO9vbVg%$CIS;?USf3xQLF^f z{pP3O5{NDj5r&)tn6-wQhAp;!J!8)j0KVgnJ5IkNmtq)_p4NjE^AWfx+ z@w`UcJ*1VJx@F@aaZ#DT{iVTKR0Fa3cQyCWNB+U+V(Obl9^o(~29d<0SAY85uQ?W) zX#Nq*2M?rtpHw- zAn24mieR*CYtTOXY}us2&Ipj3|iC-5anvM}4T(1EcDJv#B~q zuc{ZL`do(HuM$KzD}Thj0z|9&a;mL@=u_?SdcW#{>VY%b1ApJWI{ot8Kct^co;iBs zJZDx)KmUn6tyVoyJ@EhYKro`^tKA2^TNeO1Xq%_o^<8{WLG^HDK} z?)=c@_A0m7wVz}Z^yD43!1+s&X22Z|!41tm-gK&Fxe5dk5&$G*)>f-VAvr__>auw+H?W XqdteKN}D!-fZ{qT;-f4< zKxN1{@CX8eGLBmjPy}R%ctqf^1qEaX-(B>0-h+Pc@B4oAX*2G5-PiU1U)OctJnf^1 zE8E}HzM-MvO44BuG&HnsZfIyJw`&WYO#krgT=3JP6d=%s1?z9w-_X!!fPyiKROQoR zvZ2vv`|YaHpiAXSN~6)ya9^{<68)__CnV+H8#4FRfd=mKaPhRP~d|pgOk}+3WX*A$wqouDA0*~)jV1F{OW~ueJ$V_&GB3@{XcrEPd%w{ z!f(@#SS3eBbb1^WyR1RBDlKP2zfHH2ESAzhRu?7r@4oAmm0CXe+tWm{l8%Fc)Su>o zcuK!L22(OL8Zod2i8Ln^ISIT@|1M^S2e*gqh|g%K83JWWFQFRwTLUr&=5(<$9#l!H zxnlOWcFf^wEDL#Zgioh0B=T?#_lKy2!6(NoifYv&F$WV4R^pj-oK`bYCmPQye5Hy8 zNOLB_kX6t-nwD&W0|)_C!5?P#!K57qR}M&uJIofeqQ{bQ6fpuRB@opL8!XW>1H^z^ zYtAZ_l|7k|MrK$l#iV$b2L~@ycOhOwYocCoSR=TQuhMFTuu66*6=MBT#fdlw&Rb-Q zSl$_r$eKhT8M4OM=zwGqK=qQZCCKGxLV;F`G<6*bkL7z&M6akB+K%0c6iWsWS zK#a8@e1H>-m{fC-vLL0S(If)d{dg`@E;;K(Ybh4WJBhG0Ab2=G?_$j+$OU^eH}4AT ziWR%nZY03OLLBMA&1yu)M^ipese~CR!l-#E6t$+b8s^4)pso7qB8S-k!aCdBCN?67Dz>^E+U@ExZ2#G8vu&6IZM)vzqqG!KhGq{%20SXAdRXI##zkAgXkrhwL|-kj>n zRGd~f#2dWMl58sHy(YBm0ZsW3LWTh1;5j{a(I~j2RV?Y8lxC@81)cSH64|Ui=5)Z4 zia30jn_y#D&76u#8nI7e3ANT1}VT)~vpB+-f>6nu5tm%aIdsuWz9I~tY}8YJb)1rElTprIJA zsz%&gVR9~$m&4(N7Bi+@dMA^IAfF!9G|k2YZKYTWfE1Glih)<7Ok9T>@<})6D>1>8 zF&t-n;RM1ZN~sF(qNz+=ud8ZEAe5>U+@%Uz@j!+$AIsP1q7`GJdC1HWPBIoVDh?~) zFFeDk99obpX-V%1OM1;6aQLdWiXji)VR{dhODG<54sZ`DF=WupAoUlrA{cmx)0Khs zV=`YO3s#>jG9oUT7y!Km$m{gQgc>EsONwI4h`{F1ECMQqVwoISHkP!kF{3(YyI96$ z$%9ZKOC7BQzb{zlK{9G51YN4=tiiAhm=@Tnx0v=w;Z!0}RBI-ishlf^Dn@%o3T4!S zKV5JXy*?u+TfHdcch`X*e2f}R2>A?cE#^2j2(xB3>IB6l2)Dk>nPHdFSqwl1PAw!Yn7tSwFp2i0 z1QW$65kQQC5IDG$_tDGfF2&pTB~7| z%n-(~!^hKt2wCIya4MOwxNEUwHeZ2tsVEeV;U==A7I7ou(*SMTRG+Tq)OmCI0I9~a zy1au<5%#F!j}%=}J|X%HUIJw+E`yZf!fKf_Sn)7jVanbp?#3c2S#wHI0;{Vi7S4 zW&SUIO){JD2ICY)X9*({^oFy!AaEz@j?sVonnnz2gt0~x`u)Oc`pv;HM)Qdn1RPb$P!}%vlznm_MSG~MA(n7rv@=}D zsD=<+&H6k>7R`tWA7fT@46pjVoE3>zJhI@mBL0*O3j2(-SWd#Jq?Dx6wU{T94BB&1 zFa(s1$8v^pDPv%~l*Ql-422CTlRFMuIIFLg0QHHY(Nne-n`2JBl$7ZrgTja-c&B9WYI<&$~TN2g+XD_ATE2^c)2B=9N*NP&JfEW(1%bL9R6$|?xwsJ(i_zWEt)6DC*2l%j$&XboW(7)rRH+l=osnt<|HowKvDH( z1Bh%=6Pi^MDV0ECb^A32bWoYLK^{XA$uc=ZG2!R2k}jqX7YpG`DO9&O)Ml?#3|0=M zgGRGX4@|lyMU0?Q zSyLizlhMYTNG})k@fz(+ka8x&S>Qy52NsKaHO0=x6D*Rk5p`2S2@-OYO^G~sN3&J% z&C?7b8W0%*KS8l>Du$9ilT|1t9LFU-BS!Tgy-;5)03(XU8i%W1UzY!WA`Tm{mcb@_ zAG;0>cnyLnAuGUC|C>1ET%TAuEh=e1Q#v)R0AcDX>7tU()hAGowJm@`LU73hoFg!b zAZ>9+oH51VjEjDIIl;2=e8dtGePktd>FSm|spy z!J@aGUco=J^!XX7RlSCaL$ zFhC=;Qq)nn$N*G(V@JJ{F-0s| zLaT}n35@}jL@^)XT3rR(--W5Qw-Gn@wmR`Wo&irPu3g} zUM)qe6jDMnoQLsN9l=aoaQjuR=*VPHk#Loa0i{Ii%UVPv95SwOARkqCCsJk$L+Wgj z-XBGSH9kjsjS*Xvv-vV9)C}ejqJ4H#Qqjo{Jt&<;sye#^Qp?_g)!{U!MG=BCHk(_+ zAU9I7$VMg(Trg0L_3);Qy_AjOGF26DG|NgTABrFu1EotC^>DBvf#^_5lcnkO>;yU-o#5GOsH|0tTk!&V;-Gc_QuH= znUoSB@PO-h#>IVL=%}2iq)PELoAugbwrbF-unZ=P$zl*s1@tbLDHQB1N}5Af!bBA20{_aZ;Ki%LC`pBk2zelDZ~@zq$s+GP?3@?0ZP#e&KPZ%0z@hd zX|aq0*axkT8XaDuB*v@^sgETY$Wn0R+_?~nslYErNgRf?tQ?OsAq9r4_DTT>r|p
AX~X~aWX5BftrM+_65SoQ8$sZ z|4uq31M77=&zIXJ`MxaT-0nLAjBj9U&f<6%N3AVdbB){bfx!qVpc4OWE?4(-yAIAlKJy6|6)d%=Meu3ISKZp#5?LbUi z#~GY!ZRTX0e8~CtNid@p#+eeW#SVBf_t?8sezSnVHlJ@(@=I@&syxu|PFo$DK)<3KmSwr#-SG@wV(knQ5tUJ10lR{`GO zEeAdz$mz6WYX9MsBl63~>5~>S0Awf#UHDZBpUqkAGDpP#gFFHHqtIz}Y%orWpjIY&XXw4@hVHH|+N<1AC z)6gQYV3CVEcQd>lK%J;iXjY3XEfCD1edz+TgcQ@k^v{`u{^bw=A@l<(&i0f4KoAgC zOxd&XI#o)I`MIBvb40=g0Az8|oMZY&zk!+n1|=b6Jk>;}Ae-0NkEL@Qa4M{-2A=+X z3b4Uj-+9w(L6$kA1M%J~5`lm*etZ;`Gqi6UKfn7|z^>p)h@%5A;N`!VLi+JP7?}66 zy3dc?_GZo}Eb{Ju6*cH3FvhS4ike`!z)1j5Q@O7r%ooTn40K-f`c^(?d&>HhQz8Qs zbS8Ao!k-MGEwlgGGBl|p^78N(_Xksy}dwFCois&H-Gjf94y-X`ul-?wH;4@_`G< z<@1zJW;Az99uodFiy%1&4}VpLR7}qFPm&(5&(?-{Ou|~S?!)viyLO%nz*RvR0pR(_ zC_~)I1Mm>$-hymyjUix&;;VW9ZC+1v`pe%Fr_@AnmNfrTL=_<)1(4J%Z(TfB2o;T` z9uZ+=yt{b!snfZ3{gjnVkcpiUGVx= zinJB}`_$Vi;M@Uy2im+74-~!(yMck>WEh_V0pAxJc<@)Bnrpyh<*=Z4N=(g0&v-Yx(R)T9Jt`+ z>+L`Ra&sol0`LY_nm=2HQUj)?(>PPjnw&lOB1lok=e194Y4hxIS-BG+jblAt93TAR zK9ULuDi;<~s(NpvhNJ-`p%h8O&q|gcJm-FGjYZ4h$Y)*f`H0S!M+EqVOok)p} z^$h}9f8!fl^9RKK%;Gw`it1Z-RSqpL>}qU{M9dH@BDP~%=pu!vvsn|i7q4q1ze9#pfH?_Yt=3IGO0D)jzY54(yk6)Jo< zN6b3$IUkg6#Qi^a-Iq1RJEe4o2P;Kt%PAB*oYB_ct~fC$OL{{#qL%3mAb@ zjaPdDD%1e|Pqz6^1;7FtrSIQ>Ci__CJr}&sm6S7K8~(}=s8~o!btIb8JJ+X+~cti z|8k)Xmw23H!S3!r=?xZu-Ub?8icaztEVDUlxfgIyT}5f+#XGxmEB%aQh(HEwzdzJ} zpskCF*XrGY8=PwVXr0Mr&Rlr{aiY^JWwx(ao5twv_b1ZP7bh67o$>l%Kct-HNr`UV z0LdI?>wdljIO^3K1o?-D)wK`}(FL3|LH;+X{uDmot}QlLE}?jV6Rg>{MQ{+)sgS~Y z^M;X|8b6F&1pzj>^B;aR>1}~m;lTP1YZLK!Wn~KmU|@V_0xR}|D+G)IpvmU;l3{_=7z+@03{upN?3^V5prm3V>kW^DSJ~;CO2w zGx`s6`VPn)XlZ|2t>~yS3BR=ppAE?9 zktg>J*bc{fPWifv=beYM@|~01ztbEP*znHDA+4_b;lNeH^%rk0yn+v2q+rx!Ujui7+U7R6Lv!&siPKr`Va00`Xb_t{ zPc-WKY2v+y7yA1%#}{T*#!hmDYv1dAyVs2Q;)8oJr3G9|dT+%?2N8U|Yjs?rG0I__ zihet>dUiEh4&&>&AHi7emFrpOAgx#(VkgX>u+!<*g=#Gz23=|9IS}|9!~tHgXl2Up z(U7s02F*3F1Zj?#p>#dM5go_vAP_J>dw;WeW~i1qpHh_{tYt^ux?BQEIUNmk!QhrC zMEFf(r9W})-!;L#8h5xgh>2XzfyZbgIv;bqy`U%qCcDN@5}ENZH&lJs0QmORU= zp#sJyh`M2yuhU@gc>fQgtf~c(LCLZmli@wE)a6-Md!ij^Ghm`##gJVn;`T~Ny4%>< zxL7x%;HQ`H^=NXxU*8=m%9IUwr#Go=_}SliMZ?aM3vh-NIlT1oNUo|#lrRMqlNwlT-2Q)k0B17=d|IQdAKW3slhT;MwG9OH`cI1%aM64^ zhX+Wf)!rXPb0WZ3trdLrUCfitOt9p!8(omFeLiea=q$XV5Ez4yCH~LwBd8nA`Eudw zB_q-*&np80xkmp4DJVw(w>eQ2Pe=R16v(5TcVjIeRFd_l5blkHHjzQ*HjOVfWAB*7 z3T9hh>NcpNHEE@s4*TvvjIZa21%nn>fN0dKJ3fN35U8la9t?m|BnhZ>fekxOYwpf~ z3lfL*kdB#cp)!nzswzuVLC z*qo}c<0F)rG!APHoA8(;tGZT~+l%u{O%%0HoNOOTz8@GPW)V*8D=e4zQ-4LqG*`Q- zbwGmS-f0l2C(VRzXwbI!i#6}b!f10lQjIoqB*5&O{ak3?P-4Ml9oO8~a&7=leHzdo zJQt(`B@*O->3@YFzQ;t;cIm_Uq2(RuCdy3ROX7aKXnDN(WGY`=v0D0 z9_0x(ZO@H+c@q$jO9?2)lz@-*d~k(;4`escc`aD5P>+Bq$-TYetnx9vr0)sODmiK5 zf!aaoEy!%4pm)eT(<~Lre9nZOw)wW$&Lfsulm(a;F5C#%ky$TBQr#~y`uTqmk7s<^ zW6@`&RIvW?W71$CHQ8N*D^$9fB6sp2mKn@z6)E>X7gQ`DhqmjF=LIH3FexRqs6Lr- z*yiNyE4e&cG6Rr_nVY{>WS0Yai06D=a);o!K?h!cs31MHNiJeH@qp0G&k#%jJI*8Z zO$Z$rhfUWpYN=7D#N?lxr8-Q0WqSuRuXj{rnoBE8VsE_vc?X=k4ar6G1-*2;!pBg< zmCJ1AAS&!-AI}8d87&)Mzr1cXx6)bkhb5Z_d$~% zZ_4e$wQ&C?wHO)ihl%LMy3%FZG zUiy{M{L$OpW%~Z=x=zE#77v20`}0c}_>C&xmBZO=V}!>tOinBUmV-$MJo|Q z<4U9x`W`RVqtQFdRu&N~degBz&pUDl}H&l%Q}YFn}Yr9WFdv$|x#5a2N+ z5P&#-bS1n_1?!+OmX+sl{bq6)^csE`wana7Q1)t%#sDJHow+}4-Ah}WRMtm98E~bt|V*s^tI8; zU|u8bkOlQ-Q5)qrwF-V2I#gED^MxWW_d9y=1bmcF6q31tXMMskWLg#5Jzrwk$JVb+ zszY=I;}>VsX?^`5Qd2*s`(CMKm3;v&A z@9qKLt)$F~p?0ega5ZU>nZ}|;XvIPNFRPRCmW|!fR4bBW$t)6s!toez>qS83NACB; zYNC{|xX#RlM3vzHv!=`8wY8tRyNBOX1t{d!Iap7-ODaz{nxYk9MMfDFuj}g`juPe< zZ``@~=c;BGumn!JVetRG+=Q z$ulScV|L1alEr5!H8g4gK5Bh8rGNF@dVQPPl+2?{@Mqm9JEvRSpd-r{G|MORt>3b3 z>W~&bo_fvR-Z-}X*p_~JgL69U?NoIJxMtRW_*ZhS^zG&AwrlGQX-J7znRblLgE>av z+DDYEsJO=Xv2l)nbg8;*fEv>}Vl4AH%zoYQfXT>IY_02cZ~uCbRoT~6iHzAxl-W!b z*}u#9JDFYNzo$|A7#P;TzNgWAiUT&Oj9RcR#dzNBpxlzQ^JI8C@utll=@ zbNTmeEBv?vwIyCZ;SW0^!_#>%PE{W*?4FWs_CVOL9X{&%x04^C}rSYU1j|_ALQDppGPy8)K)Raq}DHGZwhO+JU zL3QQ0*Ble&AucUI*kuj3;?d64&ThXxq(2glO6I35=1az+rUOKrcWflLx^Cr#nEWM) zh3_kQe&wf@{nUw@&|c2-^BmWVG@Ul;6mv{ItOs{l<~R%0UjxwFAb{{viy~L7YN0^$ z&9_A|C(6`Q=ntkWlJ!x&j7Wz0m<`I*N@qHDKdX4e*Hk*m(b$Iu9f=u<#H2YE*UXhh2@B^7Q!?^V|TW z0W*34Ct^}h1Ri~2iCosR6&+?p!t(U7Q$~4Tj}ZgiwVMg0s+VPaJl;Bx=&Wqd%-9U= zzUVkvw+HiWOtM?o0Y9RSe4tX;sask~kNn#dAR|=3S8){uMRec})B-t6ubcRKXQWcD z+VBICWz4mwi+c$xp5O*QFvahX#z)ouET7!+s@J)!hkS)E1FYZkkJ(SB+xa<=^zLa! zbq^X^@Y{CM^Tz&CMU*+zn9M#n`|L2&@9?GY)V>reTsGg4!KUjHXTx~eY;PrLEPs%S zbgJxU2QzM|{vio)r+fQ!{|sjb232~|hLPfc+TOKk$Lc-CyDcftkqMrLJGx11ATW-O z;is)H&40njD%x3$3<`)%N=X_en?Cl#crC}=;xV+vxn~Lx^KT3oID3Lb8FBePMN%m1 z-shD~+pD449Q|%2BknXhVCcFoGN&ewquu(K-eUyrs1cJ-3UdD~fx0SKnHgL*bs>iY z-6_`fNdaZ zivu=Z0V4v?wKitSUkf}~D*Hl8tlD);2=hG)V;V0Q4y#lO4!uaR+kYBfU;UUZb3s2Y zf=`gw9#qV5=O}f)mgGOw#bR}r zckh6B&nwdl=+M?HuT&SoJchuy$o6rm>W)3a4bcD&T*_hYAWl8++Q-`BlFEH&KN*l^ zeEK)b;K2(@$E@b6%N`CQ&3BKpI?Ot0oBgiQ1zjT_ZDa&=H66f*{&nGy=t2xZD+F)uxpkF%RI_W;DdFZW!vFzDdTz*8rP)SyOAh)_Zr|7 zch1l;=cDM&4~(##`cd69Wk>aE#Ycq>Z28~bfs6LVaNz=JKT@tYyYvYJXn4(2_U}m4 z!>xNl`&F`Z%3tB-dGz|6>xlqkx$%4)VP82JdL$#5gp{RgW7gfg_LLM?qo&S>mVFwe z2qK`oLXtUFPbH;21XYI0ainMa$HesNr4S%xaYYx0;WYRs;vvTVMkqO@}~ylUkA z%ES1>1V9u_h8Wj=!dA+e3jt#rHCFej5c?)9{i{u-36Cp{x5D?^xHk8WjNm_}{Np#S zVUAokQ@lVSfFaWVb4(3jH7PxU&u&l_(v4e?7t`G`$u~U?A1d?XIULE_sCi#_21(j! zEiq<+Gf&Yt{w#}gjro8&Dy^db%zBJxUzVP^er$9Q=3rw!rn`%JVCfk5xk*>=1wXd~ zqI~f2-S%nGEPOqxbsx;N<&mP!mFj@$*H!ndk?J0-_&8AFq#h^EH`#DsSwh{+cNQx6`-3r)Pb@V13$F)-A}TC+E~H&7z$mSWdq zJo;mXu1}ZANQiW37|vXnIXU#H^z^Z|v9|Ha^{hiQqHoK5b<}qs%1X4_&&*8(406Ip zZ}^F)8C|EhaOK#?h0vH|~Jm*6+}j(6O=#x84bT%J9a0$FYmLnXg%9 zr~Gg%=ylI&+uvZi{g+P3Ea?M}Y8;95@Xvt4+S=p()~`k8Nyi@FwW%ZwEV&>4Z0T*% zI9YZf$X~1t_&t^r4(6gOgnGknCWAD5aCC(i$z$Na-sQp+^Cws75H|06(VtEuh z;3(B9E`_KF3@x10l{LO2^qADuPtn>h+_+wi!Q+m(5y=62QfB38o z_)7*333Z;2*NQ_cd{OjC$<^RGZ>p%PBm7L244Swzy?)GQ$I*^?S*9}TD^`>lsAyK{ z2Z^x<+h2GIO@FX%nJ%d-Id#suBq<}!VjFU=6Xq|^WrSinrjV64$75T~Uug%5%nD0s z-x`pjJjPB5R>U{`9^sIhUhQ7Zs33WwSNn*gS@~pSovf0^=V^~VNR~5CEF5xau>lF= zN6>9t=;bO`?a4$OJB>d1;)fXed zGU4fYR(km!T6W{N2ELx_L{sgUQUHt~f3H`F^NgUzD!$&AfEk{WSspV<0sL4NIY-iW zxJ*<|KMz;qQGj2zFdCNA%b&pHaU7jOxDeM$^^Cgk_xQY*ExOrkK=q>e7!td7_L*Mo z-uueBj9+Qb%>ygC5484SOs4cJXYy%`$fj$RnCQjte-yGgS$|8Ys7zVWvzLU&dpJ^; zF&GBg?2gTh@~8|Er1_TMyvz}HY0Hw@Mntb$=IogXBBsg98kc`uZq7ZvSXxCpZbW|M zf|&_e9t|R-5WW5DjbOP#n#rx{>~P;ZDgu=T?Nsxm247mHk6u^i+U0n_C@bmDj5<_| zD4W#jFqn&=kMhz|+^T4RZcJjs1U3jNE}4snKmPD+rXy_xXhz!YtKqjfape_uyy!-u zvJUR1QDt5uXNEJWpX;wn`#->RMA$1ez&PsI(LlUP{YwjKeIk#ax$&Mj>P&t?Z%pjwkdyI8g8Y#TMoLkKyun|49#0$m zrUb!WLpw7g?8O{VsO6{vH~T8B4GJ-W{OSN*_kpJDo@SSUIyqhaM=D{{>z$q1)k)XZ7PMf?TLwJ-%!>W+r7mDKp?sxK7Vd3P#Zo%#$(1M~e(b@610V|l^#7Jx~-sHX@u0m{Gb$Yy?TzjS!T^DZ*gQP+{!+1hL;fL-kkY#C1TxY zab~(@Y$`Ks!r3=RE~d^rmvNJaJI8b+AkVi}%}(=@5cX-fzRs)Kk%2eJZlN1%E0}## z>SHJxGVVKWOJrsv(&s(eyWU=E8TK)DJ+f@|TwyMuS)U;2&_k*4sQ6yrnM0&Z+Egk_ zbyda}4fUAg{2Dhh`aMKvX*IuNd2;7Qm{WDv^USej=pK91&{dCh2lL!psa=aP0c-=x zv6-qGm97+xkPIW3MnL%+`?clPn+%*CLJp%8qZavAcAfK^mrVjfu*TYWg{K+4kBU3A zOXouG{zZ-HC;l?D${FonTFDZEDY~th;c2`oFT0Q2w(oJ##$V^{bMRYPz6J-CHx-;W zCrmi|xamRS5aO7wJz4H?gKi0Pnqybv#|Ecysl;^$H2&HF8OoF=%drk$FiOqPTz!VGc> zn^*Gd6I^b|I|P3fyD+_8(v~&z^HW|z%O{@l6Bp$oJl>mrU8<)Hr%%0esZ|0|&vFZA zFQ42gQ>cy*(Qy&Cef7~QGG{vY5!WL+%#h1|BXaGn>cGb|6k((xDvHNz8JtWOMUP){ zdv=4?<1WKjg$ht)wB|1iTPpr3%%YxlI=AY3IHUhtTe5tKwU4yoV#Q^+Pp4u2Pk;6& z#WZ^c`nztPsOy3I1Ba1^QD*FU?$je`xgio z>woAWNj;C(XHurom-bZs9NgH%%=odL!~)blYcrlJ`H^rJ{rt!mV59Yy+I#Vj6t(Qr zEX;ps!pnGp=~q{tnCOo_Uy2J31pSUT#!A6obs|@LKgi39>V>-+6rm@w7**pkYIGTm zOcbrITxkulBF(WiYoYos6=%>rkyfWx)NzuJHMd=EWhMw4;!(=l^CP3BhBesgyxZj0 zUX+=Gzy9Xvv6fel3T-asG-7=*?dx)W_94WL`sso+!azvXgKQ(LgcbA0Le0!F#m!sC z?eC_)toNn}npS_qWX3vlDgZ+bkjTAX$TbA7=FvZod}yx9adGDJBNyWLQGtbG{1U;D zk8d+3oUoKpzM-Y*rQ$9mk zDaTudiLGg9jksKd3Gx=5JP}5twvjp&%@70UDekdW&POM3W_fTQDY@d*f%4YP%WcMS znBEWYe~(q`PGO}B))Rr}pVvm`j(-p5_NkmkGA&y^mNHHVL5VuPMXV({#W)^~&_w@q z(Q&v5i>u=*plHFrJXujb2S^@N=b`ctk%3J=H;u?o<-{t8f@$}@x*VZ)!sgT~V~P-( zC=z%CrKcK%>R|Y23cEjW`(Pe@kt?F1WJD{<^|SvH&CKe`N&5xOa8*rwAZx}F$mGad zbr;+?#@&-rGF?|i#)Zu*sFYY)>{+ViHi=0liq4D?mJ@wgT4J+7WkB7K)?zyGSSD_* zHvQ4l73L}H#gS1@@}xac4bDdQtbzA7#qO1%r_q_Lc6po!hr6iS+Ja;9G_`UuV**9Z zx<&i-Rl|VAKF)9}CkGAZNTbWCZcID`J=BDxUt3;gm$1H{qgg?2V@g+{M&8dRHTmxM zqpR*YcN%DA@R$gG{pF0cTWeZ(;}+sIRrtP2xN~zibGLeUw5!TkRn0yCkn!Nds#~>* z^BA(cFK&>lFaS7(>FwC5>%FiOBXXYYVb-fLn-h~ZIUBU$B;R?1qn+hE#+$Bfmd&2l zb#E*^UeXy|G#gf7;d0!pOvBTtea6+eU|HC(#h`s8?cziM6}^F~5W^!HObRn&?v_8BvEfn;Hwa`8 zjy%(XU^c-p{=BuCJZf4I>(?t!y*2OVV7p9J@|{0RlU~xv66*7utTZ{wa-(UVc>@NWayZK zCWufOU1BVKL6RyWVV(p^H^F;1IKT~gFw^^D?^zY7w$?fJJLXHO$pbYu!6dcQ3rkuoF-d_K zIe^OOf`nPrj6}sNPzf|bDYj`M_3=|US=!i$Wvlygdsrr`U_qts5xrR5{O+O;%tA~C z-S6l6@^Rv$vre1)K}p(J_9GTfu|BmV@QHQxXR2OD7!2A6;`x4LfiQ7R;sqxFUb9EE zDdqzQ`L9(4P;cOv#vo@Hfbw}tUP5lPtE0%(K^ZDB^;Hv@b z+q*CA$|lSfEzXzAZw(f09Jxb#wi!}2ouxBfQe!mjtJDDMRUj_42pAB=HIPw8&7gD0 zKt^bxoIorN!42ZqmW%NdHkW@4cs-eT?{m+sbw`;M2#z&FXS00ento&0dmgNH=z_55 zdWv0j07V9l`1&*d<0s1}hYWqh(te6M@;F)9;mRLij;DfhRE7)CA=k)3ml{L>G`O>Gj(8*_Rnj%+6dAq)rH;5?-U7d|xDHPi*i?bG=tb5u? z5Oz#2TJ90&)XDb3Id)mLKJ8pLu@vYe!1t$SQUeWUZYgrqx(?DEk@P!K0(`Nv3(W0I zpJv4_C6M_7qJk!f}IF%6z_~Eo*C&I_II6WiGZWu+Lg0{Bi72f$BLi;}$did19lD+Abj&<_M;o11~Wlp0w>Al#mOAS7(^mgacGY}JvXm2s0N zLr6+fA`|zA;zQh>X<+L0iemeCuY>Up#vcv^%4+oH^b-)r_Kw8uIh~>MXIraCmd8>_ z!=dX<`X9i&noh^T6CVsH$g6lAE{0%kqj$AL=7?tEV~I+nTx{U)pg-*H`w~f43pen&yb3+=)NTl z8BTw^Kg90NaR@_t0?bcXGX4O0ge;_l_%dy^$*+Q`Ot6q|EmM~ntR2;8cTbxNj+Z?n zaOF$i)`lY?Qw0K{s&H!`6@MgnXLhsQFgX~D-5_Y}G+^#K$t`K`=(YRq3Cek%0;3!V zKUefpk`%}u3yjkfQ>C&iOF!n4rEJ{7}Nk6#y=Id_(p+6+fub zs}>w=_n92{?buDT#x8(g0wBlZitkJSdCm{vL9YO&KRzgDN5%ghl8OSvzP@KFU(gYt z1Yqj1ZD8^p>54g~Po`=Hnz#cPr{};BLjD2+R@o?y9U}V)wj&B&WuJQv(9JKqAq-4< zhU_kdhF;K5bJ0`URKvuh%@BMJ;Ih-SqMW}*?T3ylvx~+Dzk{jB&Ga_g0hDQyIiOu(WfesQ1_`(MfbU9M3D^LCO+HKB5l9Ca#Q??2$gy&3HYdh|; z*1XKKrz(7Z=Cs{T0uLLf90P^&&q}k^0QFC50=0v=24Q2(kvQ?lv<%)dHCUc`f`8ze z(5Qhbx7%4r1KfA1|RIruuuSkY^z&H_ID?lTj7?^ESF|-UezcYfcGUWek=B3#oWD3Ppy%)aEJqZyZu8S7Z)t4vQjizAZ0d)QXP)Qx( zMmkh%O%PJYP|CPQTp|Rqj>s29bxr|Lir$1{V|E} z%q3ATb9-+v+hJU%#i(Chxk2Utimqn(&o`80Js4D7_gcQ^C!-eEGmL!KAyt(DKwl=404 z=`|sgCxu#heKfjT-vb>U%rP-Z*T2#)XzFd{r62(7<)i_s$Sh3eDBl=2v<{ndg`tz`hN>D-wTC1t-g}~pv@9d*uVDJ#pI1l#$*HMD!pnpd3-2DP zcx{8J9J&@U$?7tXpOwoHx(ZhyU^hmdtK;_J`Oj=xpaA=v)AVRDrXi zvwR}Wz1*w%UMFqtp<8hkOOr-7WIR<@r?!uu8qScNHq7_n;JJ2cAd*@u~qsyUel9xpyg`u94j-a+Q#U)da)mq z`u5qNRq)oa1rBU$MBG8lr=^e5+`zA>DCY$fAlM#n{~DmY0%6#JS}h~03N7rhJx7I7 zxStKW-izmes^xASR`ilUJ()n@`*TpE`#!^Ke_ntD5no{C3!bs%vk2l77jVA=^#JeqBs z%#Dz0z5l=;qqE-Ttu_@%m#;sNyPb^k!XK=t%J{+<6&V$OCN#BuoO`V%=ty#Uhh(%4 zO5W~mCaFP+@Pxr{f*xkKinf+=ZSPY0K{Z`qVa9fYz76RhWk+s>)!>E?padJe5_r&W zw4@0Rw5FbChDZA`hCh~M3{IL{1H5O)r~?Gpea{iHN5mrrM~374onW&|8AYqRR^1B# zN>?X9%J8~<85w93mL5n?fQ&o(TiIYy3C<3~5`tuDzZ&mGxPAj-ZyX9t;qoi(MC+2p zg@fa~leZRl|4NEx&j-l)!4aN+-fW;FSr$xrnDiP6Qj2i_2r!=#kS~25^FY^oE^~Fa zZF(T@ildH8$P@w?!!hL_fjN)A`vye0b`A{STiIz#7z?4U>;nND)gJLEnb2l1UVFE@ ze`+o^M>@1%k*AOVY*Y!ym!KuuZ^L#E4gOB+A?5W_(S*%n{?F;h z3CK{ihk5+dExFx@D3%K>2JZLL4+w_$mSBL&tSJldq-9>5Y$FRN2sQY|RPc#~0YCPt zd<&iaE%+2$u3pi*=tl~MKu7u?U?jRd0DbJmwW&TUT;7o%Do9J*|M_uLiVYb1Qx7b2 z4tXH82x~%G4|Pu?+dNXQ!y^+a$UmoXL3z) zjYIYQSI1GX1Uh_UUd$RvZ4kw1lI2Z)2_r$E8#(Uv8-e|nhWp2U1I-&Bw_PVe#=~?R zyO)l;5brF_V6{P?FY6W`l}DQd;Mw;@k!u3~QDR`QIP)oYKDD*AzPhm<^f~TqWLsQ$ z&g_GPFBH)Z=8J^1<;!(zmZ)x4c7H*W| zId0+mc@&5eXyY4>^62%sCEEIEWUdCM*{WW5V`C&-!A&t@ zHfNDw5-yI#l8X{mCon~AX8V$-eXG_t7V=v-1Ht%>uX1tMX)@3bI^HwZO!_53;Yl|A zp<221T|C9eoHZ6@KMLx_VNSCl+i|b;c>Gs+%2z?e4Ta-&{`@nSlG}q?CJ6DsK-{RV zboPe4L)+>`C_f|+IEReG^#TCm@%l-= zm~6=e!iAQ=V7~XS^#xCGp>vgHeBawhycI{BWpJaYMP|sdigA^?+#kGY5j@Oy-(P)t3m(W@F%%;*|t(Sv4*CkIC1aAxhYP^=0$!&V+ z;qprQSH=^KtakC8yrEzFLsw0!a}$mYUp8@Ht@@hTE(}5>ms!J!kG`&arhII)e2hoT z{p~}-!4O^i+nx-ao=X#rKv0h;?sZ{pokkiNSeLb*q51;GL#DY}@v2?MAAKaxZ5{9* zN$;M2MAmq9`4!Q<>5^W#V}D#et@wn2{dB*;qhTuSpj0a6sTGPkYv?ZnO*Xy_jq!!h zv#?|r-FKSb4UHNCTXZ+S?LWJ`D#r3HO_@66O_GV)Y0d8W+h$cKvreJBIeDX^N?p9C z6Y7kCkz*2~04imFY4x|6KQT zC65fh_;l^(1eL)=z21FXLUB)Ua*#4r+%j)|BIeDu8Xu4#zmn(a+Gs0;UsY`^nadI3 z?o7_CUA;=C2yS!l9=B-RLK?g!G+!Cq=ECs4O$<8SXZbg=RX|w|c_7uHN2Oy7}MGhS@>iuV+ZwetJ5)Jm1PWa4MAG9-|$f;PTJ|ztTR3#k@-e z{MWkXHZ;1lU*29vE~@y6+GzT+J}18OWa#+E5VV&0UuaRQ6)|VqCBoZw?fdo%95mM>22_8N zYqa*iVL@v@GO}OsU>Bm#vP-q5Nt5ciZ_3g8owu^-oJ7TZhW(}V0l5OHQ*nKhAlkK= z1+a__Gg+&%e&}JTdZ~*ge50z-gJznS? zn@dSAnWnZX)aA>_H%SL4A<~%X7a<=v>6P|~HK4UqrObMSrD|cB>7hT1K+j}vrC~&V zU?u8qo{^2$HJWupxRW^Z`EEHhdy$+4KBn)v(71PVg}zrAY2}`>-Ob?c|WcC>UOc`~yS;VfEC1x}-%8Wn6&k-CGd&uq$x6Ex#5KW8sZm85T= zW9!%Xjq!@$IOf>82mj3)=VW>GfyH&;QPe7qF9)lKB>Z)sIUeel{rWU13?qJ0;x5C! z&H7dQfRVcJGx$xShkg0xaG-=T|Ju-XO^>)G{4O#jml@e2i z(&CMpZUuJGa7>VRiPG9!>QuEKg@!{|k;gZ;Op@BF0uUS(K3m?Bh0hMR%S!92TnJ>1 zuzG*5OJm$!N1qjPb!z{D{Pm#u0l3g;ZRvrd+k3$1fR1esMG;|5^vMs|n-c?P=q{wv z9@)Mp-HiRf{frr%bgh~u2wmY@%k;@-rM}GDIuaH?qpdC|R4KG+-*cyDE@f!Rs(4TV zY2}n>lxfq4T})FWQonxyD$RN=&}n3FR8TKJ`N|oDrq5m|N$YY}cc2Z?_XTyNjmy;= z8g*2;Djr`-H@New(I=tGH?87=fU62Fh^4mVN*lm)pu{FCQ^B18+ur#Q2HK5A0IL1hxTIrfm$48h?xO0)Tj+F|QYwQ1c!^Qp z%;H+lA2zjILRm2`s_Xou69vxYi+8fJQB$7Fay8syc67#c`<1NB2~E(#mf*%GWq>l%a2aJ(yZTr zu-iQo$)SzGF1lf>vdhh0@n$wA8OLX(mOWRbl&<~n;b3MP!QuUyvz;r5uOY;8{aR)D7Z0vWPGH8skAgEh-v(FX-C!l(T)g!sWP|C>-z_$s?oA* z6u``u3R~t&0nF?|g^PlDrorbtxs!@EJ{{Om!h_TlhXS360*B8Ih-jW}xqJus&ve@} z73Ld_4u$GpoMGkQ?!)qO+i7F*mL#>e+JAEUVr~nc%|LoBeQM=31)c6%a~&y&FWaWA zOu1#q_BkH=*qq^iHwNMUau*V@si6!)g?mJ=L3ldJ$*4J~?d{15`HeYY49W|aYG`Z5PsaQE*Rol{v z+}G__%8nn8i|5t*-j>6t!}O6t9&snT+6x$-@i=378@-rrwUseqOQ61y1qEJUg>JkL zo6xLoB7s}@%M2wR-fj*j>S~!b*Ns{-hN4HZ9$4p5@Rb)F4Xv*{voG7`)gt~rWmiTM zoS9rJ_SXbOOWjLlVoK(D!}w~t`emLe?7_$(r2J474Fhe;ui z7unXUs#LGs@z-g__$13WHvE=F40xm58e~;-ZnSIBV^lTvIre2BKHh(tcEw|#F3j%R zQ45s{HT~a6BuJY76Nywj5^#Ah5Q*^r5D+Pf`~MV(MEo&TZHbMyig{)D68J!F@=dDz zmlLkdw?aX^KVH6}tc`ib{WHFG!XjM_e%<8VDT)nnWt;$4Mskf)gUQb&8OgE{VF`kM zqwKt=di((5yU$~CzgeAq0%g`?o9_(;7w&P*W?H#Ad@E*z-J4n~Pdif|&Ea1txKIfE z4-O@lS1O)pw;RDzZs?#?;>T533 z37sV(f#vgIMd&8>tHt~ST49EUV|l81{((NSyiq7@BD$kz@kd?A=YeHOn5&@hschxinkZ2@ z33rte3DJY~X&sNnc#7k^?;11pl8vu?p&h{eD(DrG9VtJ5xxCX#{jEc}sGdhU6hFRs z&ji#u`MbLb=}WP}Dwj5NDT%1PPS;aDB>3@;D4BQvL#7ji4pB?AJ8h5lq)iE8I`*P6 zxw#};0BJ}jb|L{U5nOf>PFO8&bD&7>^KBtR*TlS)7`cUW59)qy;)BZ9hm@T(qwi+qFh9j$?)+Vsw*1wPjk9sLHNOnQ%bF}E)6`LZ(G z>rKJz2iLPmESfTX7#=1VG#n!%nj_-49kxND2vtQ?b&4Pq36F22vqoSQn&gMd4QtU$Gm^=tndAmv-W-#cU z1unTi?$5Vm>iTZEH^kRJd@U2rBMMq<*wb`(U>v5gDOLb|+4){~WOEvuY?K-jA?lbu zQXiF6xG-YuAtp3yLCG2;Y7;%uo-6EVJO$yKlK%1F&jB*AIKu3w-(jCwJ- zqPld=p4G-SP>MB1J(ktJjAm`LPRt{XhGt^4(3<6fGcETwtzlOARP8{c9PFAGJh>(p zc((R|D9q8@WV$c5!r)z>jMN;9bS^Ri0|^uXkdRuWfs8s5((%&#{#H9B@r*-_ubRHh zF`T*!) zVOH<2;dU`af+dI}*7y&6JRMO&eVQGHrj_ln!}}jGscCBIJd<<-7w{obp~bXS@ZnSa zwSPFffhE>P1>r*S`eH+;HWR2TRm~IMA=w5%gl&#(D<}`|ZG96)Qv19bK!1LF@qsO2 zcRPW<~8NWR!0LbsU#Qn^Eu%MO?Q$qRu_>&^YAKmRCIlk518Mt2B zq9v2t)oY(<-QbM2f9lvJ0pd$7oWLe66E>2b0TIJb+9Kln;1MK#e3(y47X`MAS;%x8 zp+A?iR`WFVTJ_0vfJ*1@Nr0j;aZ#gW z9Rh_%XBn9O0k_$CTLXpVQoE-vr8&-P<-A2|>;i2Q8we{D1J`%966r}$85LZCzsvr^ zD+Q>lLapK1ZagQb;4BY?NiRWVB)Xq9MQI#>VoZBNo?6$*(liLpaeUjW+p?m;^LHT- z0qquuz}r>iUz-+8@9bU3KmATxe(jZ{3fS8MxmQ-?gCH05744mS#NW0F2=)Tu|JnbR z>5%CKu;#>|5ph{ik%L;oVvmLYc@L+ywihxHe8*hQY6is*9{^pE`Dv!3qSIb4@UPQH7#IwWiJBNBOb#0I?_J+(MKIQr26)_KI8c1> z2JK%bZB8w6w1N0O6xt8LN}S8ts;CJ?oOVb8KK%g8R9(GoyI&3Ec@HX4yJc7&=Aa+7 ziDkGU>$ojqw@S9c@C;jF_`{oNmq>Oe8G9PM*0-SG==RF)!K6WLd7t53;Lv~Ce|J-FTT z8Vq+1guSb_YKB46zJ~m`T~aXWHTSctIsce4_@|^5wX`!-?$1hG<<;*>Wq;;md*L}8EAT-tAcmSRckbIXvD++k!kKJJylgEdDE(&aPQ=qa^7JvRmSZovZHXBGE z;(Qe}vt7vXwG$EwcyVmy@URm#4M}$2C@BNBGvlL2m$pS9ndt>cX`uyb>`y3aFDY+5 zlLY32FzeMl!PMU*-r~op*Dpx?xSzdr3K=`QSFE<@rNQ<;J3Jt&p~zAlb>eE6K5kWm zyHg7Rj_7L#84QHOnDU?s#0Pd~1F%4k2>W|R&s4~L^OA%aLf(T-<+LjH|DwWNh#A?#QunpJYsK*Fv*AzI9BPZ((%)#C0GRo|Oa;Fw$V@BhY+~0x*zw-L@%T04lRR zwvqFLhU2D!)YSqL0!+KKx_*bt?1~&*$TNK z6`7bwDT9q)b1oCfNsCk2jV}3E=g}zp0VJhh6_J!d#?ho-GU4$k(>WUl5J008kE>z^oOHQ)1xZBykrn~GIqy(Z9GXhAJqNP-$1GB zH7f}9#gDfpjot&*1aKD*s6%nGgme2qjlv6e!IAdI*9p<-7AOnj{VYz)oFY*nGBH%o zMb>V#p*;#zGLTRtOPf#HC|&7aAghoK<}mvLaI>JxN{!I%PoUV+pvn3F00BUhpn_0< zhXc$rk2rz#wx#QeS{42bdB5(`bdB;m88_tS;%j&(tg=2+%uA~F^T|t4!I%*d@ei*g zq-`onL2{z2ik2k;Y3R;8v5N(W@gQmv)|M7N1(&(6Kc8uO$r=4I)36$>^RIURi{-L1 zqu|Nd!aLLczYM}cQ~jt6j%|>dSgkw|?K(4;x2ei(5CwSVG8*LwR6v-K#19Ndmy`y@ z!-J_18UIu6WC0SWoMb(OTtn6^Wz6n8DeN7WQh@8{jA-;~h~2 zK=Z2b#%-`0$P8XR^ucCV96dp0Ca#sloG?pLtePOstwE^R>0B_U5~!m>up!LX@&bnN zHC|PTyBQe3RpHxmx+W<;2tm0ojY09swMgqCv$wi&(;U#j7)-Lf35 zK$e!pn*T-j7Qn#NeCU;e{;&A)x4p*p8bAip$ZsoC=x&U~1wQ%C9_nP%kbUtp*=d*~ zr10FA+ST>s{dmg?v~tZ=apsYNRwbUpVEG;JjFKR`{NVsmMKRaqw9PQ@8x9`}`>J_6 zI^0+K)G?tBecng(G*~}yXMH=Jb&*43x!Z(wIQQ{O|^B*xt#PMOIGUA8viuQ#fH=R!q2Z%*$mcRc1AaOgg>qk=R^S z;x3u}v=o!l_DStcx7*dZFARFOejc&Nmk;GB8h8qOJX2h`pZOaxdv-p%Q9O0CSI>1i z0iEBk1;k!&pKI|)!|H&d#TNPYB72KbJ5!p&1}H)S`wV10sf7H8Ne*G&K=-OhL3O{p z(_{LZYoYn>>F(>x{pM$3uhP1xBjN(ZdtiCgC?8gY28;{inJ+c*BUmZD{70B!Kr@rJ zWpiweQK)X>XpBeyx>F_z{OE;l?1o_XLk9dBzI;=U=^%wy+D7bDLa1Jg%n{umx*iMl zLaEn%X!|p`UuAoatW9RDAn=qL9vgE$o#`kQMd1YTG)UpLgU<46MBc(5K@lFr)!OI} zU-RfAkeV8qucu#}31~Vj&($1DxX6?@H1U<$X0c~JUwUmNb9vqm&9#sL)3X_DFUFTm zdcR=OOV8^uk1i$j^D;fwycnYYX;HKsWHxU}&ruL@*s%knPh!KSpHPGd0K{E^MCg-k za}DU3TWDZzyCgb%dwpst6-g^Pb#c02I6&Bo`O<@D`63|Dyx1jpS)2V~Dkg1X8EeyH zmESj!aULX1sO8l1miHE?-|;lU9;6dnyu*7|c;aJR38Q4?Jdkd)*@4RbJjrqzSjvW~ zB*Hdlrf9N;R=JTkZ|$2J=PIe%?1IDo5Y_-y*B7qIB$K(FdaZ?{(UW~euk&$97$!fS zqK}Fe?M80fU=1)`C%WBM#+pB4#Uofv z9cFqp)%D%&6{9eO2L10K44_sx+gG| zO>vm8zLK%|%nF;m5c9N3?R zYs22fo!_fW->r7HyW>W=M#Y2FFc4c&L==GbgDbTS-PBCj0LP8FY9Vd#2(y)694c5* zMAkt~BU2geoofM~e2OKjpW*dbl_zj7(5Xs{#d=MsV>PcBof=S<=SHQex*AS!k2fXD^&xwE= zz#mysgy?hWjCY{5_i<=5{*3W98fAwL6qr#JZQj$$>e_Hpi4Y8dpDyZi&4peDGZg}AR_NUnADdS z4`|oY7-s5N9S!KN5z(GgO(7@vTG8crty_7A>vLhKqy0y~0tA83m1}`Ayph=5^Pl_K z-;7YUcyJy_eX2@}C_nujAA&B(oOWDgFQ{+I3trV8c-jfhLju@c6WDy4DycmCW+^Us z-oykD(MGe+v=RAyXzHj-j3pNV`VFCxzmU!1{NoVbVUvLO$Ze5>qfR8jV8zI&Q^r1{ z+L_rYzTQs}CIMp08hC|gY9J)G+E@n+jpQmO?Yh+A6kG?e?5jNfI;?^Onc(+ia=kCQ z1^Tv~-GfPl?A=p9(?Ao}A}&|#qr2J!bDfvBg-1|Gt(vfRFM+&QU>y`Xh83cKz_e(K zshWjfHu9wv{_*;0yX9ouQXlPE-h04{@^mE&CFgLf%%i|9bFL7B$(uHC7d61i36rYX zR;htFW5WShgTlp%$L%(gz)DF_@_3xi2_#(oAGk2+iZKA1!_}6W+2uL`Jznb}8WkmG z|CvU}$%bjs@8tfn(P-C>&3;-^eM5hN<4^K?NGAP-`e*YxG&P-nqDt*jKK&)SF#d|K z;hVE|G%kSRR$k4wylW$&iBy=WXn1%qYPNGI930I33_S@potxCxMIdToq2I9iDkaxiWK>L8Uf#;s*ClQ7^s zTS<9j$~~8~Bhsc!b8|!z7BN%0qKB_{yzgsel>NDCbESNfF%8jUmShGF=EhZ>d!!*~ z)AhM(RwEm`#adu8OqMESYCJM(ZX*pzMKBn6J5L0w_<+nc9rnLxu5DQ&KrQG1wUEDM zEvuF^;t=a-P~-I8CR3$Ar%tPA()R|Xc{);jXGY9aImHp3CRtA-e{ztM>GK79e-3fp zf!=L)1YfqE4kFSjwlXN)6`&5CM<0k0e+|>DZ8Tk{`}v9;#Jp$)^Owsv=4;mwVJkpv z^_yByGN5`z`kI0#Dm{aDH4bvyo49XZZUAm}^yZ7QZAclAwe3@_<-sH50R|4IQFiX4 zu@5~DgBAFl=N}R_78*7V&8a#%%zWc; z*7PfIcDpxM)T#MsQavOr(Q)SUy`#M{Kw<-XfVutd%H6!BM?E=aORRZTZp(>M<=}GP z-Tgo%NfA{DX3o2`l4BLL>Yl4qguysOh(IC4jm)wuI6&^=W|c z$WdO6%!LdQhpD3}@b&3@eL|ti`c!tH<40FSCi+FEj-3L*#|w^`@~ZqiG`*s2Q<_UF zu`&B}RHmCcMccZNeH4wLJo)4`aXG)-`IH(W{F32waExKrNWS;<-S;^;!3*230Tg?q z@Jg!k>m4g~Ii;IP3WiDRs-Ch#WbXUvLwhnM&k8G8Ad}0~VE{^nR$lMbt$R;*=<1ig zr{C)Zv1u)418ZShH@vWa0Q~k`1n(IiQIa4$>F$BWljJcSRv()DaBDY8@z)TVkfXoPw(#0Y~f1~=G;CG4kCDR_;q=0VW39ACN; zy)u5%v9}r4u<^hr8VAg}rSkl_?gP!<||yE@dS`%{aX^Lw(qt* zC%Aj=j{?i8zA%?U6Rn-UoHx^be}XBk%fpX5FYL%^tK4?A{^|uk^z$s$yNL1b>?Fr& zkOj8{j7XQwpGQRZnWI2jdy71;?xykQfr*5TNh|F^VERu=xIyMEX{bI|MV8f5hv%Ub zc9jc7i1lfqZq0i{rZP$vCpJ!gApd^AF63>`7qA_W%qT<4W$){p9$0$phodX*&(qhs zGIC@AgR$u?c(lZt%~mfSvT6LuoXgTL;bZ~cCUUALH#NOr$oIqbUki;AM({>AR_!R` z)V6u~Sp~1@u;|#i-gw`%6b4kns+6W&60ZZ{5rn8mi7`WpYbb8Rb>pYUkq(p6aBf) zxd64wTymbUR}8D z^?7F6%Ae_?R;=0mq$qY{n}nQ60%Y5pR(O)?m22l&9xKABHLqDCBbU}eiO7s{AF(r> zpbve~lW}F+Vk5&wu9(13$iKS^dHH@P&TbtBl7}gVHh3#WWIa(|l2UAAtYm_t^i|D- zhhCh?0r#?>_gsN?DCJ9@zs`k~GJ(5Tibh)vXFq@^O;L7(W2>NYSr~SC1{kmJ7fAXKuO} z@~T=E$2rFUt4M}7Th%x(qMjF})zu+`{iKo7MG)#T*oQw1i?f_GNPa-b_{#T0bwy){M4d6aHuC8Tu&H!!$ z-IRE5XH7rLHm;v1W-|(bYw#R#eLN|qBsd@?&>R7`Nr=g>;`P94zAwyl)vlNakGiC~7CGY66{AELb!m(U*(Bf62_um< z{6;l@*~kR;;nYs9j+&xeXe}-E-R8VSo{Z6_wu_^*zWmfwGA|z<3jGQ})+5{h_a@p< zRbEE7vvMV0oUi>7)~39jxQ@^G+HlXe`_4Em8PykQu+j}_>HoFm^f3aRm4$ZC>uV%S z)^+GkNb6svv>Ywh%}A@!0ln4)pgPSkoRP)IZnW;haYlxM1WJZC)XkgNK!Iz z!}V{50tSston8UCRc}ECr+4JH|2!5e=rD(yREt0kI0!yoM`pH)zqp?ku!vm2;a5k%z0~&qk79syzyCjq@&EcL2FONzfRkRBt6LoOygo~J zxp5!(;{}D>?TCosMwy=;JXDwyZhl{WXB}@z``KHbtXD5=R(cu0*F6_DW;f`^=*MR5 z*?K#X1&GuEC)j;{IObsBo_wWkH|dVTBmcnoVo#R)^mWO}U1#mi%?kWeQitNhRnTG+ zo}R5({wKV#`w2;Y;4DYQWtBX7jCQJxxMQ} zN>hQeqiJ7nm3Eg3nsogi1PyP!k_1Yd!3cK$K{Q;% z_YWa=KV4zmNn8bC$I*9Re~-SahDngp2c)-~6ihz@lWF6VG5L?ln2>Fi8F(zHzz?+$ z-PV4;iP4r0%_8!2Lq$p-C9q3=|6=ERl2))QAZT>IXD4|sW!mR$^M8G7I}p@^xOHq7 zU-0Km;3WVJCUzSHbf1!j;KutpSf~14;pmo5%nmpYH15-<`{nYj^Ey%d79QuL-nY(b zKh+;lj$b7P;yqT0F)dn~k=oFWzY}3R28iFAu(929npeN&nFvxWTnN$c{?vP*M66=2<*G z&NNJLm+Z*n46|y*hYZ@g()e_K^J#YgNw#a?%sCLjRrDX+Zu>XAE=W+wl`%e=C(znB|e`8cGlzz*+u#;Z5&wL;4H9cd z|4kc^I+#w@Ukfs_heX7nf;4}g*!i@Zko(Vl%GLq6kr~Y*vXu+^du2Pz-6r@cCD7n= z=O-Z**dS}v6K73zyFWd_^Pkf{)4Cdh46&_Vzb84%`$y@-guVvkJSEI-iu(#%!gu%?(w|;MP&id#Bi%F z+pf_5AsO;4R377g$kAMw6xU1-v9_n(2Hzu)F1iRfeqCbY!Kpgn`+C-`Qyyjf3Nc}i2I_#l0U@lOs?r5 znf$He8jiTyh(TiS9}tJ+VbDzl7_-~V+rwSSG~hXXASU(%WbgmqBP(EI@a=iQZzW4_ z0ThOY5?$1Gw(H+z7bF56`^~b)6#M0lpC9Kl_%op)u%#ITonw#etbm>l$$nNL=;5Zm za?8rx7*px*hs{AgcJXtFaP=U0H6;U zC&%C0{hN>^Kp?id7XDuF?wFaN(uwcVViQzn;P0yj|J=zw{^!q;d+Km(@r)lSsV*BM zQ0ZPk@tcbvOiz~f+V>8`c2LfjvgIk1qHlj(dK03=lhlG)nvTTlS3p7HC=3GDh|P`W?R)*kydNcJe#RG$m!P z{YXM`?2fGTwR>tnI^1z-Vgn<8Mq>mRLe(!-RXaaDdUu_+4o@=Tq!&lj1J;mdty($el?8 zOlKautz=*2l!Cj>%pXln4*~bSRPYDx%ueC70ci)7tjR5rQrVer3V&u40nUxo?;i_y zRkk$O#f)?U;cVaWxl&V>-zp&X0MH5%^pV;-u13jt0EaUY($sDdebev%_0M|#Js$l* z%+$pd?nx;wta4X^wd>n-aK-bMnZ0y6*8uCXzf$D;6k(3J1ZtlvC z#*@?{r>4*l6a>i@^6x3smi6C1j*O3+NSF7c9Nyk0Kqi$TZ5omM5bER!DA`sb`JSXU zk5O=$&~Pkh>K)fkn`!V{$=eji*Prtw_9bfX&Qx+A@&OzpcXyg|T#f6`dWRsDfnR;^ zxbwX`Wu5@qt`r`MbynB59Z?O}()kSy(kn_Ou}Ci`!h`V{RkK@Oh=D|-F%!*2olC}{}%`3hcn^S-<4 z0Oa%isj0KA3qFuJB(}de9V4dIV$}7=-F#LB$b9(1jf|b;#l8mwn!G7{b3XAJ%aI+j zOR{wXee*q9!ev()HIW-6enK?!pnBAfe4YpW$mG6DUp~0=%$vUN|j^5^5>C z#gSV`Kn-e+H>Y-4Sn647WvkJ5fBKpF)uX@3E(a8d^5r)Ncb~h8)OK!VQzF|{ldr{r z_F1eye`c%l#BW}2y+awQGL|y_@Z#=hySp23cnyr(B9!#%&$0YlYu!SDnWZEKSN>km zpXbK`G*%Ax?(*qfa{PWul1&v5COJZJKh$UZACJa^LQZgZyP6lfOx~pegq#CAAi-SZ z*neH#{=c)hkGe>Kv)o!j8^?k;+IT3=?CueOqX-7}J(k zl=nCOP9@!QiNE2$0nAHdrh+7i0DE@bW4|`WgQAny2{(ffAJ>;I z2Whnv)^5t;LdH7p$;NcLBLrmsug%2>}!n>##m;o!>DXo24O5|5@t+D#xj|)d@qMO*Z2MR&0p{JT+cPn?|t6q zxtHH_-_K1@P*wNfgzQy}f3F+Vp@t(3i9(cK9- zQ2VE>AO5^PsM`LP3CUp5P*nemTg6(@-;mV)t|mL=C)AQj3?C>2!$>b@d6}`i#x!q+ zl+nI(LwrqHFCzWy9D=)N;pH{c@^nc^wiyeGlwC5Y>s+z#e%zJ6e%da#o1En(oqZ-> zYw%9?-fMG-5TfhOXBQ6wc7HWVe0|$U+H>T48oF`dYW?}XMnwmCI@ZwT$jlosyQiMm zI{&=CG$Ea@vEu3>vYC4$6-rEVen zt^3sAsC3|}SN7;@b#_yI*_{gxJya)2*YM@rV=6dRDSE#w@>cSaw@YSnKm+=DtbJ7u zxQB>a3@>2)CoZTJi23N zXB>tAs~(c3%GxFaEHH+roBp-}ff#1U|}Ytiv$#pR5dAsqf{FDuQ-`$A|~WOl`VCvJw?h6S?Z zzNd1bdxS+xOO2yL!If_x+2kpp7yt`DwYX3__efWwx)ivLe*IXOkXs%J<>P{b3-aj> z;(?GonIT#gMyMuoUGw=xq2i3Mhw15D;`sH2(~!6UA4I2^`P?D($pQb9a(L9jYAsgd z$AeEcOuf6s(v)sElO#51kF%otzggZ{na{+5mHFtxzOn7f;02~R&>ooGs<&%bb*=wP zAPhqa$V_OQkkFH{kj&sQmJm7~Im!?$Nl|8sGmmj5W#-+z{drAMS7`Pd7ih4UByoR8 zrELBy9}MIMoRLw8vqM^@qoN1FflA1z<4w+^5_9@Z<*DdohQ6Dd_O%o2)Hoo0-kn^+ zBXsLS#Z^oE8}KQyD15Xv%YHbv@-uL;z%2LzgY?(pV^`W4LW*R7tXez1kuR=9URULN zgi%pHSh$m-rhCCM+wctVacTF5zk(WvVhIsQDs_1*Ydn9kA52qC4{LFH?j?Xy-AR|u9j)Mjqye4OhJ`=o~2Rh&(UUK)` zW-XT1e4@01@KeWtk`7=yS8C~6`Bf);{2+CB`N2?NM=+Nxb~Tvrq|E0JoPWcbvpNs< z?N~{*XMP{;(aerj^y}`-Ej4eEU>Mp1dmJgI+-n5Hz+`$X!4lYZL!TSZ^{6TD8v-ax zl;HBLYIIYHFW!v4F7JZX)@BtYd!I{FCz4@wEmRVqc^06km#U0COR zh&W))uDoin^ksE8X#ly5QI4U7lQre2o2~p`M#(JQBFcFf5BcmACw+rK{f3Fa1qZT~{^60#w)uydF;m%gH)#69fv;gPKn!Xl z(OP}8p-F$ZRpRJ^R$Wmmp z1wSc+p|2cN^BAoa@-9l1J$wj!nFkv6ZEwAMp@m8qI1fW`>^mDF82BvM=HLr)QTprLjir+{f_&jkl6;9@BYh4EACmC}6$5$fC|GcO zmqTH2brzy|Ci1nB@8L?nQO|RptdaufGNHK7<`Ie4b!SyW}+dS zt0|``^dA1IL7rJ6P$pQjgu(8U0Sd6#-WqL>MZt~^q+BlqLepwhmhjCCx8wc;14zEB zmbt_~zCAXp#UyCWvgJp+>5nxwDW3>Yj@l)DGMiIS(e%0VlSbX6-5Rkk6p$nJmqz1} zUyS5y4o@EN%XNHlJ7R8ZG*>3>iS&!3`9kXOs5?g=8{x|@>-LpW+guL#cb(qw>c$Ng z&lfn1+PXZ|7F3pzEYJ4*JilB2dPEVQMOFiWSev!=*K6xowld|Lf5_x#{z%JhY06o4v5BY`^ zLu07|rEr{0U?L{K&Y@o9d%`Qj<>GVdN5Thim|QL>T(zXT3lva&tq+b8qXMZl=-B3# z(db1Hl-j3nR4%$&VwNq1`M#B%p@_1j2$$nIlDvXKes2jj?Xy80Dr7_1W!dN=#nr%e z4AliUBdBe~s>yAk+j9Se^o|FB=pxAf*o; zTL^~O3uL7V(#evHGfA;*TE}wNlgH+a`({1`euHiAcGH)zo?JXwYabJ+*cJ?;I9CX_ zhzBN(W6%+)KfCABvxsHgXWW11^QrT5Wb&Q}?BIrmM4IZlE!RBwIM<2oNn@%sSdmOw zCJ4FNiCEw0(1yFW2@0Fck<#(SwVN&;H-U=CIw#XN`*^q{YveY9{Y0ui7-v>hDpC6H z!Nnwl4ti!}edyiESLLz37|t>HdT!6mHY<6+C`YO1^Qtp;91iI|C zhm9%FV?R$Fj^{pBk;biV5+$m}vEefnm%xv8Y;OUJBvZKV5zsS6t!d1!%SvzC5bU-L zp?9q;>%7$v9+I})D=HY|ZdXH4r1yLu!y`j8u78mh5Anb5*=l+p($~ZG%RrZ;L$q8? z{#=}@p{8v%SjOQ7qNVZ~Cv$bTrU{V77tR4tuLQ>4-8PT<_w@5f>0uDTc`q;Nk1Qo8 zwPy@`aUN)mo)>_1iNZ3E(4CrQ0wIz?-mf(M)r3Oqc{T>pBy%j@jUm`jTfL1`F?u+H z+o14t!56zF%OZHG%wzyo6BlN7zj#9XT{_>`T``E9HtC~!yY{J9)J-|=!qpzFsu0gn z2+L`4*n*uvemG;5K64Ht`l=>R+UPL=b^kA@5CCdoU>t56>eBuKffM&S_3EbPk)J>p z0#Rt`a#Hed#5=Ibh&QY+z0Cm+9cHclEBy-OzhP#U&ITH1R=liFvg{Db(QO--r!MSq z$YvfU5gD3)o9pDcf+<(%!g>8NZ2@!2{2^P=Gn#2#T417uiTG&)0}z3Fwms(d)E6wQ45PV~|#kh0jbu2Q|v+f?{#eQracG#c<QQjRi}4T$8^&LjY&@qf`47OR2R#`XjrV_rH`b>JFc7^RR8WU%0J&z0)X3>4t$HL~r*UK=Tv*Xfs){xEp@m_DpIoC@A%56m zqQBwKr5*1$B4&>`F|F-<*@bQ0tr9U}_>iAeVL75ypLe=;>Vjm|IUcMY0+Ec0lxk2^ zcNHt~_HXJ?2X&Z^30u1-h`Ok51k!{~pZ zxp){X4_7AX!#QTSAlZ=n)vjS@{&|?-I7(J^!7LLPwhxMZt-p8UitSU*jOm=3$)5*% z7zgTUtT3Tuv)a11x7*fl=gf8+@j*9q?GYVe>Xerhxi;C9Rd}k<^GdkY=uPIU4OB(cgMB`#CbLPaR)L<8_Wb5| znjY}m>-&br#lbgE5x!jor&3wqL}=NE|q1I|ZaC2i-iyX@To zAhKyNCqwurj|sj8G~8KF-8~#@`R#D{$)})~KUofQpaHipQ0d&iYJ+rq}GEx(uL_Q0&tC*?|kdpvsY|yxU=XtHY)4M?X8<0~%T?TyN}1C1@5= z3#o=o+_0nBK%|qaWZZt8ET@=@F-a@z4M1#P8Vz`&Zp|a& y5b5VU{^t+=Q3C{3fV0wn2ioPG{{Jr+LE*SjKsD5uzM9Da{2=Ep{e>}tMg9*{VmYS( literal 0 HcmV?d00001 diff --git a/docs/design/architecture_3.0/images/framework.png b/docs/design/architecture_3.0/images/framework.png new file mode 100644 index 0000000000000000000000000000000000000000..992afdfff522f412c307a04a9fef8a83b3e45129 GIT binary patch literal 139722 zcmYhiXIPWZ(>6@6p@kM&=)L#eA@tsRR|HfdRX~)4UIIuF1r>-$ZwiPANC+efNRtlI zB7%rO6af|Sx$yVDkN3@&B%8f4=UH#)O5XMA)!}lnRp0_36>iTPOJk{g2kZ zCtiXunr8m&UE}%FiHtekjX^zbF*=PiL_q5viyxJM1+aU5m!eQ-v9)hXN zMl{r2Mj(tpE(XM#mIXP z4ieaenLuUkNrbHkFv2Uc;wmt?A_(n(nvLg}2oU6`@}U6+A55vCQ-*_VanZR(d|_>r zKNeDAgVTWKMiSAwAp;O5)EP~cLsWT$7gC`UOl|X0TIJ>9O{a&8p-dD%1?h1F(05fq zZ9C>SnYkTueNlRV0H{8X$z=o;DiJ=ZhCgVsUa1NM-)j zWn?|fNfdJ3(8}ciWMrrxV2`IQhqL1<4S=zJO2Ne*4XUl=IQ*M>HA$Jsvo~N|maPN` za_$$R!OIjxot(#YF>uLLH}#Kr3Nr!W3mFAZ!M^yw3M2qIFzt13O}Sy13=Z#b0mXGG zIHm6(l{exB@MG+<$x4RSN-oOuBrapX2+kccPXbwdu{Zw-cx+4B0%L&U6)$?_1raso2vWhoj0^Qs-Av2G z*A8-kY^_2SDV%L=m(lhYa1QQE4~EoXBdpC}7ic@&;fiOA>1HzpMu>JwIY2Y|!Cqdh zKX1MpQOMS~>H@@6A8DLv9Y(lJ3V6OeKC<0ewex{w1sN0n0_gF3 zFtLsC#QzTPQ7lkRjQ|W4>Ccm!=#U>rd(m~_N zkl9!O07~6|C7v-q5U>*9(S5NXO(;$Sc<{Ieekw164F(J~!v<%0#vKHx;fIo*x-)^k z(1O{>mTP|i)@ucPelhrSA@e%gnnv)*kjpzGrkjQmSSWeC!5OI?A@ZUxjppS5%6(Wp z@acM*;|g+1!|9^4Q#({2M7JTL#S*&*cF#SD{fC7Vhz z13BiYx-NGtG$;W_f4nJajok>vey@Wcj`~a48Y27xigcA(ohEZB>pBly*2#*47(UTl zpV3xe{!x}8RwYjBJ4BX56Ibm5(CEhzB16P5TR@x(p|Q>wL17mt^E8MOJT?^v!f=b6 z3Z^e9i~GxhSLAzHUR0BpM`rccQ0Na@<^s+D=o_KsNwkvyALFcL3=kAoPrd4SCR1eN zU-wthxCt6^j<;>^n{cRV;Jt!)yM!>j8}HiKXiNR?PL>OK=eY6$PRMwJ5|ZmtmM=TF z{~*_{lG~#RAHDi|g(Vzuu~tVR*Ue-fnbo$bw+u}=zH?812l(Mh%;wFR%?Q8} z3#14Ad9#+MWcFGx5Xm3ZNpwFJ5K3eC1R1tLuLBpIG^&W{lQWiK4Jsx_;DXW_$9cgw6c(13zVOMu6ZPT_T!d*zrpnN#n;AWMb;1 zx{u&qma04IJ|u+!uvr0@RN;20*WPVYNFE%L&NvfDxm&Q~K??H+!Gw@M+;cpGW8BiD zq62uA98!poyh}|+vRYfkm1HhAz|Ls$cW)}m(n8IO9+d>?G)y5}>dZ*93oBVY>TiIG zmr!CKLKMg-Lt(n$`jk2Y?Jf(O!FZK`M@fGapu+ZfVlzRaT}!%KkbK%V{tR=~q*;AV zsVtBjOO?%-8KLlll0Y3;fWmG&>6dn&1ErmOI0Pj(6Ue0%a}UE@TuHe+$yB6B#nVPh z`6iI{S%NDnte$THmAsQd@os0*M-u1Hn_fv--o}=f;D&O5$rjohh-C|;5v$e@YolVg zp;R|uz-A>XVXXRK5|u1r%yBW5V0s`Z+dd~I)Qp9$&@`hWKmw-SRpk`tQfGv?8d@S{ zsG8lzr){4nS%(8R-McO!7N9`a&QRV)bv3q`#5j^B9nBctPUUlKOH~3vI=l&#F0KAh zjwW+ba0~;)FtjC2uXGP%)%wxIPi56v$r!SeA;VqhFC&KL^=b!qnjf{0U==yYClT`w z$fkWu`#hltJfkx~0W|O>)P(9?omurOP=iCd%lcN|5$NHT9NOM11eSpO~te+IJ5XN>(ne4%UrM3G}wGf4oN0dWP2**BG3<&nzCIkvyo9C?v~|G1Ge&s4HhJCbc(#5)bE02vO&xawqSw^{as zo)VsGfRPUA95VsH%t|6R8i=<*L-Is{mO(`bLTLugxHiCBue5UKvxZR^xIgRasakRh z6CqG#12$H2U8O&js}3f}-A=NGOVeN3cnP7Xn?l$jO)LT%v66oE*qTc(mC2umN(jR! zgMXpGXQhE0-lAH?8{RKnVvT2W0S0kz}{R6yCAitLV=tNVz@}? zpi6>l-b?ezG^d(trqpU1U`;OGoZ6`rimwald^<)QR&YJKEzr=tsZ<%QHa?pjBZO8G zEAs1Y{vp^|$-i)B`WVkKcEev{K)fC5V?LRBH57p1jGOf%+d8>>mThzusn^BB#LM1N zYvvJ#2z(naU#OYzQh@-*vX?roZjdz;#tDwDqFg?-_+PlKAmhzQ+W{i*yB!m1-(e$> zc*Fiq^ER4{&x;J^_BlCQ6}d6yhYB2(th8O5tu+mB9?t?(`mRb22J^`*tzH{jmKGy~ z3`aGk&q4}Sw2&bWba>;9N7YuJnU}eqiGZsTl0Mkg7c@{eK6(Z!vGLBQ;G2yUT zK*frL;%`_R!^XMX^C|SP-*-$9vfcEzVTdgan~CwkaPu~jqMA}64A}avGOg8Aifx3k z?Cw;G%z5spgHSEW0VW(6hhZUGEO;ub+uyC?cEfu1enUG zI#_rLWk#e9hJ#i!z-Ow^!PYhs_4RmfF;(iQcB*B?H{(9(@)lQdNixQqwnJGOHVw1= za1BN?ZH_>44l4%iJw-b0L&&8dRjN%kZ$!YU1ayG4r(B?uCO<$Ew&Bg2VdvRZqVynpCCbHLK;o9j{l&^=;3kc&#v2mQ>Db=T-&iz%CjBAiu zX@JZ)cB3rQZi8}WGclfwA4}1MpJ;Zq?#_0TP zAlf!zBgU{aV?#gdhSt_8SA@6iS5Vl3b(S$xT}C`tS&HN-hh8c}&nx=|DnK-Syo5Cn zsRomx-$-I;%TndWu$J0x+42EOo`+sKxw4Qf_dGSEN86?WYSm7f!v|5Th9Lh{1GF#DZ@lD@asK4s2n?k}>4-o0kpagt(IGgJ zZcY#*Rgn{@8FEU~C5W-C5ZS&IEnvucZBRpV_LLSZa++QN7$pU5OYwCrR{;z!(5eiy zk~I_-YEhh2vI8D|L-R2Vp_b$@@kR3r^yqbo{T3aAwiV))H~gR(6mm!4$fuQ&a}IyQiMlpN&()iYF@B@QrC6+ zor!kK1#PDG@3`(6-2c-`c2Z4G3mtOTq&%frDqLr0s|3YWk~eqrpts619|H;>RGNf| zbO8#RYVfl=n#Rk!3S*=-ENp=kAMvtGYK>uMfTP#1f#R_#Ianvh$zI4M{d*fXEx1=; z%~QlZ6??QtB~mv$e8U!}C1T!2!H*Mht~TGL5;2byrP~UQ)Olo#;8gH!xF~QC31`wU zAz>F(zCaFzYMJD=ZLk9}>SIO5tofnPY93e<87U61;@UsPELi8_%?H;TRphytw?Xih zHbcB33#s%mC$!N5n1?(0N-GEoTD6S>d%;>dbwH1MiuU9002K6c3!O6Gx0BF@!RsrO z+n}^y8C-?}ONqo_2|)Re0}GO{hteXUxVJa{*m7i!G2 zkB$EcBU}c?)3=wJb%yze z{KR&fam$sI`x;CAs?_sAMM>lZ!&GP7jRkM6;B_>SKbjcSH|@-yfO}IDBSq4K#9;n4 z56ja|%-;>m)|I?REtX}s2J+etdUt&O;14PosbuC7aa@M;Jade3#KIM+0)?;Po!*Sp zV_p*?dDFf}l842dkguRPG;`X>G6I!QxG3`{zcvEv^GOHNZKiMSh7`-yU7cahwDpKU4-_viFz~S3S^M$1UQJ^WUQt0!)d`O^3 z9PXz4@-dhQInQ)bT_>~w0 z+thImkUnSlJ|#Mf(Cnt4Uwp9}$(oE{{FxrWQ%Z>2xm&az(}Yo!FubjsNc*D7)mC{Q zAELo24Cm!od>Piuk9Np^bts{=HM0TZ%_G1WVO6+CZ#T7M41{LWY7e!<{F#4QHDjYR zSdnn8Xj?2g(4QNOxb&WX&SB#eELSOl4#Fa%k8t-cZMt+ZBAKw zqUY0p!)sZviQTn1)BTd-BHQ?+&(Wa;RS7Z&81Db zmAsn^xmV+%{6*JQ#HvZYv_Z&Y+%ilx=#Yx^Q6Iky$c7fSk}_>ZJ8uT-q{=$h20fWM zXfryHf_FQULrrRvJXR2HPdL#0Pti1d@oX+Vqw5a6d^H;E^m~4^PuIP=N#y@+3dDc0 z!TF20XQr8dTuz+`Pv8syH6;wt@d95LbEy6?s`-sW^E+5QdzdRB02e!PeIyLU#_uq`O`cNWGfwgLB~{S+@E(Ush5t?;2GWO<=k?N zL;D<@m3r}A>hxLGE?Mv-g4k#i^mDAkLgEK8@e9ti&yv(C|>75M{i&j#3r z1024UYEd-7r4st4*2>ay0;&+2`7K*&xTZ$wel9bkJMBZhuMf((Cl-DEPJW`L8cS8x z)o(@poW6H^b#AF=_-`#uvjmJqtGAO5o;l+=v+E%YEp^I#>)0At{?JQyn>qMn_#lUF zxUq10)g(_5hONGH;f_fht!wqWey5Cg=0`F}bfl12kVZPN=U%zgGF$C2(N)DrOBGPL z$|tq$ZEaZn>PE4J3wo$!c5_>U5g93i8g}f>-Ez;o+|H3T;}5#ajl6A=v7fDA)j)%c zc|ppMa17aX$a)bV(pAa4T}_u)mZ8-rjfd2iEIhsJ1%F-P0FigeY@B-f3MBT1>)Kx_ zWAS~AQ@jNb%$R@ORTJ6Cxb)kOO4uRGwKtvPLrj1ohZV4U^JfYnx-ORFeg)D2pbn_8W6F@I?EVSi3hN09g@xMw9t+oH8F%2 znv#lH*q9W~EJyM^>-mxq;OHVT{~@encX5Ikz_wwul0_jJ|Ja$~QAc?BA@ijwI`K`4 z-fXuv-iZy(CSwFBJwRZo9r87yi`=W`S3vImTu=tNlYTHU$~zxcH38AQ@p0hW$hjL6 zx5`JSATe9$%@xT51>)qD^gM3#v#?48f=rON?yktxbT9T3!);E6z?GERgS`u_kE-d^gnzxsJ^TUY{z0utnpTmJP!#R zc+A-KAidJ40k{7p!@`lWduCtq#cc)?9$y^qe|ugJCeuhmyZ#pYD`ZdQ%EsC$rdL92 z>N=x*cJ4|U)$DujC31m`?$2D(zGs@#cdtF1A@-V5%WypeDfH8G5_O|*IbwThD$@}( z^C|@*s!GD&kvOvVk~^`5Gi;A@MEbX%Dk1fK!+89BnwYhow^$L&pdZ3o z4BcBja{AUO@3GEo2ii{MtPt+lCA-*|7}ZmLV5f@2*+1h?Ee)5p zn?OmWlWpOu%cw`bNk2u_tyc1fj>8fiQ9<^I*u<9URq3HLVCSleLAsrWz8&1U$$Vyg zU};}>nJd?TvLCMB}nPUGe<7A<(J74MPb7nA40w+(gHO((ox$DF?DRKX!*78 z#tQ1WLuRvk3S~S)YbA#tHt|Qx*!T~uKz&TYe3qJXfO3Dn$wALyuI~Jm1D(t-KqX4N znu~!uejz@2#9lo%?ovrc%l+g5;EPafRP0r9wFTw!Lz+ufya$6m-87=1LqQAIPR`U5 z4@qk?k~C0%|LDB9EcbFHy@Z;>e{=9W|)5LF|oVCsCkdz=enPox#wVm`^Q@OO}p+y0s z&kCe#xW4Rp_Eb@BE%Dq*RHRg|GHqKnx3aQ3SiU`={_MtgIh~`jB6?d5!ss_TtxOXk zguk1M_w?-L@m~*653QmeC4FS7Wm-%;eeg2slF!ui^z`1^bUUO*V$kz(6WvTD$*qDb z-?QOD&lGsxJLDN18JEf~Ta}BNFDO1}ceGU0TIh`c89i;bdU5ksQhbdDgYIN%B3#&2 zxc;60@*Mlkx{i9(n)IxltVTJxXcP?N!@(^FgyGgV7nTh+*7OKS|9cfO$p+vWB5 z5`&AU%5!Ltse3k$eUK9in-CMc`1ksyO&i`!!W4PXBm>W(LSWZb-QB{R6m#R1r#%D{ z$<`oUjwcRLxMV=Kgpt0)(D z(rE53S>cl*DH>2?%?E{PdYJ-OoE}b^TI$X%z#HtdY$<54xg)7ok^(j<7zmT3XExS)gq@fSbh;t|)8cm6LS7lLa!>KqAT3hl_*|AU zU)Xl_ri2ODC7#=kKn+R7!qreFW=XWS4Y?i^rX*ho$~S3aN$$Gbd1S`w8s5sAb7<3> zB2M`#Z(7dhL~`F-t7WaS$`pTl-}`5pDWH16!=i9K8=2Psy9+S3z}n>VQsvLq9UI`;UdzT?SRTZdL;N?fw|j11`7bH2(Ua0TxnQA2?( z2yh~wTK?2oNqJ@GsuK(E0an_?Pjai9y|G|s6|&sL^lP#A&!o>tLh*7B9L^D@*7&uF8>P z!)3Xa@xbD98b<9v1vsW@(|^!zZOP5pev>7MouCNC?ztyck%cUoz?H_adKc>~u<%gU zm^FQYSQ zf^erkg|Q5Ew$F~xK@pLTdGcEk%d~Hv-4qZ1-qPHN^OQ`3T=`BH$$}%y?|dp!H7tLx z;A%;K9x(b%0rXkTB@u<64Gsyp z9J9CZx$G!+%uu|7$WFR(8JRu%owe$+ET{zWZB9_N z8lj-M51hnuQvx5M67a0pI@h?==4-xi%@~%)d;#uz_ijqmlxgC_NK)8BFZYZf1NoFg zISnJ8Gb!HTiIqjAVIs?3#>7(BemC9XlJwx^Db^qHwBc7jjD{9`Gd+4%Pv)Jge0U`o zz1X$2p-_R67yR&aPEasJ_pUqYq+r0HB30(ADO^HsSQ3J=rOiyiSTuBNtd zt`sVnXZ0rvP_uQXOuE>cT=(xk{7$dUyA-#+s@X>kVxf42p1!90RrSW-j;4wVhnF{h zBG0_ER!td>u_7uzKMoN&dGoP{9%qbk)nuZlVz$NKsC>=9OzRrCnEF%UF!GmYaE&IIx!owC#R4rlDG--+=j_Mi=?<_sJ9!*!<`gqF*Z1= zL0;q7Ky+Jf<-R>w2uX*(88EJc$$O{Ynnctp^1my))2_jS-$*{=mM%oGbVG+j2X);; z@LZ_zvO7{=bu->-fQ1!pm?;dI0jN~VkeHnu(b$TYr?Sa;``xUjW$6^Kjco}7zI|h| z=Wzb?YU+IxT3-YDZ;>1=1&$~$mgj_~!M+8iKBDo_Ale*D+D-E3>f(0|GKZBcu_lwt z6?d%8;#N&ST&Gm71Q(MjhP8pkrH=vfO*pErF<84va`|emL;{ShnuX9lGB>*k`O{=S=~&MX?TW!3MxBA9n1?ale43${t98ZC!1Dn* zbcZx@1KOc5@g(wXA=c{~t)WDW=wVz%zTTU=-=L2=|E%J`dpSjIB&_YEixR`4J17k? ztNS^>6?lDs0u;xlBFJC6?kQQ)O{w?_l?+MMbPxKzj|(qLPxX(p&n+~Dp-M|iglS^k zgirKmq@5n^q+!Y)p)7lm$buZP$4xbm{QaE_`3;jM1I}Vgk95Dl!eaJr)(=xWmjh%< z1$f8{ksN1y`IS?slA*kv>|2U-tdj_kNjnugBvs_@(%3KolIfj9epxt+(eVMkWFf-x;--bj6M$}K z#9I0&z>c0B>U^B*>h9SR&zOg`Pg$U5AlcBFr`&P=IuB~ts>vTHu-prJS=@aLcHA^y za8?#uzvOHwIeacL$i0%CLv$M|Ku+!4NmVO?)2j)vekcS2z&UmPg0qoj&M^n!#!8CV z#{f1)SXS|TB(0e6mES{=`pK!PZ-yc(GL7vsA!3_!V{>9zuMYLgWNY|w{xnK$&=>14 zl~8#((ex+HR_h*3cc(LJ9m2l{nhRw;QDZpelfwu}ib7IsLopQIh$~AXL=oJEhoq`C@3j{CSy_a}Yb#TsBh4d#C26~}~4W{{~`*kSk?O-Pp?Sk^JH zOGZvK>xqphk>gX?ce2vO9;sYBk9oEJF9GtHZXUOQ`^Qp|s>$he!2u9lWkzvaB$t># zy$#qxQgN9Bk6m&m(0a|uh*sHvUJo<4=g8y^T~?s_8H&@7r2G`_l+{S2cQ#+-p{^1a zDnm9ZK;i$S7rwH67xl*>yOuKXu1avorI4(P@+7XLCp0H0Y6 z=~LwW&<3J!r(p-|gRKUVz5z)sDa-rmj?yaMy^|zjpGH+aVxTxZm8o>~hP_M;IA$rh z67o6J;AJt5kmC*eJ7Bgd?ne8(^24i50~*%TzH&VQ3RGyisCE(_`&?_4N-_({hXpwh zsmX}%&lD(vCYj!q<+!zT=k284n&got((O)CUa2!=rCPT?8sxUd209Bo?XzYRX82Lp zR__fOkN}xcl?I5d8ft>|x@C^=$vZjyb=bh(oO%1aRGS*K^zmTw4scCrFv>jx_SMur z8`R;i3+|HR8-8N|W)E$<$ybn1(J)!rr&3(a1gjJ-^(W;BBEPG~Zg?4lquI;jmcfR{ zGsT!M!H@9=MDB4j$A6(Ou_40RP!$)D#V1c)13|EaVgko-QYuTm-bcdeqS^-f6&9wa;vLgfDFAgZicm^M$ zdPPlNG+UMUu_3k0uyvD(mX<)4l#2OV6UAx!49jv(Q`+NUJqa8Dg?60@+n;v*>N3>L zH>QQfF%m<`khF*~oVI(4R=Jds-e#Ra(ebo>cAo+rxqV7aJY%E70n*}s0=cCEYnK8m zmm1)Yhg^aC))c3zd+uU9Iq-wC+%JELV^ioS0^gXK zX5dmzB^y^LEcE=I249o(L@XxD+?X zeC<}2^?J7RD%`cTS68Tpdf%7@Wtg{4nB+9Gyr0IE8Q!XVp2>Qs)}PGDnmLqxt8JBI*@%sYi%fkHTvaUZAX=sTp1c<4c;moXYC+@B`X zH0(!-nPb3QDMQqB*@12w`teF#OkC9R4%qI@feS%l-iHni!}CO|;`Q*;?BdK~8Y#Fw zl#K#&yqkKk$)JNOySjWQ`BY_}^&(Hi?AHOgrn`66_bGwX?x9Gg^i5|FHeRI0gxQd{ zbyM73aN&N)IwZ--lES4M`h^lxy0vcEG-zDc^+ya1$_OAEp#*XTq3O<(%Sn{mn>S@R zlV+gC^tO13;%bsS_l%qIWK05rG>ET1C_%Uw_psl+X;MktC;!`pP~|I%RMX4DQ}EZ$ zc7t;wAqi;K>2^|SMTV1g8xczF=fKgp-h*~-JAXuTP7sSPj7yE?SOvc!amMxjmV!+; zCtsoDh4yz_`nFC*TksyIly?#Nj=Ez0QdE_~-zHNnWrn$jC=A>BU>iEUsTos&zhy9q z5X2*QKqSBtKR5^{rw}5cmjc_@2_2w$G=EPu&6PS#f{aPCzuP2&`cNiN0vo2X)WdN` zX@{ni25321gNglAl>3%~;!2u|^F@uF_9VQOk`*q~G8|PVc3LOukgjCNeDTynpt_QdJk*V52O#+VQ>GuuN!0$A-BEbC<^3IYJY)O)?x#_Z4#*s$6j-lUg-sBc0ubWWD+m;?kvkO6 zn^^}8w2sZ(6j(_sgk)r8{ydlsq8K;o0h)abdYZlR%L`H01yO7Moj{A7FSlOKmx6(< z_{H0;)Gb3n@l>?`blMxgTb3CPXSjwfi8lqNI5gzT^_rxgU-vWr1sn(Kp-lZ`*eD)X z1}oJ!1op**yZ*G()O?w3^=jmuG;@~8p6y(-Z@`}=mtP==(CbR%UsoaMi{!6TE+t#P z7?!*)T`C><_G4tOdG2LLIciumR{V3Vve=A)hl_Db2AE$rV`l21OAiBL}$OGzDOr};_JCn ztSt(dAsw_lU+<^mw=MT!PwvsJZw`OscjEeQk6zD%+-m#l8f4s;b^AB+^5e4%uY;Q= zFMMzHoe#bj(}}z{@lbmlUYm^B7x{4Pfda1KZ~IHM43&qn0GJT7s>N{|qwd*_hrnuZ zySU$Bd1HhGU=^S_rih6r2|o=<>~W`1U-&c$?5>-(h%dx!qnF#t>SJgNA4 zKoWWWMpQhDBL73FF??k3f23hIotmWq91shycfo*;bZ8_*Drh%G0pm?sPtc+&Ba)|M4|aP!=4d7jxuvX(458hHNX9Li2|Pp zVczLML^ErX8UVk)z8uHHi>JIJaNXp5E&e_&oqTBKe+FQ8#MqDge(0ie$Lx^ilYkt+ zSQ91bRT|`%1?XR_`!*4Yjeu+4&mEJY=7%>m?eXgO?|r%S`p1JS@-{K2?cyGcW9+lo{EXZ_%^AX_ab6>inbIG#(r=ZI%9;U@2^weH0}UlR;sfGBwi`yYJTmjw$;G3^1#ZFVr*QzjeEUzJ z$D*cM&&ly)39nb4fI|@Dt&;<&(Z{#PMD@E}fc5B1`0>(WzyE8(_wBLCm6<0luCXXm z`6@MDTbE-lxp&Pe2Ngi0SfEkEAc#Lgd1Zp zbiSMHT(!r48S*pF-YjL}%_htXqEd2x(IgLF0pNB1CwJ2sck+seiMD0}VK zd0JRQqxX%k&0-VY+wxAU@NbKLFCvIZtk7<&#!pYhUcdSy?pvlaPJT24qZeL#MpS`SI_1sDkuhl$t!Y736I^ouS8585Z zg$idD>{}zgN@j_DI-J0gzek%VkG=tR~k&^ZWuzZez($ z8-{0m*DL=;B?=76At$RWI|#0HjNK+OJ`E5n^=FW3Cyfa;Gy)8HTsqtOV-+*s6#Fej zFttDA(}v_3s~vg{RhSPC9XfCTwyA_3zKUkenzZ+8Y_D2{-J)iMe{DX2bWM<&2uY33 zbM>B>Q}Zg#(bhgjR~i>q+dkxrJ<58NxT`2#DmNX5d8KTV_}N3$r{~q)FTWl2&^p(e z4+#y6_Mc1Y?XIxrbe4Tv&j+>-t@v*5ZrjBB_kN2BS5a3T86WQyw>X(H z4-~wH_iJZj93VZVqif^Xsxy$~W-$gTyH~Fyo)zX8v7x1(9YP-sd?`!QyohLkLAOQK z4Wr9)s~}mzksAG}VNIr@4GtL`BEXM(z143Fpmb9#KK7X*WhiY$)+qbS_>qCoBb>f{ z(EPm&@+nTFL)a+CrWnRnrrsq1_Vnlo#FMx5*ZvmuzNtvHOGb4dr_ZgT2fLPl54K%< zXW4j9UX}2J6JP}ixzE+JyX;mGbu#J^kI9b!6NUje^K?z5_}>3j?E0fq)m6?*W^Q`3 zZlXf)LFpy5+=s;7*|>s5<<+(bBH4$;AMd5K#phEp_+`Oap(PKC&>i7S&D=aZy=yG^ zTdn|-c>pI}Ze>Bs-n#ziS!d%9o{kste$h4YGMA2X=bOEI%omqtR};S8Y%k?VY>4LX zb8*j;lrRxI9*hLRZXcOZ2MYt$!3PhpqB9yKO{7}Iy>ic&nh^}Zb0eaN>b8ywY3#N`r4^}_6yGApr{D0g6Z`H?QW2fxce91Xy}6!f2{PrFl235dapPwX^u@^~vbFQMbt|^G3iz zq7P>68beiV0`P%6rf3J?Xuwmn$l}0ly5`l%==fU~_Zf%Ah7qMKla;*}z9;v7@3)1X z^Ag>yWuWh8CIz7R%T6`F<3*-2VNF;gYapn!Me$^|22ieY{AWP*6jBaZHSrf)=ZEm{ zA#b)S&QD9KH3Wr~JFBPCkMPk=%zJl7=39TPJi&itXM^-ikSZxDT?HsV^Ym}ux4oik zzc*@>o-6T1O?$xjlTwqB$BzBUdF8Kf7v?>C7WL$EaDjEg>rbD=c%()}H{s2|nQ};Q zaOkYPpSb$#0obfd3BW{QF4LW_PONfZSMxg?OA}vzdJ6l|+3CrCFIx5MR{gc!u!_=8 zV&WC&qG%>Pqt;~dnwlD(xH*SoXNQYi4HoqTI3f9uasa!xE?p0a?)tKuJvvFPkBNx~ zNmIOff=f>*f+jAEgyfV&e+_u8Xtwa+{(HYEd%rJl2(~dqRdwynZl|MNb@Py$_W{r& zoL64v^qv_Vj90W=Fc15ec~6DQPxHRf=di`@iQKn-p!p}{z|FU74se2+%fUOdag|I# zbLFj;Lj*M`zt?kj(++kg+ua@DS)%mYo_cE8TzBws*@)Usa?02HbJZpRCieEpk8rmuSFX*4O`R=5e-a)7$qLf$-=x!IAdEFgjaZ)&fE%P=Z}Mz?{Is-v$C#Zh`ta~pocpg;oo3*6m#+S2cn)Xp zmrA=^o|W@f2QJ^}wRf~5 z<*)Z0BlvxeW7PBh=IB~H0P^MaKTkI#oBltGrQpv4{u8^8FUnke2kd+PkdpkGu)1T6 z+nsA{^>F|*#HdccR|c&lrUG`es~=wnye%cpB;y`MJ^*4>B1&r`8ej#ruxnw{3-;H6 zogg5ia8}`-1-9&&fcWI(=Ozv0xV4+$4Ct+D`qm!Is_+SGK0bQSIcnDM`TyMo2t2|( zR*F0*gz-hzjg86h91Uli1CE)HUa*1PWBLb9^&ZUX~4I=M);R@xh8psJ_p zfXckT3np~m-roJ_?*fuZqmO{f{Tf&I)c9Mdd{4&r=sG*g|GkiFk-RBe-S5WHsmMc-Pg$HDj`dGM7HDL# zuv24Wuk^%`CDq2WQ~lQ=QwTE^HO82ooc#I4_Ept;KAR6l&+%ion3Q4%MaXAoW*~Yu zayNwW@oFXEVY$4@<6WSuT-QQJBgsoFMQx0|dYN5J{_h1!7CyXwN_p-1cSug(p|lJ; z5rjKXTWlBa9jDw*8qZiICIpG^UHbaKMN3<2Y-QWvK>G{5XL>#x(#h1s1mv87cF6fM zB_;IlMTM*h9fsCb;>QE1xFc&bzuHTc^c~CFsweDGyjY8Fp6hI!WxCR_Jfqtiy5D|j zKe52EUWBSPJ2Mj!O0$}NX(i3|KQt5}9RwEZVRxN?Z3Ra5GuNTmhUVs|J=)fok8ttb zolrL~uiC113s75J7sRhW_^8y}$2& z9v;Lqb9St~;$82K!|wYG3>7jaBq9wF^6!v^X4w9)Xp5a%1>)>sHF|#g??ddD-h4&Z z_g{k=|6W^6s#CZs)u?~(IbH~;NS z0;cH{WjAQ{UC`0$+ivio^#4SiAU79*q`S#(_0OGz*jTx+3j{B@}T-{j;hkCj2)pD(w??7JBlb%7Y($ixkU z_FP}`eMU0psGIwJzJ7ki#kz`H@hrhhW{kZ3?2NDezhq1{geVTJw7BEjZGvTG8|M5b z`GTDcDygb@Ha51r`_(6Dl*|_ug4^0U8t(lcsjb?e@RKPb9veGtHuuxY`aYR$t|P2C zUyf>)l2+VO4QOsL@#`h!Gd}m9Ez#o%ZFHV~$?2>G1pN1^_A2faQKOESXs&Z|b#QY# z`;yhsRIz!)STQ;_#%ELu-1o7fDkXerg{`L+`txU-ka8Z?&rD98UG@7eLy zO37=UH_wccz~E$PqiE;i7XOwO+nG;tg-hHxwRGgTl``xP+4%k2m02igN?^vw*lqy` zCiM-kk)gC}!=3EM{AsTTL{&e$jO{EdEHn;uDt$eW)_i0qe~_~loELcVy;t1Nr#7n^ z6{pm2ba+UejkYY~eL)7ndw+F-h=XU(nwLCcA^3Q{rK-C4cUIt5(`4XI>+hLK8Sh2Y zK=quu9Au07xQuh7XOnVIt{&CQ?8KAPp8F|s?m0jlPbS8vkz*$*G6!{fJO<|ztXy%i zu`W*%2ataDlz)XB(M#r?FW8tBqci#wyP~-1+0;L5NoPAvYi5=FsvmAv(e~vk_blmY z8L10Z_VA_1*DqyEO!#z|)dY;E*5?8ns9$xJiThXS=+vZoG(MEq`KW^-Cw3`!+U2$H zBh%HQ5b~#9S5k?quc)XvYA62Bt`vK6=AVIv2;c$J`QL@)ei4iJv9b5AJ7y#M#>($Up2EL_L6dV&T0bt9 z4h!vrjHRk6{%y3-)cIrJ7#@SlXFp-GDv-|n&v2ZT&^G*tGlb_~x5JL0|37Ax$$$(P zq#+l=`@i@45bq>3KzUlAyy2%JM?ErSWPke=@i2V3NFj}GP_?KRB0+vOIXM|xQB$)o zPo!mR9QZIe_=1+GD&*p{^6==W{qvaWhsRDjI%5MwBlGhYCEvb%3%v_dQ0T^WbaZqb zsjRQ7I}Cv8a=$K(CZhl20rmFw4*FPFxMm0y5f>j;T6U0?l~wTrH@{&hL=&B?sebT` zf*2VahsLpG>+9(3zl)BJ7A784l^*AY-wq3))0kRTd#n<7=Ko>CnhHw>;ePN zqHu~zOQkz1Dt6?FMiv$>FWTGNWo61u_ywGuon?A4OifMQHa9j-jiDl9VwZYNCYNo6 z)Jj4XpcWOJqVn>K-P6<4P+|D#;pXr*XwC=_#`ltsA3stOqlM87gh5K5o0qrTCMqi0 z;BTZ&mE_{$GD(+PUw`xmJgUi%TTrlK9tfbxGX6Q0pA|EKu@(gwkDucbw5*+ee+Rh7 zqMEUy?=^~Uf<6drMz#NgX~6xOzXXi=}3&+&u`MZwc1`@HID?^e!vXOo_!DqS|-Ff{ZS~lTGdTbqwVL{K&dG+3hQSH zEK3Rcc@yXKC#U@H)6Gip5j_Lb54zmb>(Pv^dvTgUU9hV?(yOHp0>Q=MW)~X}n zYRWkjOxM_*Z5VVo=I#&6FDlWRKFuZMZ-n>6QMw#_zL|o0*KxuK&GMab(2zaUg|@9Q z?+S)q1~IBhxdm?;<4A!%Z`Dnji5b;9zintgEXR>L`I&mLy?9w3 zetEfjwIF*uXI-$|9n*qBBoH5ryj6XhEghR73dwQ{Bk6vT;O8x@x`G~Ka)-%YB}cJU7PqjIvXWr;{SS{0_x^L`5D-LfyJE=QVus2goEQ zVG0{Kmp4Ezq}}EgvBY2h^J~7QqUq7y9$ZGwU_W;fBeH0Ue%ykN+e(l=Ax(WM7Z4J> zR?on^{03QovX`nlLX!orNOa5M;VtcJJNLYe5p5vVKP@;O5D* z3?t0IDmD%aY?f}-gN@1O9&{xok=5z3b@obp4BA}*-X$fS_CIo&OGjhpg&=MIm?^o& zdid5t#7TzFPYm5=Rq5jH(1p9oME8&IWPjl?A#lRw682UnFQ)@3Xci5xP{+Tt!j1 zc6H>Mj>{wP=0LXf!Jk;!5^KT|o4o5Py0_(I2Z270O?`iRn>B}Q2MPjta8#_-Mwi2pXie!u=P&d-AA=s#_oB~FY z*sKJf1b~Jv1kHK$XKA6aUWx5s`tr70=-Gw%oO^eLgrmt{bSst-PHRmO zy=0iEJ)TU#v1EU+L|b2$SfE#~kQ8#Ze48%e!h?)qwAj&4PbC+UqeS{~+2TiWNhj|C zVvWlZGg*btb-WUyVukfpf;=V{M!KQh7ee>k(w}EzB#OY%UGU@bt~)2e1xa6_2b1Gy z9Lo<&j$cc+S}7-U$&5b{3{ZPyJh_`j|(Pi>}YH)nfZImi1s2CAgZo_7l}oA*De z)^h%0WRh^!Yw`HUj~@qb25iU$ZlMu|Y_qQDU&P{a!t$*XvrM7TZc^%8n7`+)UfL29 zQw&>r#H>5DYmXLFvPKZD_>>T~Yu8&8b3hemd-#f8Ieg7S&PY3EWlvZyCYNPXFDo9Z zXPTU{#D783MeD)Kv2rVuul}b{5YwiTqRcQbIjcaK=tZj!*Il2=Fken>#C|loGkq+N z;&}7>Sg=AzuhTW#Gz-;%;tSA~eZE+rIWCHV80q;erx_MHVse*!M2P6^@7pgmsO#1Xxx%ikJz5Pdfy%dnrYWkgGw6YeKZ)m zzL15RutIsWFT$J3mGT`Q>dR@f>KCa*N;Bf#W+E*C}z;-{-c1fw6|WwBAt$RLL)=u_p*?6 zpBim7A)~l!X3aOL%`{dN9QU&STIOv>@I@=l>$0kEHjgIV{WxQSU%A&+jkjHL1Xnb! zmzIn7+6y6~HLfAbO6SH&c*dFSk02$HJ44RM%U6-d zNCws+SAju?Q7b`c1lzh|=~Twyx>7kEuT%^PbMwc% zyn8)_57BtG(pOiD^KVH#n7nX2rufj_&|F4bgz40*7wFzdjB;kOm`t=yHtQp#_BoY0 zN-6_M=$#KKM6}uWSOr0|tek!=W6s@SM?BNL7Mfgb9teft+KPRKG9QFdYV^MwiCHe$ z>94RsNLXCP%qbp1@>fH$@x$WEIq1;{oZ<;?qU_Rvv)~ZsjE}hH#R;! z{X((7Hr(L8d_Ooim_sAe{_Cqd9O@bGtE;uQm_Xo~5859}t>R=UK|8xThQejGzXH~- zERj2y{!}M{tFUMGBTq?>GbFBZNrx^;j_TB>bVDJBZ&M*0Gg!=rv7*~atCLKmctc6R zakS7poJFs#zKj-|;9%MW{kPb7ObSA-6&x-$`;i6A`->5fOQ(GipZ6Yn6nb^G_)a#k zetmLgW}}Nx!28SXH2-sCKi`v2ist4yKo#iZLs|<`=%NuM59hmvK1t0pQi&meD|>Lw}OY)0Oaa;YFM(2DRe z%rmFI2~OV}O%B|{$`B;5DeH68tD{R{sI_bwBJOslGK}&OYhZL7jp$vkD-KZWyOhAY zxpylh#<+?q%gWC88ach`=6U_K^75?Xs>Soq7tVx|i3$CjlR>-8;G^mIeD^8K7|Eai zDEt^7KbhYRS)=VhMeXdk3WOjNvI2LSj;5?hg&O%x8b^K<>$tdcT&iCWHu%-UpG)7Z zZS5{e>SE^qE58vy!Gv|cqiH)@E)bivP24-Nl{7UqtF_(S+&rdj(x`En++o_>uj6$z zBtNXLuXEMEFDxuv-`wP>R{=l^850l?5Wldn0Qen38eT4~rQmDj{~IKxPTH`?khgg$ zg+B9|Hvhn;HL;kgvRt^>@eAC4w^;TOELB+boaQk@5^PSCd;NNA*SyobITx^5o-DP& zrTP2!Z;#`7zq9em5S!6_CFK;BZ8((F8$?u)Q2lxJ;ZYg@`XNuhzkhf%=cUcx;V>Yv92`(-HUXFz|0R`^_H3uc%>w`i>YK`mTwhl`g{*pF5s{P{ zTmVcaTAibiN*^BI(Naqn%tL|fx6wL6WAN#IUL&fBHhyX=2KLSoZi}nKX22RE2dLz|Or-X>DrSR3!4} z?&!n_D9zOD!jV{w?GV}t4x|athPFN~ej3q*147%smvjd%lFUen=)RAS4`8N%@(n*d z+!)Wk`}8vz=;u!33d!N&p?EUuv4grKQ5o zQ-YdrrTr?Jo24}Gqz!!3%-NajxU4(eoJ_D&xegO7yl}05xl8hhFBkqd_2cLwKdQg` zSW)pUUn#-c*SDmyQd|o-J`hG58YUG;W#0oM1U{nNu=>88UEymUgVR3o_Pq9Xu(lyM z7z_f)uZ)2GBmowowzk$^wOALBY_cE(_rg9r{-P6~U&RO1&uReV@U;yM23j!{*C|7A zBMWO!&|eVtXAXWtE&Y(YN>%xNe?bYQ^z(i2Tz@Km-w zZZbJ~krLFix7@AAO(e&pl4?3sT(JwPx&h+}`0o=QBl|?vEoX~jH4_3gb z9>03T-=r8A*a8fA@#LG|oaX?a)fwQbL4JI^p>>b7dGwk=1C8pNt-}4*%}yRM2wJd) zD~}4-3_TqSIUUPsKL0ZSCRPA*G`6q+h(?Xz;I{gDv4(lycR;hiC>Lagoc^w8-r$}z z^~;j<`jsx~^d^ds)~L?$H4yL_AYMK^F81dDiiB1sbS!}2GA*H1^})L zECB31DhA3Vo&WqHU9>2G<835vUH1&1zrL1TwyDiIGfB6ep{u*U=uy92Yn57`dB<;c zw@=3lfe9tR`g4MhJOJz9dVX9 zz46pM@!&c3`U-#nP{2_jtwpBM3AnE!+0xOAFPj}0YH_F;!% zqNqPwk+)QpE$=o9^;8s+x8&jTor+o!^jW;3BO;hfsw;|JGd2edq#qDJYURh|WI~oE zI@I@g?NXRrD;1*2M_i-(0^}*U=se}-jJteaE*Wypu`?t+4r!S9tdW$*#h8A@9fKo} zkQCwTJ!?YDVU0IoA*5_UV!IR~6Ma{znygLCpbIZiDV(z!Q(-ibrR-kX5OJ^bR?K`A zL&}m=S&h%_L^#n_f*fy`y5Tj#H+!JJ0%j#bIY3%<1l(O{C{vU{f_4>0yV`_Ofklz@g&}Pl-vGfh54f&RnsE{5D#nfq0IJRUl6Ia zHX8d5zrMQ9%F-pol?Nyq$W0-v+rwq#1%i+jA&Mlf=}!dF7I*ukCP>K(*20f8hU2G-&HgG z>Wc7vn_)U>xo)$P2}a9W4m>(32BTw@^-l=b`u5B1UJ&nmmKvKA{q3`{fUO?l8`7S zl0KEaMgR%@E5?@%&shh%+EX6zJ#8pzS6HPw^YgRela?9(|_WCoRZ`4Nml}f^}?;LU2niZD7kF}}rjk8VdLd~tc){!{U9Z~|T2lM-8mK2;;3m-BRcBR=*{ zpNO6{5`%!szinuIrGH&P{1Qj>G)aY#U@rX2scEd4Of;qLP|pxo6_(1(r{iTdnaETj zg2j8`qW%SZPJN;IuyFJ=T#wgs$4CsOsRZfD!7Oe=K8*8HwPP7YLg2k3pfy@aV`o84 z5X8uLniZu%wca-*q2ED{A|{|heG2hGNfI0&Jk`eB%+Hxjxki$O52eW;jU-gAi&L4a zgv*_i9{q5mlZyath6HZ<;Y@%UPf$0Gy18_Fgipu_)C46mW5fhxa1)uOkB~W6=WD<} zl-1PKpw8FiLvQy@p`nxS53S6|Cid*${c5#wIh2fC)TFh%$THZhun4~>;jm~ddCg^s zV{3u_VW_&VM|f1#hK)aLNeqxupsHmBO$9YyuO{)&8qt0fr*L=Bhk5R<> zWeGoTKhYo@{iD`*|AK@H?GEeASInz)J8<`85Atw?4@&!Qc`t{UJZHIR_PG6~gdyD4 z*jN6NWpbl|;L4z*?LeAJK>JWSGw$ScJ|SfVB5iBl`fxm00!sS30p}_{r)Y*0H{aPS z!Gq?K_}BtF08A+)_$Ja07vh3xnD>Oq5(qp%D*@Ox_EFt9Y4u+R@Kmz>IXFCgIz39K zifRCC+NUDT*BtjJ(y5bgPnLVfV_i}rRXa6jyLy=6VIH3PAe4zZ@mleO1?dAV+_qvY>&uBrE!u|atniJrDoY}R zwfpYQMQOY<2B%K4)YT8mohEh8x=h<&e9R-+KJ{6Klu=1<=m%kH#>b+BOF{9>byUZ> zDm`8Wv>ounDX0p9ORbkBJ-!-+@oQzVnqj-BdmDvw>O31T=h|=jTxT>^e5u2Oio;qq zB&j$1Nlb`>iz$9xNx(?>Yx^#S%s8{*Esbl(Y7{k|{p4W>#m8l;R!Gd03FoPn^=80` z>*lGmBbRb+q^P0w=9_%N>n3J7oQV`T>zEqG-QOyT+4~0&qY=r;Bp^pQmQd@w&UCZ~ zkodLn{zplDh6h&UEIkp8WR!xGg$R)!2tAY^Zyy6u|KoEyC+7+rNa1|+lgb+b2=Q<#M* za^Q?3v2wh;x!BMuD4`9muW;@c+4N(acxKz7)DUYr5DA!?6HX6Rd9yIqG+nz-XA&#dqud&E+mM zQ}saaW^$jB*kr8c5_O4L%pzEn5!?CmwLfKaF6C~gx?zHEt)+~{{+3Y$iQbU?Cm(Aq zyEECyVBKIA@!95=C5QHu)lP3cyNShH|11RVivDdkNacA9Dx6`d*OPL9fp~5yLFd*- z_v(PhQY5;#S!Mj95npzSH8cueOq667zFEadvw|;O0i8rxUb2ESxD0J91%gn=*|K*v zk2D^Hk-Z|PAj77oU}}U*l}I@CYP%@a{xIs*Zqut&>J2A!DMi4ea>=G6>5mp}Sw?Mn z9Qn(8emQw*R81Exbe@AtxQ8+Fx?lmTl_^ISt_FuHLPIfLVnh`r!rhl_lZ~*qFiXND zVa#MW=PaKgwY624Y0zVgCt`&8XWoT}#TA5*w&FWusfMe<@x8ITB$`^`zBe#> zpz|2AS6_$@aLSlEl4>7dd=0ysOT+~Q|G+U!a7}Q%AyhqIsGv`w96iHID@gxZjY3L2 zJ@Wj5KmIaId6?WP;DsD`Y>I)OaJaCo!aL!qaB(=>4b0Ct6;O5zQo(lxnea#OUKqDK zp#}CwXdUd9yO2Ad3NwZow(gAW%_{uQSX9X~guE-L>G0vOfv_Do7aW5M^EFINkO2?M z5dIq`xdiFLxSAq+|ziQ`TB%OBwP|+0Kb4iReUh}pr$B?H<*NBgkcNdW(`bKMG2z= zQh_KchY2kyWU9&CfPTiPzY#(r3ZHoS^5y^f%EiMo@C;<+!Ggi32dhkZ;Pn!U#^((F z1hGo7sRN|v&_;+M{tpZ`m=qbd2xdz-KD-Tn&!`9<2p5Gnz}es!iqL>u3PZd^oCS!k z5ejGz7T0QrPIzT_5)8kJ9!0Q+>?Xmy4WnPmjlxaGF@X$XlVF&^i9|43!aU&HFnT`S zTs%7bRwxrjD~!kjvJt)w4~7@OMA64O+o!Z+Z!!5$RxzMB1qlra6QR6eSq0y)FG+}g z_UJ)1F_XgZ8}SXHYS?lZ@nH!NRp|Tt4m3;NW0~iPw-QAumlrJQ=gDkx468tsLXBrErW0)))2i_lAf)kA#bB2CfM#*5w^pNgD2EvwM zOe5G;m}g-Wxg=UAnvh&9I_ND7(=ZJfvpX4B6~vAnTOB^QHh1oyZe7?a*J~Oj_!ztm z{n&b!wwwjw9tRmj1}73)5QYtBhhgg`KR&VK&h>otwd0JS3Su2jbc6EqPcHc?`gH6$ zyhu!LK2S;#1|KHe*@Aa3&;sKh=+C{~BTNCvF2u~JDD0~?MA3K~LUn@}>E!`t*h-6< zn@^RLv;#27@kWtFxTGvWo0n(10(oz*bzYkQoKa^1M8Gf(zdO@MyoKSrH?WLLf`N;2 z2!hdT;S2&ri6kqW4TO8<(oB6JoyC$NFbw0Xt%6oUTEaSEoG@Hns8g6EoZ$xM2!=x# zE7z|gw{QkHVcAlQ9>y?+HcayB8>E%k8J8c!GfXg-XbOuL^J`dn7(Q2iFU^uR9JB1# zVpEs{Tx^N^2J`_`Hy3jXn-_xsB7fydF36%4#tyoJy8zL@8KS@2ezvn^*&Q8BPfqS3 zeBYZa9EQ=cm||8bkRjoLIIGPEmXy!uik*$ohan8!+JioOGZT-Gj35$&0Z#k_gT*Kv zgD@A9I4mVq1u_Veu)x@YF)yw5(1&T=h%7LH$u22oIOkC@_s`hIf`Sn`8ifT@$*Z#nYj^Btib8%Td$wvanuYtlFfokgTFdtn5PVbd_V z+>!X}7amAf>IjcG4o1z;x?4NZOtx4O3!Gd4E%Q#(6KT{ANcOG`Ft zYHBP%H70>`S8I*<)G@;`0JCp9@P-eox2H!-OABUV!gO!8-ld?bimY<-5v7tgfPQp< zHK)0!r>6%>_1s&!_gu!qa-ElY&VKXe9WJgLckaaHnzs0A8yd!r z{l2w+zCT>3nInfaK0Xd^J$Ukj{vJI&Jrxy|MV%2oH#fHx;HI#mqN5*!nvIMY?%lq9 z8`SZAWCW(w5pscs7xsA3uJ~1?rHI$;{O&)vpFKm(K@nh0z_fwWX+a zTa+UpBm_Cv?IOG3Y$9fyZ5 zspGi^=56!At*>8IpL=?Gf;wOYqa!_Q`=HIA!4fL`_@tx{xw&CeQ-;s$UFTInnQ}QP z91Ka(ph)n5#C>cATEM!6&bl;A`)*Bnkfr0RVb&vM3H~Z1+TDrY4ZN*%+*CEYn&qXV zOwV^(0pqp6uEiS-DGsA8!aUlZ_4T(oS=a~-l5|-1?0IS-Vxg2C?wor=L`39++hTjK zwzjq}teZUWXp7#(6}Pmfr|zRpN9B;@iZSZ2{HQd(U6yVi5{s}NX7_*>-BqesTC z($d7Al#T~TvXkMh>sOdaetiG_XApXB?A6cJRrUI5=N5Z?BclLU4-X_=&afSVOH^Jy z)KgixPbOo0a+0&sq{&ka3_EScZB9;s;o;$&vdPVD5#u_?aW_DDi)Py9<>le-|M|nw z-_zrqH+^|=wx5uY(BC+}qm^;EHNBq$RN*4SHYq8|qqn!$nuJwuF3T3!Q|bQxKIc2N zk470TQ&r2&e8%;O@89-X6%P&!yklZz?Z~zTBeL-XbhB^SU2&9`lXI;GJ?DQjLo(~z zX9NNv1f1HIdd<|-RMN@GNou+NrX@J*s($+uTDL#G9knkT21RAQdZnvY6CWS%3IHF| z*wj=ytfs6C>{f1S+8r5OK^lU_?#?y1nVV-QDJi*t4u;C|@;b{8`R*>r);2Z*BFS4S ziw7+eRn@N)8F~f=cI9Z90NR;*==<0hEn18d_97w4duJEX=ym}GhT7B6(>HMn2)KDZ zefpCnAkVc40Vn`jfb!=L6vSVdE`ipzEF!LHqi zogInr9gpG{^C1^&%fnHTPjHPAE!pnWe~TZ=2pgx1RxVJA^?ih0xu~9WAsDl9q&4n8 zvUwmu;%3;m|NDu~n~u#QKgn}5`NAg>dvt=V=_EzxG91tX2oL*bjCBa@QehNz6@);g z7$#~_7zL?<;22GXJA!b9T?NrM@(*)_D-aS|Kt*bcFt=()pknX`gt%J7SoW6C`}WJ& zB@l7G!7$>H@A)|Pxltrl5Dp`Wa7Vai0^W^88o68s_t7q_DGX-5jW8)AdkCY&$0))g zoS<4fD7*bKZV7~wFA+REmJcb)i^8db&=@s>haV-7!Icv*w1_3_Eis(K!fAYS8MGu- zAyAQR8u;zLap)G8No3RKR9ooR(J^0Pbm1&1VJ8x0h zy<~5)SW*>ioiz`Pp!d%y^kxU9JuR@Q4{eBd!b1N9l4@S%yd=ggk!X7vE($k>Q--DH z^j{ku-srLeC~Z_fHGK1LP(fh%?4*8|m18@2tCd>rH2h=#H`63z+e8>yhpw0tQ*_G>Ls-8jg3Y&oQQFo|zpSxlO^n4C z@;q$}vk4;|0n#cA#{s`xKA`amT5y6-h@%ri#ya56cn=@c`zROKljRr0)%KB7W~iw> zR_qAx*zgf6Ou_YMZ6xMW!ei5bg4f!y%{a1D5xIkRZi2rRXX&&g#Jd<0rpca@G3y2S zOU36a=8n7@Q^76?gNBi!Tkn0{`oeIGCGdhz@KkL5D0G^w&;7vY=6!En@(Dv&HD(sL zO`-;R7h7MV^7mVm1rX>S zGEb+jd=MZj-@C&ci260)w*I4ulGfaV9AVc$X^-(IEEo*To%h$k-3Qkd`|;miz zMfF81%bhI=bI8fQ*!b!g^7*6BJO8uQhV=a*6B7;EMU9-^!mIOF&@+-fYX(M!XNM;S zCx`+8_bFzqXLBs^zo>;S{X!9|9c_cndYkwSN@O(j-)0bXGRP518Oh5#zPnRH#kHQl zRng_EJBcH8UT^cQUNkJ&Av1y7v6R*#Jy86g2urYHvAX_G+cZq2JoGDTxH8OpJ}C_} z9+W&fX@6F;w!DAxWh3LEvOZ2hi0sRm45`P$k57b1S|S`+gOqpA&8SR_!bE#iQ;D6# z_KpiVKlx3X@iU7)PpE87k9?1n0fcJ#wRHEB|AG}edpG@ATJ==Eb0)z%ap&dkJ37C8 z^~D2Lj}e7v8T!e((>l->UpEx@C(w(+^lR*^hi!;Kx&4mhGiEe~B#edZOSL*S#|VUu zn8VcMsnt)un>+OH;bhp~^!4%~!~lB9DJ}m)u&I~6e_Wg*;2vnAkutneXzfoc5o0Zl zQHCuBtqHq;VX*bQeE{hS8wh90<@^W7{=`xg7TWWWI)`j{y2-A_>I*^p$sc3>_CHoY zMegZQ>y;VRGdRV*_QJ_lcwmSJjV6Ncl3G+}I3PR%_fCT75^-8t1d@o8)(-=id~r)k z5c_A<;!=1ic>Sa67JYWYQ%ydcHt=5{j`ui<1yd~czk11w}EdDZmLXH~z0SJeyoSrLz1+Rl$E0 zUH5rh8v@^VFZ;*p-@ku9q;hi8U<3fRM=mbgOh!UQO*==!tBAQ zhKDSrz}ey9WC%4Y;LhZs7vY|{1ojxZA%?Q<%pHPn1a%Hj$8II-*>gEtkwG&BqCAs- z3IngOxYdHrcAw_FRv(#jTl@19RftDSX!8fRGWDNKxH)S`#-}!$xkGr?!PZ!+xOqV8 zvzlLC$k((wA|0`a553;szU5ccCx36Vp(Da!FJM2}H)rMG;E!%`YsIbX@9tHe097Skn{SHK4Sl*D%z0VzY`-QaM$hkOdAha zzGtxrac$WU&op#R{J0Eau>{&?oTC^qKcvMn!qiSh`hFIwaiU zXG?bQlIz_)))A?UZQyKEM}E>10q#2ldg?jPHz~CDJUm^EL!ZyQH(|I{G2}{S=VGO; z%(yN)PE+&NZP}5~L!l3cFU>)F}pQ)PPXBs_L=%l>4%9EHRDH>+gwam;|G-Z%j2?a`fI4UBFqs%E}rd6ljGx?Y8cd zfE_4z%GANnADek?VD^RCZ|m$3u%kOi0_+3U!#Kghu2@e*w@m3d{k+z zli!6Hv%*Ut=%jE|b{he#?8g#rvj$R?h}7n^|EV?4j7 zoM9XzjE#&{7@ANW2<`n?;cyx*jBHFq_*ytM+;vHiEEifG`8L15BUDG}_~+$Q&Yxo} zl3OHw3rLL8w8~%Y$YalLH&@`AWmrfn<*vAO%!1QiOeX!TUSk1Kr_UsmGth6xrfWZC z^y>UlMtoUP?zF!@GBP?>Tf}>JI!nX!Ztye@^uM(LMU|y%`^DyDl(Y(0`)6K5>IOPK zjq$)HW4v$?%ket?I(^IbQ&{k+55m^xl2Z zJ=KwDvwu2tZnIUO`^Mn0y8|R|Igd%sKfO}^ETcA2B>U|F(R|>~s$6$SX!EgUG!g5d zx%nv%PrUx#R3rWm{Yk6(bcvSZ>7LttI<2+o%*KB1LrIf+aPvN^YNM$(Yw=+X|75)Z zeY7xc72fu8U%dBgI9v1R$+XPnIac#eZ_AwqueW2B{Cs5KBOh2Gg3t+eEicKW6+x-@>kQ!l{L`bbZ9KMbSo%x8HyBXDd*n;^&z*Vo2_s@iZNHq~ zb~dbe5q*l&)zw|ZTM5>(l%s;=`01Ys0Y}QK5tBNFGVGJ{l%Egu=Dfulxl!V18*)iZ z|CT=Sq?@^?w_nI*-cID<0}WW=n1_t6p3PStoucB3<0i+abW+{t`iO_}pPXEyiy!TJ53+GYqHYt`MEx@+o{+e6XQHZVZ*k6RoRyVGD28DjB?BYh z!~2JL4ucM`ruh8W(GUSBD}=vic&BS1+PXe{x-~?&#cvC6O-EaE-O-|qpSNBbybmk`lz`-luo-xCj8GOr4HYF6~@9{C-PBhDfQvU55KB4;ThX*|USE zQ@3f!XjtMk3UA5@WdldjvvyeRbjH8o=!E}uH)lBZMVduIy=Ar0=iMC52eH%*1L^=C zDi=W7?rBm+n5BGi&i@jXX8!q|&f*e~b?>UDwzkeWI$my3+MX zz0fZOefNkIm6g4CTl<7b?Yq6YWP{jLN=kE(v(#7dkwPb3|u!F!VPXGF{|XfC4!}==kVCvKy!7OJ3~f#SrW9ND6fJgZLGKb`$bWSs@B>TFUSKB9goA1 zFo7js3t5EsgZ7nVX&k^2%3`dO=Ub10cMBvmfZ= z{qmm?vU<0L=>F|xLplF7C^+;%DE=3%rCaS7u++S92Zy0;--7)7Q7$fqs2dUYQj|-V z3-|nG22nvAlzr+J8-c9L-V(nTcw*k|G`@KLh){xsrZkFmW&d=qQC1`_IgUe$BV#?? zKHob3%jb=r+XiM`vD!C5Gq8RW2lRMohg;pCe4v;~B4zQ@3G8b^6W6Y%{xfW;&v75XY2(fywMjvxaV*ac#^#Vz>le3zT%*(LPZv2 zvtlmRz|=vyF8QR(h5WtUhZ>u1nm2rA-}W960x&lLj65AVO*F?ZZn>Oy>sH^``}TU4 zz=$k7W2}3|V$KliuA5!>#Ka(d3$HJcGU@db?yxrhEteg7A^GQY&9Xjy^tVszdBZ~{ z+9*~V{Aw~5a*Zbs>BUpP8|QocpEHr*0$sBfq#Jdtixqo<%LNm+AX!q%RIP^HQ&m+B z^7oej&Jb|TfTV{V1Gt(E^@i#@0YYL_pMVVV9wIeEDm%hB1z|#{<-LTw@B668rl|5^ z+%Z-o5$iS6jcDP1ehc{#T)>L07uCW@;n-zLwbDW!LPT^@VbRSvm&b8eN(JL1h35RD z)uXjh`PY*J%ZL6BFJHbH{Jg1bRcG`Tb)X3JNHN$&7EDp4%ywsI1)}ejkQls}6-Zk+ zX?yrklTAu2)AHAKzd&nV*Nyv5BCUppuw!a)28y!7X(5IF-&eCN~mdz<$|ybia6 z&QntGQP5a1NvuI=K13FwTLX~A5U-cQYz zQr0t&jGD4{Ss~`jR>s5K_9lA~h8cDd5v+vqt_tRsXUB)fjkX8l-W%WJo|9uc8@7f2 z9{VV4Ik%E2jzSY`?oD&fMh$U*ar?aWJ&w_hRULF!-Ic|QQs82EpcRQ=eD(SjShDi% zO_FiCthE?Oedt1_-2sy#ujCxWt4?4?g+Tf@Zk0TGsPV{hWk$XA&w1&=@5z|c<^Y;u zQ$>&#W$>EUCT7;OjOy2OF?I{^StyMX$ORoM0JXw@h!2WKuBw<*P%;iF8JGZa<6i9- z%cX~a!e-MDy-D`=<;&+9g}}9xgkD_|F-cU}f;~;tOnD0E$#iTCRtpTOnUn<>b5)jb zEI|OQa?F7||K^yZD3c{tZRV2+j7f}X82huT7*D*X_KC2Zz zYWNnE10r11O{(W$R*cF>*m#hr$z4!q@gf$Tht>pUDHZZ(#m3ir95=SaCZ(X6u9L0q zVf^pEy@|N#dbW=jG>)?+u}nRNsDF(AKz`|vTLq+EVy6Xs!Jf0!i6%7YobB#iVf%b8 zf-4nmV5?WTdsnOfkE^$U%BpR`M(I*Yx>GVT)rNn*>>kX;(&6;Qn~B;=Fr!(3sIgJv&Q%EeeG48~{8jIV znt(?m;uP?AVA*fq4oi7i>oyLhE}_Dbel5N4Pa(CU3 z*8gr%hh|XV9%y=3pjzx=u*JpREa@)@uKfO8O*|x_LVHp z5oXRtmjhh0{hqCEAF$@~j~+rZln~=S3%&rkfITV6CP_=h>^_7K81|j;L!VV}Xvkc3 z!}5+7q|BRJ+inn((#pOC?~#w=-PV8wpVk-BTIK9ML_i5Pqb+NAtuJVeFpEa@o6ZXe zS_6b)|H3YP(kRIF{@^ZC0|h;R5lnoSFQFZv9yuDsc3f|L!Syxy>rV3#`qCkg44R;8 zzmT3|F^I5VGek>;41De&sw;K{mOc9|?>Yl2EHr{FrhG`t9t<1wV&Vu%!o7ou-eWc* ziEhXd&tOg0SE|#b&g|3B@=T)j*S56{@>dm6W52rLcfFJULOGusuSePFM;vBdbPI&)7IO|iv`QT~A*IhkfJNV|`~h`eg3~;mhD^_?@Avt+Ckz?D zCGG9)2f%_t%&tA-cT{|ojQ=pKfa8Z|o&vGvbRLmksLFroQ;XOIYo`0eA<}NKp}RQg zKId*z?kk*d!e&W56?(PrOZ$1T3zF!#U+$)KW7)Ix!5qvTzDp!Fjz(LO)G{2 z@e%9d#ZeOU_1x-G#;7UZZ9X^|+(IBte=2}`0XL(|&SU4wft$DOfI|)eGKg&FrD7p| z1}p%6mae*i8jiY&{y5bWTjEAB4nt&Y@3Juz-V&3MsXl%B+F}~cF@<=JhsZzd{*u}x z&pE)kz{K0g`TjkgE3L|ik6d{4QRgGq{oEi55m)l^>H8Sj4L`a2hF+w&9k!VN+;YUj zzwo~Q_o=ifSE-V~lZ)rjlJwF;1q?;kqx0K`4<88@itlL&nG=RBb7g(X!}y{k=Wa$x zrsqv4Eb4%DdQ6uw<0K;da=Q|dx?BuP0I+)naG2oIqzHNxLjmsK^_Y*C=-nr-gEf@ldTDLtj;A0T9z1*6rxeuZ;HsjwT zfc>=#89pLX(l-_vdHMNCcT68~@t}5c+`izjr zRNIhnSD&DSw1fFY{-l)Dk-}XUsF4Fu#LV7_79?W_#E!#nWzkQsuyJQBuO2JN}&4(HMB`WV31u)P5S4 z_v{Y_H60<7$~cw-#~G9i67u{rTC2`8bYSex(X1p|W{Xu-pGnaZ%mhKwq*|Z*Dq?u8 z1ukX>aH(y+b?azn8iqzqebE(`HiVV)(HY}WxvHOb>bGd;&fr|$u%AD0Kj8)QihqoN z-lfj~#`?8dHsQUs;dy_}cq;J}1CdKOik$4q;E!VYHu$aje4RPw>5Y@=_TIbJC*|jB z`962f`)5a}*1OJ)SRQ!SWxzm9aFMx)+mX@ro+g?bTu|jw{|FW>BrJ)>hDV2%A8+#h}MJZj0DRxbb4zT z9R<002Wq3uplS--iMMfaE#F^^v$3-`t{pgVz?<6%!PY+~S6AP0y?_1|Mw^~MB2@?h zRxADXuE+m?LPanqR559LdD7=$GhPzMLZ^84c0%8aTx_2)*=D3HT}FY_uJ{}&lfX30 zNaAWwxXJ@R9E0hBhr$t$@sJZ<9G$cZ@zWSpiu;CaX?LsaIADkZ6-bMXmj&+hi`eU5 z`Dt~5SG2rW@_f&Ii91=rR2^M5JLLC*T499=b9A)8Ye>f}Z_;o)t9&Qw;qv_3n_pk7 z@la1Ql(j6#AKW!lY@6TtHD935@l2)pQtIsCBl%X_mmgeub$1=l2*Fd)DzzoaAO4c1 zyyZD(Jg;4CL--K8J=LWL>9v6EXnpEQ;5*HC+3L`{%lWspkP!2;zvcWgV}|2GDX$t z`yE?>Q<<#7oJpnbua>KlrV*(lYj91hU)IK}3`aE@@*u((m+7wO2JK>ao8rMun}$ov zcS{1a$i3Pku2(IfK)BU3kMp_k^W362t33xSJkNu6ClHlwesMG*+cPT6SGm1=ZM3Q? z;8u~+^9QpiyJyRU?_8AVHhmVnZ^RN2ZVZk4VpuFJRHDfvu^>la*;5~`k2XP3=Su&_ zYsPJniy(CN;q&L<6x$ym6%|4{jv{Fsx)pCv_Lfz(wI9mMW4XDx>3W|m1^{2~*|SbE z15ZCdaey1q}`(Ok!L@GW~I1|hSSr+y^h7;w6LqxKhzVN1Xvp~V9 z@!PtFhSHTJo}W`Dv12bDxc7>Z1q=m~#mV+Z z5v;cuLdJejPMx+bnC_HH#9eSb@k@e{Sq<84?_l!OYxPa>FG&KCTGIMiU( zWCk?563aAf9R5PY;$Uy(THZH7;FR&T=HvQUNy?s7!zls zUze2GgxR^{=Cyu%GN^MgD?}sb=2?Nw9ML@CMzVkGZIlYa5v(rc`&c^$Fz6se@aW+~ z?D^IR<6oH%fEanlyodUcfeRDc7*e=gx|~=G zvv~IT?YS5o%)`|eqfBIZ$>5AzA|h|1rqbJjpEFq=9v1lQ{-(PM?cA8#$+?nnlM%*$ zd;u~T8^%pvIUm zo|Uc)fIZpSMy22{*Y?t-yT6k)XBx=x^owqidp~G(^1dSvXX?rXF9?rr;Uwp>rw})= z;L?oAST@3Radpx2c;yVu|G}fq$lWUO$Mbf544btV`a?d?>RGpXmQzzNIkfmPa?;O; zz=t%WjY@$H@%2R;0ISCim**n`=c}hFB`nWIYaMTU9Bnat_`HC1TZ5aG^Jc_^SS6g8 z)OaA6nqua0xHTK7nf~!ZS)#|5yYDLn!P4FQSNng4V%;))Iy9#}L&i6jVtd7mTGzOr zD}KFml#x+t*3NRd|Hok4b>jZPL@bBFx9bQ?aQ>B%Gg`V$nn)?T4`Nb=LyUXM;)i@h zdxcBN_+kwcN6RHHZTiW%c^&u^;j+NCo#oTPWg}pg2Oq>XfhtX34r01l4i!?Mdx(Fk zE=z^e>Y)lYUF`+K%Wlt=RN1<%SOr!No70Y0Gu zuzkW-(77rNfj@&#V=>T%p}<~_huj3L@%lt1(axS02PvplsAJhIl5@hLd--e{*#|uo zNLe5ztY*y~y(FKk9s8xZt<~7NxUbX$U!-iSKA@A;D5T6Vq>N{w`r7fn0iCM8$|HJ$ z`W1&+Ct($mxbQOPM8#-wT-m~^Rapr8EOS?mQN1NS#pw3x65DPUT}1r(Y(aaz(R6*? zg-g73!(LTeFQnasMY#P@053{ud)J2Gw2X8>NFtGtR={Tmam-OP^( zSz=-f9#@?`_`WsKF+}r>jq6sHl$l)e$}9K66jxN2X#tYh_VBj(PtOuIJ@+<@YM)ib z6sA?dokoAu`1XIzo)87<7-Lt!9wf!Jt3tg3JtzbgByDR~G_Jd&c5f!%AsKfRpj^lO zJv6*KJK+HAmP25SKC{#& zgG3>r357V4Gj|QKUnDeHeT#2yZDsUMPrK*x6~Y5hbdb}~AZ3XSaqJaMD4%iWfhfY~=Ow5g^s;ZWxWMpk1chjLL zI2oXk|FNJT{H^7{CxqltHX9D`A$I25_Lfp!K2=e9z(Wp>R$q4>iNSMB;SoSzt zjGPmfYgM1i2TMJ+Y{6DM)$x9(h)$K8SIErEK!{}&$`c(ug{Q9Wc5+Zx;yh33;J(~1 z?t*l5x#V3&$qBskw{iIi(O2DMzkd(ejVA6~7u`ZpYP~@rlY^Hw`pukGS?P-D!U4lh zY1(^Xzh@C2U(_!z4;=c_28HRr}Az8S|PK>9(%gN^E+l4I_e7gtzMe>j3P3nbFAyF2v=Q=^0KstX|A$&E# zK)#P5iLrn;GsgZ6w22WOf_xbbxaRK0<3XLKi82MA5 zDfqV*>hAEHK`P*&g;#>hKEZ>mt_~p-wXhQ!fCxUCl13;nVKJ*WoNXr`#_)%j0Y`K4 zo-?$9Qq(oN&l)qtg$rkBcGE*ZaYWA!nHTcW4R%_pxykPsu}ylsl1u&OS#M@YzD@lE z%M+~{X96pH_R16~#*dGlf(^MAy&7xhuGe-0CSO=+=-i*aY&8uH@KPQsDJ_9MOMyXM zIw-qB)k!a`9rzFge8hlPlx+D;XR~@w2|f%Ov})OK9_X_A=FlfY_Vh2EW-AlY8-|`loomg2fKqIE9oxxRXDemfM9Lm@K;HLU)5_mg~i<+1@FThcxYfE-*Pu#qH_a)ZyPDKbG>&(F^ z7H?9QbTSjRpy$s(X;~e%`xP0eK{z2%VdmX$2CviIdCjFg30Y&rd6+y#nfz+~SBM=U zA&dLbx>V`lT5a~|dnU|8-m{zF9m9gkU{fH*ffqnTZp}%YV!X8G*|DPcj-G1k2WsR$ zA1u#5UT-?#b4#`pp|lqe3E`31c9d?Bw%=JK+iT1)n5h|cl$U=*pY4U=;k;=&g_$vH z@iFPI3p0Ck28x+@fR4QX{+$#s zm$SnSm|~Vxm{F^5{RL1IV&-0xo|s6k#0(WAb{PJCW(|e}J9`ksQsV{D@}SrvHgpaWerM{WjHsjF~|)bE25^_y&VZp|lo-y_1=F;jYAs8@*SHTYp};D>0K_M_Aob zA8)^}p$-DJ`n|2M4?r{^@WjCLjX3#?a@09RHt%aJV>F^RgU)RRve%7QMSUyMIfK!Y zoGAn{h}^#&sRHqQn|{OiR$a>>%`;0Ufvka5a=RXhnAn&T2KB+r;WZ15XPQz%Ih_w@ z{Vt@nvv?@4^R4}Uf4BVVlq=>25jFrzLa7m2u?5m{G6>Rlu)&+xjUkZZuGksY?diCA zJL#;q((`D;1;T$U$)0*@SQobw3XO)Yl_olD5hG~XjaP~)#Kg(BfCL?|R%CLqdQhNU z`^G?zm%=(P4yep`#>@ef?!J;i4m2i4SI@q)SYB>lcFeXXp4=;F;4P)*MEvpQ$L5>O z_8SY<6b`clm*O~?NTIKV){cE8XJlq%B0cb2BQu6*U8yT@weW4;Z-lVly0!8|Pft-& z5@oIp-Ty)D`L04oN5`MhQdXdAG3hEQ;u?7FphHs4c)Hf>!-o&BccB=zYEr3UcNc0u z9s%2iAvKFFH5iE;YmsTbaWm@DAJFSOGC{baT%d~Nkh(dhxk@MzM z!|@u|$LfN2V?T7hR+*NXPjf+%v{Lv2%0c)_`A1w2A~b)kI61YiHY!(I99vACE{ki` zT;9W@5ta|-kK4mYj{-R7iL^8tgyDSS#g;VlT(zlhw}uK9lE^aV`op;Azw*w9@W?(< z)szbA!rSlIRQ!3WxTaNYLfG_1SKhgt_HTTYz&kg0ZRPEtUqPXE0Ce}eUFSB#CE~my zU8*l!Cmw&ed80;O zuCyyM>HbM$hSueQ1KzrLaA{xrFG4ADAy<0TpzG2qB*_Oo8{MM#lZu=5y3#3*Xs#rQ zsHvFiNSdJ}n#3B-HMN=s};?|=h}C~_~KaJV)LqDG^2KH`Cafe7~! zFsr2Bzttd;s)xcS;j8mKpml;qg1*}#2`Q&eGb;Xl2`FMC$jKO9`v~zoW$^M}cTy>k zLW08Xn&kQ8&@8Cju+Y#~kqtVefg{yIEiR@%+jYX$DO%)FXVe|z!%nqfwdcQb2+`os zpb{uD-;0k3r?uz3a5x?4GUU0jeHIcH(Ud7TbKq|Isn6mGaUAWWhyCbH&Fv0rALGOE z(mzxDDTfqE=Woa7m&W51Mk_rqy*w6wl}NGjSK)J=Aw$nZQQygkgZ zJQtntcJd1*#6b9BNlhMfHBz=2j)`|p@Vp-dbww`hx0{aUy|mnQB^RAD5;1?ZD7`Nt zTIqEb9Us$Lemmi-@vwlq*%9P>X1mzx%P8&3>84= z=((o*SG!KmTaz+Zju_Cn=HD8!_(P9@OVf=1>gTFNPyt>5n2BPQCd3TVMq$Rb2Q*k@ zj?T>b3j?dqqKpz2I+=ktFKhq3a={u$48+;~>`_y#Xn z0ec5v(mjWR0g&z1NSaQ3sFdV!SdbU6o%lKS&AJkl8x#2~Z-DdV#~yNGYuSfu=4~p`n|YBFHO3CsC$;%l9XQZ#>#yXeRH#4vch>~qhhIJ z?YPaOl)~rY&J*z@w( zA1e3G9N~*8nS2P$xIskI^yk{%NSPaIcVX?W4y=MJ!9XCU^SJ&ZV9~ed+^W7JC9&xgN6<-c1#h z_1|LX%3;HSl7raMW_+1M#$#phxPT{fq713-tgLK;IXOO*abGH5p-emm@66oX5KurX zwI1dI5!--4t3~O;s!b7a)_OYPZjV;EaD#X#6R2~sh>K@L(%g%ItSe~h^HN42q}70E z=P*-`+>Zu71YSqks&Qj&Z5+g<KT6Fk8?_J;rRPh`i{@NQ7(~|7xuFqHOOFV`ZvhuL~2{`Q}#dYU-|u>C|>^B zEG+dj5i>%|Q0eSJaIlT;B(-l2`9HJ$5y8vE{UB!$#&2c(cF+RzZHyY)j9vwYr+1_< z@t4zY)qONS(hnHfa29d?(=6Cre;?XCPOKG$Debx2W)ne`tX?`Q}ZQ*nxXw{tdo)Dt#7Z zaJY07*;XVOOxxXs_@PwM*gve_pQK>?+wLUgvq6Tj_#j`edd~oYFMV$HY%?;Megcx_ zzmhQwMwc5te7=5z!xF16dE?f&MEhk&#F+rG*MspH)P9)UN(t;23K({KKrV0Z>5)y< zQdK2_B1^3=8Zgb$Rn8ki(O_G_N<&@ob#fkGm|6!Ab0i`nl7doZv(luA30>Qo0}b2S z6Ehe=VAcS~9MB*ZZ{4iIDuR*>#;S@dV}L5iLf~n}Z2IeEbNM#<>NM z(9{CK>^^8Xc;xGaC^%FKRQ6Sy(a8)p7^18d3>Gkv&j_=vJIpeu6It)@OUzbGRRrjR zHj2u>Ay_FVJ1M*Gfq#u3cxEuuK!7WSgc1ZW@{Y_<`f?p3w|sEkZUh?*Rf{n@I3yD4 ztd-E~rj=jl;W`J=Kw41Egt)Jd_xuSY6Z-ySPH6<`zoZvnA@!fCUm?AYVPB~yM2&YH zLsTW6jaX(es9KEs+c$nbe*Whr(u5==fq9=(BiO^NN45_HE-%|vnGQ)ebRUgBMdd(a zfil8~PsqK<6fndAw>lEHt9+)^Mwwo`?dSX^E?H)_C)jPcK{jsAl~}pTH5RE-pHhEO zFIUF7fr&?byhQb~nl}pyt%C=cfICq>4sx2z?CdzDqv2U>!B;d8F2MnLQUr3jx|l-W z5I!=A0I~vTZ5#`(IEeuDauzlVL>?y$rR|S}WK@D9LT(DM#1A~k?t7c~R@zAN1}i|L z@K#wID(Bp{LSNr%47kx+Lv;3pO802^NFSN6?T$^fVHye-erKN_%z-q`R4b zTOvH~jCRPZN3F9@&wO_Asx7yPE&uvH(U2#Y;7eeBd&AUMT)qk`$`1UY>L*@hQ!?ZI zXne?oWul_uAb4CA&-v^P!Iz{-TXSGD>G@LsyL%ZXE&AzywVcg$v)hqhS9=Y#d9&ZQ z);i0zyEL$S@Lu}TG4wdwzGOPPjl)(>^L=a5Zs8;3{#;ln z8$85CfW_w9$cuP?C3KQRjO$P_hZ4avkg|{CVr9Z%b)KHfvii4(MH)Y$dbH12_GWu12NI?&@5dt4Zyvn#ynm6RUiL!2 z{+jf!L!klSqF^ z-PA&#+B_Y@d0DYGNP@4uCS~Z+eNc}}Ok~nqI2d%OMxFbL@wEiO6HAXnuz!b>klH#@ z39x8^UA8e^UXDCSo&BHtfps72H!ARn|1L)EFJ}(sx1NMTQX=hsUtixVvr-C%u^WFQ z8g%jEYyZ73;*a~xfKKw?Revufn44a5s2urf8b8*07I z2(tL|zkD%;Z0vmjMCPNJFY@3SZM#Q zoxu!6x=j#re3K8!IcGA0;EOsBLG$3B*+iZC!2h1soJ;VD+rm#mFx8S}8sOAZ&m-CV zdrT}8#bAR-$y-lUG)H~mT6%(JjY%Y&AYb?YURB6(5ZMFd48n)kWP*qCAfZqQD%j_u zhMa8(*F{Jj^kXDQGZ`;)QfY_vU1B|405{0&U6Vpssof{qy7Mj-)N3i8il+bGWGU~@ z@7w{VVbGI(8rx0`xu84*pB1X7yfH@F{@X%vj5GE#&*+1XGsY^;sye;|u8BMH|9BDP z96xh}4r8<9aY1Y=`ntSbv)XAs!GM;1avDfKh%YebI`Tus90L7G+%b_LQdVD@)D>2?7Qkf5_{vynobz zr|G7UA4aw=R{aY1@2w?M(WjsYt7vU)O^Y^j~&;33*UHcXielnn`DFE0h^5}1PC4Kv;j^0ZpFmT?cFb+UYdk~hVLOAz$`RHWF-LoQ%7SDz|>)bA`G^=obtz-9V zuKEFq`oB3L)dqR+K2YZ1M(d#x^K|_US>HzqTzbViU;a3W1PD>V!TbH?bPsIe#xU;M zKga?|L4Nel{~8T-HG1@5bZ2C6DO0Q3Nj#rV#isl)Kde1FjTW4Kzi#AtGMiWS2kqL@ zA1vSzv0>S_;(#MEsZT9H_B{{d{^qBembjqhZte5ag1OM9|Lg&l{oxKu21+yt(r+|_ zeE4!Cop|j^NgJ*8DhOtyWzHtTX5RxTY(gwFi%$b9c zs!(VoKU}ES-r3m%X!$PC-&8@iMIPKN0H&e601=Ej;+|0d8gQLXEES?+qNh;(=It#8 z@)`k7tG`34tM3Db!~n>RAqW)(NXy7XCnU(3nXxc{5(cBVI31*1er7)KC%3N?4bZwZx`-kXA8Hx-ctIzZSviI;E5T#+QKr#q!2>#h!V0yv* zz$P&=3kxT~GsncnVvl~aCWD~?9~scFM{vEU$KgUE{S3kizSv!8w+9RIv7W5{x}msu zCgil|)Ob;kkWu|^`?7dbSRYT(k5wq;>KTn|-#bjPn^nvvCjWr8LK+|>@gVqa0^MiH z=Pm?`Rf1&aFP@ve7~PD?%fgVxEhb2W(g*%zjQf}-NH>8Dv$jKNhiWKG?pIik71q+v zB`D|uZf+{-6XB(w(M)8d#KaR6*D=&1fl`^CnwpvfPVsF^6$685OQ0FLjM;q0D}y3v zt@MpJqgnM!)!WE@n zXNL+FeZPLaReAEn&KzWzHFZFTzOcQ!J1uu$h0?Obv@?Djh`sOR!r)u_5f1=jFI5&Q z8X5wIhzAhh=a=cr(k#+{X>Mxjr4-iM+Pd)i_3L}UrgZOXwEAY-(cYe9$w#?nQDQw@ zxOTKT>2Z@2WS#RshRGaia9+HJ2bGzIOR99aH1Bi%aNan5=tQWVCBs1B+CUB7ot3jp=gV!x zP=St%OBpDsufY{d_n=?j*gzH-?&+nar5-inmR??9j2a+D3nX@)=rlJsQ+L4wP!JJ0 zz9og6wA3|tfc>)z_ed4umJGxgBQ8dHP_L)0ooh2zc58COc?Orb6NJIsz${?p> zY+`cwh8eU%J)tiCFPoE@LNhl1|Dn%X=^@QSG=D{ zkKpQ|dHxxU`f#j1WRZH*V_`0o;Mod#LsVVr94Hf@X{$t{B3SUiA;6WkOC>8t){hKB z%0dpDxABdgFO%TQTow{EL`9({!%Y1$D6Z%>;^i)ELJ z9=sqVZisrH+8nONZ4*Yi*On`B-Ec+--w^ej+8iTGhzn^1Nw96j&qAWpK0*Q&e$8jN zP{C4Che$8@ELRBe~;wr9;xbULH~Nb+81}*K^Khxm(7zKYXpg`E!P+4*pFUZ zG7K>+H?b;C^wFUBa1I68=hu+d76y=T{~aMjuE^Y1++;Iz3)K?*tYm1be(p*==oLc3 zbWfU;OOXYXdT=X*g~gwADwm=9{*hV#`BcGI$$!u1ZnVVz6Lf%VOX%MZUr=si{`*?Eg>*{R zUnh$SFC64e{`Yy=t-lWVw`^a+pZ{sD{QX@S#eW|^l0en^_f?3`U%WxUR5@b;nM8*F zpTE|D`=wfCpsx95|KEED3rPKasV{h+|3xKNz7K+|1dKdqNxwcNO8P8OWWFTZRk8f^ z$dT|gGl}$Gm=jg{ghfP4xl}K<%9QtA-phPc6m_3Y;U&Uyd?#BT)kh^}v19KD3l{NQpC zy54w~Jt1z1PD~Vv;Q>tv8xXq44gfP+X;3umBNt}K^BHWk&I9u7z6v3B_SL9>l2IEU z0Ck(NA#dnJOYkAEZN9SJtW;Dg@$=`;tl_f8V$b1Jt;NOgm(}@maQcmxm-pyKt8MvJ z&!U5#Y$DVm^yTFV`~u43($L^wH+^`EksGwKo+D~h)712##X`Gb`F>!<8BT~PN%Ej3 z`-Fj&YCj&JuxVQICEw6F_N*-!8yh>qG8RZNPw8IrA@~F>Ni)PUikLY0C>2#aJ_yT9g{dR&VG&VLomrbFoBrYWWIh;IX9=8`WFRx zd2%W8giih`+j4RFMEEb)y~bOV^L0aJZeE@h_VaMr`2_Gpm2b3~l`0;&Lf6q-#RMZ* zYN!vC{?_zcm%iCTIL0~&|3CX{(ZqtM{zKH8Xh2QI;|mw#U_qZ8;qdIKq1340Bc7R? z&E$hg0bJ!og_l2dsnzqb6ULV9)zSjrva~Uj`0l!eNADKdl@%%)jH&}`ggH3E7#nTQ$Ge+;M!|>h&~eX zIJskU^BPvd?+x1h=nd>04=T3tQ?yye*dEqTlOzH8V`uEN9t;e5EU$m3iW3!%z)RzU|<&w}oXM+Ih2Z{vaS@Sr~F3Y=5`yOMiG;)RZoTr{~lG==l^j52m@T#1JyRX;_P@g zT8WbQ675W%Wz=uIpqHi_P5)qfM9`!073o>KB#J(!zUHk5J_EEVwaWp>g0^EJuDBJ* zJOiz_Lhle%44d3h?E3lh38=8Sf;xpi=o)HZ-GzQ3&?QK+0(!%SgFqZ1Xt20lT^wN_ zfilDPZYMX9@9e)B+#C^mdQTNq(Lnqz) z)Ut8XW2@#ml|Z;15vx-3{0>1(N=hf))v;V-cOtJpC|nTP_M{2}g-8;V02>e?L&dX~ zHE|`zZ8s2oe?XgtKfr0FKhQkGE8Y$vR{@R@(?o%GDFS0$0K&oJG?6$v=)6J$T~zsJ z5!w}z)WVYTAdh(0yoY%Lde|VuDzJ}0TQN2+PDWl`o#Zb%1w@24c&o=1L3G}U;6@Na z!mAtlXem^HhpX>(px!77Eprg!90UoSE&Ri}QWYSX@itdA=f);N!~(qUl}yI>)=Qyo_V1U0shSq45(fL|$!M z*D<7Y-Q7hfBQO|xWb^Wjc*uoXe;4Xir9g+5^I_r1v(^LNEKX?a>`kww)yuL|Qw~y}9G^GUKm4=*3oMHWdFlYKWw|IiQ z^t1PPY-5b(lp%@^yxyrRgv9npUd(m7TQUQ^|$?k9fcPgc10 zI!_3)zV*p4!?u6-NB)~jh7#O>!D{x1;{RPHC| z2hwN=4`*k7UqW0;O|$Hcbbv(~@nh@@72?6yl7iCD2(LNOfvyZ`7&?Ovle#*Eumf`& zGwQ~F`Joj%q+^WYe^V7Tfb>vDiaJjK2J5!}N22J5QdE8*Li-$P4gkI&M3v%@6|iYx z*B6j}p80>@_N0g8HL|0ao^9Iae)$GlOyfUz{77WS?oCG z>{U$hfvX4ZuYiNcpi>Ins-TM_FA{k- z?_W0?PiY407um0W%M-gfG32tdnX`B75$TOsdtvirTkWLO={o!~Jkp(py4oxkXY)pW zezy~o20=E7tJ+bgj>pRp%d{WcmzlUkLWy?HE`W#GyOmQ>_S)G}OmQLjW`rVR)OAfCuA`Qv%Xo`*qA^uQj&zr%l4s5yg--}~jInE06HzEF077vtSO-=kt{m%CxY5?|>r z5ru)y8nt#@D%mn5F3ot3ioUl0_&{^|MUdNE z;Hb$59k4&e&I8UOE*5R3+1xYI{i-oZ7ZY2#1%*si3Yf`YQg-$gUhcFvzGVCI8B~q4 z1}rTTtQHH~i7J zC^kC3 zk$}C^OFx4W%F*(dbihb52fqlSqJdg34**MAfC3M+g#ciwezh88|KYSE>Ff;n=58HG z)@jvx3UeidflP>TiPlJ|IX-Y#-a`WfkOog8(k?MVht7ukD+58{;n#aKo|W5vgnFaY zRBF&Nwv70D_;Zc>F8hi6wd(;w3Z}1q8gBd6stx`waXrxivU>NbjdSgIMZ_rv=5RzW zQ-{$fZ7F|$_Y|6=d-tmN{`Lm?{CnodPcy+&Bk`loGjuF2Ka`@c~q zoSR&eMO}v#7ksvHx-Sm~9i~n!7?L)gju!vX3l;QwaCyc(zjt!nCYMqAYFR9q-_7gy z(6}An4~wO}6MG@Og7Iqqs`eRuPrVn7%a83W-NfK2x$KfG%$1=P!x>L$mor7gBGe4YlT%>rDnM1rO?2U z$Fzf3qex%T8UcMrMY*8h{Re$JVIv{`H`T}X`VnHzr9JQBJ1HKilN8o5MhRY?P&{*9 z|Mie>6q-BA{Ey(fin2;y-piqv79^Rt3ckIgs96exII zkr-gh>8D&`NntGg;t*g7g_Bd|nR#~Jr${*M;J_c?VB$|o*SpupL4ftE++6e1lmqU? zsmTC`FmHMZ8SZR1{-w-|vnj-P6R~8H2>5#qw|}{&#Oj{;*BC@7SLntn?)Hymj-L z9r8b_3$VDodiU<#BNkRxmqk9r5p}x1@;F0`1_Tcv5U6)sPLl(IDZl`p(6|%&;|e}b z5s1*f3KBg=dYgBj{JWR9dC&gW8t$s%s^a>_#{Q3jDwa4#PdG;8*VzQZU&b9J z`(qis&kbf|N5ScTAi~ku|Gj+W?(+HLx$|jHPv4~owhl)dz*-+1Ix@U&P05#LU zmJ9IMB%f&ODJbfRs?dymXRfmpP#LAjO;nVXe9vQ!b|-Wz-}?s+?z zn;*~czB(<6F!Jdwo80M-S@~Vaui~5%x89|OSNKI@fG<)_qtxZ<0pkQM_a3R&!{2^Q zBqKJY8U<4lS_8qWS`#smEu1HupZ86UmkVcj`jd=?g_l$JoO6`YSnP2Sy7N%bMnFf` z%M^9L5xN1`2~kV$uMJ0eU!3x~ZX1GTemfLF1C+FE;-uHMc93OB>0Cv_fF_QqH%utG zZN2AuADBo11gVd~K{C0sV+=zL{BT$i%Dj+zfGVQEZnJqU+j1%fG=Y?eSRlzrWBEJt#sf@$c_{jmFz;oo$ukYCb|X+C?b?vg8@WKLR|6!r#p zq2X3hsr!Y-4;S{Oy~TWNwAbO3B1r1$!W8+VkE=WV5G-Ya*X!Y6&8dadl!U04q-*tS zLV7hl6{31Jwq3oXnOR$jwj2mgHqX%1k zcxZ3`_T3Z19ZlC7F_Z>+F$1Ki-ZtgvqrZz-if3}c=^gxY= z7&0R7tto%>TyftPC1F5|aiYW#qK z2fY(YUW^MsaaLQlB_$B`mO#kK3?QICKa?OvQn?ua48hmV%Y*I^^L-Z{RU@{2S%zP=ILKM(4v#=m4Yv170MRuDpbu91kLib~6 zSKcyJF-2ylfo^RGJar|Xn8emgdaDYM4 z(e30<_G@3t5uGz?FHfzxs@CrydTP)U7(y{7G$6SK3Zkgcd>EeMQ;IDDF$0RdH2ivJ zuB95<1iy=qXAH=M_oA|`@Pj9Z`qJRzW$~+aX!Tn0CoAlDHNWT>)KwFCo}<7x&n0Bh z!v;@d7cgYp7Wuc?_o-Yig}v-VY&?sPZ#!>r8c$_$-?CJdAC3`VleD`di4{O_p7`zq znnUT`W2;4x4!k(W`+kOR#jh@eS;S9pQp&d=2YQ3>`Wvjk#^$kF54t}hTsbsV-H+Zu zrzOJISJJS}V>4Wna`e+5SUp8?nG}k5F>rgwsW3fTMAW*YN*=+^YOIEv_*j-P-DBHU z-^1ZXk*ILUaC&NUSq|&ny+F*Qopb-haCVhYczSclCbI+Dj#$udsBG&k_#8MG&=6@d zsIdSo*rb2}v`7*xG}n!9!{ql}dB8w6t~h}lvv#G!O)_0O_E1A<`dMSnqsR*2Fki>Btn*vy6#r9kv!Ixtbw#N}Fa@m`Mpp zmvNGf@n!)hm|Dy;F}R?B|e4 z`%$>3`sYW_>R`SbtS@NgwEo7*5eOAX!XdyjYKDa5_+&$mUH zpC9jp(}|}~R=E(x#l|*io2SV@aXyuR4Lvm;QHkTS5|o_!cbg$9wWR^Ke}hNU0TYl2 z31{dIgamC$5PRKbx|T+7I7#p&Eo8GZAy5W^#agLoZ^WE((0AW6(iqy&n*5kdmJ zVrLxSC$a%`2!v^iE&3SY;Z&eeGcEAVY$ty}5v>vj+>ihNVeBo#vWk|lVLFrsk?vAD zq?D8{MZ%)HyStPSNdf5;5RjJc5Rn!E=~6;Eq~o2(bI$j@Ki}&jt|#~2Yp=Ct=AL`* zu_gr%A)J`C65b+ybz+R*(>{4Z#NB*Il#*WolPSY{TLkDFo}e!9yxM{gS|1@pV{UE; zFXU-`-#rY)BOHT9J-)tTz|UFukfaCvSHCnRs80qOz;7f@5hW>@6QT*<0tjM)|7*~w zI}O|2Tr-=Kyo<&i8kChBMsk(CLkRjku8re}mqm&-c-0!;UgPFkNzz$ZxM=9lE66B4 zdGPctjhvpEpnd;UfP};a73@5q4)jDzqo>64%}HM_Hv!gbGB-E-soS^WGU8-*?3jHT z{ccvR`bP}yuo^vG_cP-@dFPQ7>^FPguy*?75t&v(Ska#sDD4Ch3_#+Rq54?J&_RpE z=e*tHQ|ZD?xLE$W^q@5{IYT zZeEwQ#~8R2UBK5vEO`kn9Cg4=pV;x)aZ9?WhsKpL{e8uwsbD@)UuM zzb6OXa18O= ziSR$GsM(rQw)JJajZ678?BFaLtKC0GcVh)S+im}|duD@|A*Kg^$SV@8R&xx~oToKpGiMde=UPma`i@JSEb=t#v zP&(1CDluP`Z}PLd#P5VL9Gc-^W%-OyDfN@v$U*Z_4&_qlR0`%1C1b@;kEbh)F=3z< zHNl$&e$EqZZ8?}+GGNf4%>n4}bS~0qgX~;&H3^WkW9T13!3bON7(z7ytcfY0aA}dH z8#0J8x%2J&9HckJybjxd@Rg9BuIlK>i`YA@f$yZ_fBBbcXgK>Jcv#FuAX8m!F%rbx zauNg^4@1ZTkiWA*?UTK<4s>+GlvGsmpkfj0> zY&t+&pCs>kLej2p*O=(vazGDGphr?X)|kOU z13Ooqulh~BRFUY)hw_Ei4$I}!3Tk1-U)wTt1YGD?;$sV2%w2yCtZti6P2c=^d5PR4 zF2QVjWBtkZ*DyhGin~=p`Yg@+!<^U0vvJ|=Z1iXkhqhl86)|;*gj>w#$Gq%$YuGpV zB+ZQyukr26(q^(;_1_y~l-Qy7B3_a@*j;a@O|)VTHD3u*)@*`QUgdX4j8tabu@}3h!&mc6-)d|v$R1JEQzK1^5{~YZ z!)1X`j!>0vluV>fB7M46E>2B@feMdOKy_IYAw0s!h^jiu2J-}JXqP8zY$>7Ttq@|0 zM%~}0(`F;dDJqHx137*pYzzAys1UbN+sKFkigy+5uAF<0C9K@WT*6BHIl}BvSPmRnUb1YEJZ_=q~6@ z=)Z|TxJ!3Gq1&PUiT&?%nime8{X#vVKiE6tc=@}l!R1%Ij_AHl)ar%vnSQYh)TMlfrZ`;~45-5@}$^XE!UgJnrS_ro&J~1w2%!p;l-u)u{ z3`59~>pn|v>}}dhe)Uf&Ld)T;b6+ZmmeOYYW=*Ns899>+SRQ3H6}^_L#S9%#l6%1i zdGWDqTji_0T^Xiz72t#;lB7o92)nEeBh#bxc&(wmX;9^CjlXWvXon0 z(1r#aoAfXi)s?1|^Z#)HQV5hs1&p?UBg`hNt^l?6U`Roa#>g@Sk(fTQpR^J5I$Ue7 zwj4z`N#M``$}bZHbPBTihzW`48G-2UKu;eCv{mN^4W$|ot-|JrXib5r{{!x*57ZSk zoo_cH+(byjLID<5XeXesM15Os5Jb=js4{}F9DViAExzj+*cpKNUsOeuD?4yOg(;-X zu8L^Y#o#eXl&L@wIIBIjj|*^<21|ZXJNdfxc5F%&JRy6S4rFJ#;enM6Vp7HbFHr7r%$&h1m>hgjj`xm z=+O#br(c+H9zvLKaNMFFNnDbsj)r}AO@B)%ga91D$_sClR5^e{!bkB|!16m9JsPd= zAz?!)F_7t>fdc{cQ`(ogD0T9;r{d)`i$U?Xg zRjwP_S1+Zprm8I;!f+vcMu?<4>}p40QGp$)fQFAhfp4sykdT0gGaybvc*IUle6U;~ zoB=+)3he8ala2h#XFkdOW`Hl_~ZSwPo?lLT@GfB@>IrlwRPPH6GHGT^_QeEv^- zUpLlfPW6G|e=ryTF=Y%S`A1o;mmxtcotY3dtm2`aY0h4c_vzE8-x1XDTL16}pNwZg zPz`#1k%{{V17r0!kV%$-RkB*y$Auu3z$PgBe1#CjLB?aOzSo9}%5P!p9r9kZk zpYw_eAQ)@VZ4?ASsMtaC816Wtj2IRMP=XL2296R=Xs1*c@G1z?a6`frG&4mdP~%2Gc#HFXdB?xNVZg_(rMnMo5mz*5lv)sPr#t#@6Le?ZFgaE3w zPCgg@bG2VQTCiDAHDLYUt$<~6X#*qW0jeqFKyY|jV`B&y3V>9w(*9$AalxhqiblnR8qj%hvRh3WKf!6D`q zRQ23L;CzU1zo-b>@s8Fzn?ow^75KKW+`dDM)_@pVvqus8pQw=k{aVhhi|To3A(suT9r-LvaG4Dk=nU zG6Zi(8>T4K$ighE`SSx=4Y#05bGf(Ai@=Y0O%E1$L|%! z@_=;~6+s(fX)Ggf{0Bu=sbB|x#6|*Qi*RJvixObJfZe>W7jc7N^`vPs3Ki05k4?0SKAf&B49~`^;evlRI&T-nS2_%8`;G@>>CTu*qbs zNJ~MY6~G~Qnj7 zw0PWki1TTDY)aS1>XTxvrqen3O$mpP&}$!JU8M-&guBJor)yQeIWEmOHw5ih7Zz`% zte8Cco*&c4O{JnZEzM!ZUjYH$@f@Gozx~7SzyW5qrWuZQBJGLY!BKGaj8k zjVVeIMG2%+PAprQoS4rqYDm=RUH-e*bDzdwWk#0jZrLx*>E{iZtq=B5CgvO^cDOIn z!g4wuP*BXOq<;<=nV9WN-Qn!o!6*2Qk9PX2&%Vy9!*}NR7kkNMT|&vBX3B4qz1gGL z8P9o=FSA!j5M2(gcO?G!Nc!vz4ChYRfLSg$8K!MqQBVFGKp+NsePW4s(%I}e@w9%V z%hXBBCV~9j;`Q$>VdvXTQE+s%1W04k_#29j&gn@S`f=6jyOVFdtZiE_*8m7D=ur~i z?}X$B`xlhIOg``zJ`!RWx1Sr-r}2$_`_hf?>_SS%afARc-CVYy@4lFxe}0quv>4q( zM1kfsb@BbxB!0Fnb?o2fUe*|qeoxK;tv$QU8mqBCHcad89NiLMfyygTLDhYYjg?gD59tXs2gu&#?CSi0fOq+N3i%A+Au=(AkJ?Be`AwlS^a@ zF`A%u0=Zm5)zOoW5JKBU->&U>({0mXR|OqAc-U?bW3=_N^TJb!+xrn#t(*Q*VLM- z-Gf^`7pFHMl_xjqYBs3b)i5wk?x?4q;eTSMg9p0t3@?8DUT~V4eBKcKyI8Q?XAFZ2 z_!Ii6_t$W2=af+q;|((+^Rvz5K4QE<6NcdS?N!VN|M6rmiHHJdB+1JOhyOdV2R%|= z>y2fhh446t{sbf`p2j=2p4PuTV+Vhsditbux2pd9OT{mt0hXMW_JOLh)&X|@xz+Q^ zP8s1%Z7wRl0LrqaFud+hFmyY9yX5}X?>qbJX%BoS4V@~#ug_Nzn_~8z@}7UgJ2))nPp)#nKF6n_A6z#vC>zuf z@gf5p<%^AAh3P8Iw%@|@xFy|cORYBkvcV1$zRIoQ0bI!49QLinHt=!dYYHZ@&OElu<34*b^Yd-)|li( zl?~DXkNw}Z7IDd6HXfK~XgR z-`3a~V%BEdrn7Kg^5K82ks80gUvpyfIX-Xs)7yizKT?_do*tZ)ctUdaplx=7cyGm< zcJJrS>pa&3v&hR%9@yW0Hw-lKo}&VeH}t^aHsnOCHYiVFxeql7-pn2O8iOpxo2uG> zesVJ-^9WBh)r4=mk`NHa2M1>y8NOS#kjJZAi96UkMeWaMMNE4@5W?R*9p_o!yY1;c zS8(`eD&x(NMCkrEs}aWng@lR!wA}7Mkd*tSthU$P^r_DvZ`Va6 zJ)H9#4-kmet<%h5CjPh1XBo0jj2=+dZvGlqjIOG3k$4(Op&7~+*MdQ@*GhOLMfyI0oH zgr7}&Xz-qDjucyC4>(l`$5NIyg~2ZtVgpRdMZ|5FL^G1NVldhMKl8)Jd?!O<-su*Q z5ZY|`MjS_yW_)&XIWBzAk7qrpaqaLkIdsEIHy*eQsUEI<#{*BM(UP)J>A1?PQyaEy z1eQi5y*q>I{(ZG1iVWw*KNwDg)yW)JTF*YZ~BTa{ifD>Q{{N{^9E1YL>QofD6zgzArJW}o0+WTJVsd065(x;MsFEg;Ykw{DT zLwN&?1Vk4{SkI=Pupr<$G81%50VzU+3+vp$}T@G^<3n`?MSX-b!)Z zWka8X$kq>`I?HjU`11^}SG6k~IdxjjId}2iI#QYB196WH^pCJoC80fe4cK3rRa?c&OBNY#h$ zz;mj)e^o*D-Z>uC3K(t%*oIT`2{)Rq#K`>AXqT2j*W69C)>;*^eDZgT_RcO!o;AmU zp&|F*Vt!YBr)#}DmSEMS%kz?KxcDY3kei2+^vgcflmIA%moc7xY?n_;T<&ab5d z%viD=>3d`fk?y?P-wD@5eQJG$z(8&vo`RI8&3_7a#a2!F>#7o6aDvK_d!`x>vL z4Ho(Gsh>|P}-x;$2N@!qy z05S|K;^$0A%L4Zmy*_$DFqnQ7pi-E{W2cUxh+ad|fy#|t-={fcwV)Xob-$^&%8VwC zO~InU8Wz^BN>c&5lRUtZhDT}038aBv3$ZsCrfT|G6T5Bl>4Z|Xtc9E(xZ=Lm)Se*P z`+8t+Q2RW5-@_Lc6`JYGrnSq*n3P}Aah}i|JDv)5!;b! z$l88jdEi)lX^Qu@!$zGl!mO%^-BJ|At)r3~%VHyK1Hlu?I5m3;0u&TGdmaKTD+&oa zv2vQ>SU+A2bREISfaeqHv*I&&?2cW#CzgkpdxVUGYB>y#Po&sX%G;Hxc}vVTNg1_` ze$;;)cMlI!2G&TE=a1q4ZnjT+YA45SNZ$$=c+%$)OWH4O9+=R zBZ8nz1clcJxoHl`&Pe!ES22dBU)H*Xt6fpJp`g*{RT=4-)`dC@HPp3&r2|XqE2X-)i2`PUC1RwS-$ArL`1KWWoOeN@&Rp*L~h$cFR5}jv% zGlEp$Gq9;VkaLDr8kdq5)^4yckzZKDgVqEF>@o@%`vYb-y&hyBLALXI85`r68K(_J z=5-RK@|jtl=%DPx^bLiH_7JA@7MCo-pHT}NA9rkDHu>0|*0PV&hguMC0S-#8J&%uVmDZ{to~0Rx^5|x+|UYUXu5vod32Y@;=z(PoR#egrl7{K$Z3%M z9K)AyV(V|_bZ0L_jC7{G4}NtVrr=%B5%8Vt__m(28M*Xx?zlTnY!Qd;ec2z!w-b*! z#FZdeTN;tzNPQIf_B=l6(RIVve7K`!t*v{_(8o2^$nc#bGQ#n91QHer>a(YR^yXC= zzHDl7=KC}X+4Ayk)9(G9CN=*01>cP(9Et{$Y_CJf7)3W5TD;q{+q>%Q^(Oi#FY;gb z4GYw_jJ>@SdY}}EzHy-UvVqZ>i}&imuj$*wn=Mubv=ndby<)rPpO(8mB{yVRf0~$J zddu@US3Ghst_vj=y8-HEz~6=uPl(wLNo2!J(EI87DW0Ks;53j2q6eZ=SC!tZB507s ze1pU#?e+1jLw5X--dsJlvzqlfWp0i>C?beMP zYySFYdFsV_C8!nG@RdemP1~I4=&rDz8$V~{2#ZAS{2TV0!{Gpt! z{yN0W&3ZGwHw;{C;2BPqo3fIB?7d*^HzWR(D%81N&ii)@9f$S@BI7BbYJOVumqV}N zMpf@u*fSg1`iGz^>i&p%pGd*bb75*hSP-;adV?XYVv25ZclG^w60Q@+jPl)U@k(B_ zCNuZ7FbmzMhK46+7yZvq(B-r~fHw~0e;|PzXgu#{x#GF75!b)soG+Q}x_s39?Vj-< zEAR_*i=Jq`>vtEJ&lp&xI~@c!LqjTt1t1+;OVm6`cBgJP>Olpv~PJn-yI zcb^kU&DDfk7MEt=c=JriBZ3Qi9I;@;6@?mBa7V?*w?O9_d`+rx)pJguCkX zC}_A z_@{9j&t}lO9G;f6J*|EF%)Z5;*QMKl`R=d#(z=tOT@4@QldQKR?VWLXt(44{isxf* z?I+a!9k-*{|JK#-KChQ>Z##pvr9?JPts)48>>kJ1<|m~$6R$FuTw;;BP_eOMXY#+^ zn*Xe7l@`HEbBIQV7luj{C`=!s!TTO3qCN644pufD2`UOI+YMmED?p&bpLrT+UYIK#u z9qv4J=WhS%LafryA$&qqbE|Va2I@vp)nJPEA=v2x7WU{$7Qpn6- zQqPW*ad0Da)G#-DWWYbiR7#_?(TJq>V#xTx`58ycZ22poNxUDBJ4&ElIk;tw^{P|H zz5CVS&LlZK8hFkJXYS^RkoIDQ%C2d32WN8>fq?;L43X!r)O)EGQ%v#q%UaV)A~n@8 zSHuzsAsyf;i8agu?fo_do(*Wi7xfL;#v+w^ZsHx~4wZerN1{6@GhK zXL3(*QkQ4?2@lyPwg|!y@XX8`XjrKB3HKWy4_qT$34Fp6U54hpT-7o5mvU19z*eaBxjA##ej6+9)h7ux) zg${tRBzUt}$kKF=B_X2)3N-M1RsfRA@QCEZh%5E*h%2rOH8r&htN>(c7a%Dj!7nmA zlA0L#%kx5*O z-JtgG+S=pwZZAO4p8-=r)E(r3HY3+X8qjHg+|RxBLTL3gbbAGmnBS)#cMNJ@?wwxz z66{@w^|m+bTJXc7M?yv+>27;OPZGFr4T%(((W_;(*Ri3Q>EAks)Vl3HFCBUGep4|0 zhU)I19a?ko21%BXO`0xbp$8b&?e)G8e>^zs-vGlI*RA}NXsqcuUbHI3(5_T7yduN= zrH6O#3wv*`%Iczd!T$I;E%65m90YU6eO>@Ts-;tGC2H;aUG#a9Ec8`f?_)-g$1EPk`4zk9c20P_XY?lBX!)|gbG^Er;O})E-<_`8902r5~f^yd}u_YQ_6J#7L7Q z*oq)Rowv{|J7u$wg|(e2}92m_~URkDuqc+JuJ1#27f;j zWJ@wG280@9W&E2%EF>0mWEL_vPhB{vz6)>?jCaxxfIQ$sLtCMS7Q9zGkrDf|%(dL7 z;$$}(k?}=oYVZFNd%QRv>lZN{*b~SiNp2pKZGKGAu<*8kbeH``XeSzq6ePV6L@Lz@ zpwU1P4e&029a9C=;TmA}`J}z+xX{tTqZEDYwu|+ahMV&sm52}vgO@@pvPkJ&zU2w3 zR0zSdeijC(5?RIH1rQ%%b_O6t4LI@fvS9>e6OdM6)04pF*8Z5RxH~cj3*K2UvSnz# zF5bd@AL=#~&jmTdoJC<4VL@<<8H9xl-iP6Fu4cN^S{>*&7s``_77w6g#Xcve<%_fdKZ0 zl4A0uTNogZ5OpX>Rk1MDr9o^Yjiq*bL&#oVpKt9u(clAa=TiI1!OLRqF3Tpzv|g)%g8n>KyVA&ZmI#FJB}omZsJ2?wUQdCblyhH$Yn`&HF7$?v!-q}QY>r8*N1F(ibhJw zA_X;bamRkOS%i0*gm+qm_n0WQ8C36|vq%O=?s5nD6H@dQ}dieG8W z%gz}e4V}h?_886A7-3y+NUN(FEWUgdEPiLf`pHEGM)!^_(}Fc0y3b&Hc7U}5&jJgH z!@*!j)gM<~?$|7uiNC?;Z&H=@^wo9Hy*{Je!t8rdRJ&cY$B0D@lDY2(Zm`BNW54H zGhNH_!=e5WZO_=pXBHpppTN-vvr#RNjW2x9QH6^CoxfZh6Hte)%a~v)@QyZmLs_@^ ze%TWea(Nw`s5-LOIQ{vI{1`WkUYqh-SQFkSM3Y)$f+8}==BT23p)lVKa=7t#oo3p3 zY4-;^2QRr})DL=v%GY!*ctk8OZ$xoDEabWrU47kmc4cw;hp~0#dJ2Oq&=2d{H7Uw` zsZ;2%K?eW4K%o*@YR!geqGmE$NA?eGlH<-Cqm88Sa=MuK=mSR?tHcoTN^>(6HMRK= zJ-)oP7!;;EMUxghJ;prU*~v>h?~T-P1xW^ia+1~W-HgklyRbGH3sYD9VxgL!P#TJj z-W+nl6@)C5orT$2z$?CB8+PoS`Wmwd*Th^gP`>i@7|K1#vBmwwhttR7+cI%(BuV#3 zn+}hVSv$xFtU7#7-|s6wz`f6VAA=q%Jz4wOY2`HceeLHkZ^A*ebs!3*Ak+;jtEvb? z28&u+G9a67h{i+hAtg(oM%btY^6aK-B+o~ji23AlthCXx`D2-0z0dQj{&9PRKiBl= z(dv=^-^qz%szv|%81GP$C8>tPcJ+CUg^#?=4wfayzAOI~uVTWMTJyoFp^3-~?YXkJ zyK4WlFDmGv2jL)@TX8<}6!lYlC9>pHK8nNMsAr>C@+BYNF!l1rl{~{;@ffpFZMRVV zVxcHGX7e@THB<4|$_S=ntZT72rd%sh&4Hj;|KU#hjzu~z;V>FJ0-U;gnT=dytEH-c zMWcToJ}G`E)+jpDKx60VeR(fQWv5v@3fChZ0=x~;4)XRaBl7Xj zhc_Nwy7fm%VM&{l?h9E-a&#m!I0IeVd~32Q6v(z{*;O}QNec)HrXbzy`H0nVCx10MeIj16gjES9sH{93p9(1NtDvv~u+HHLm_!Q|j9plAB*R$TwFUjnbGYsNx z@dpi)TaLSw^e*4i)fbzz`E@G`D|?iWg5))>LghEUkCIad(Q3lV(kYQlxLTT3vSS>D z7?g6>MB_hkRBALegK-x_nR{-8L@u=!dFT+Z z>_Qjf@C6$7#`P>qO}H{CN)w1OLeE}D(Jbe*2)Z;;iU#qo25wmvW-u0f(;8B7HfvtA zEwi))E%?|0og~xvQfnnlirpmBw^u91ECZ;sH7G>sdIiiJl({B?|05(}ao=JCr!`Wj^;_*6~XUkq$p}{B3Z%@Sjp3JzWOnBN94(akA zX1|o5$=O(>rVmfgEV(afmqJ#_D}JT>mMaMu9w1XfpTjBmvo54^dC`o2A_ z9+scD^s%LuHYXZ3R33gh%}8}f_wOp6yjAQn>-Tv%F@H+7s4{}I$lq|+1h6!#^oC!L$3AI3oM*if0(!;5H#ke&Y=8RbB zi6J9%Z<5Awe^#Eht=tG1eUT*d#*DXkZ~3+By6Z+0GC!x8YfXOglUoBOJkz5l-qN%j z732_TFp(?95sLCEO5gkW7*|kb+{7sUj0g{XS2VgC_Jh9@Dxqij`NCXSp+1l7y}ZZ2 zN4IeJ?);5@2vkLB;fIPuad|mk?`q4tmqgj2`KnEhXI zBISZs>~r11Xwc=~b37VLC;BwEl-ZD><}$lg`|bN@6Z-v0a;VR-q*rqp70U+d_pXTa z19&=;JKT2{+BqQ!HKY(HfD-nQK$Nz*vU0x+IGkzl6k(P7=BJmwS=D>Fp||ZWQ5LHYuOhTkxifhGCbTp+h~X4sNf%`0h6XEC z)?BWyCN+^^y$-@MPg~9Ozd8WcW;2wC*+~+JhC2wWu#rfQR*xb20#R1yGQi2DSm0Q7&R31%CHkySZzcDwMo_3vq9A zV7-#YGS3*2xSsI^*<7&(kG-WIIXO8^v5fN2g*XNbk}x$ib)EA$P=(I2Pgb4kN$qCp zDuUcJ(WD4MXr*}{qom#BkUfe1AgmC{ui3R>C=q7^1H$g!iC0RoKcdQhzT5lS*G77+ zobO4=c%giiIa;-$@isxmX_53@y26w4k2oJ{!Py|Ep6g5oO8LU!lm)H z#_tGR95MQvRbse`+Jqt3g1OAGzqaEmSWT8oxCLu|(A4d&d-vdRiq6yt_`9yZr1kOh zkk|Uu6Fs~*yN9BGd1kepDQvNmFc%amHyQV(7yJ2ycfTX*00^=AO%SGyS&ux_%i+Lg zuQ-;TpFxKSlk=e|K-SCK7EYeeE z8^O ze?Eu!{FNc!YA$DqmX9O33Z?n-yqQ#AnO7Y!xtIGMn#K(3Er~=ra5aDFuIUWnOnF(} z6vnh7!x4h!C~|OWwtvR;F!o~P=!|Xb3GOf|jTlwHckvex{B}=RD$#pGvrlNMO7-fF z9g=#sy)As4cJlMcb+QwY`czdmkQLj$qs%9CSf8AU6QkOg^puN?Vo{WJ&KQAxs)pbE1PxSRY?o0|(jX(lh2D%x=fIc>`DiLkc;K-RPD`q)}Le@OR_%J8YbcFOb)cR>cDLz7j8+dbt zxRips-M94#Oe>zj3FbmWL>X!krw}0HLX~tl1(4xa)12|tp)wnu0s(b-1Q8q9_&~KM z`fS{T2UVrvAfksj=T?9J9Ei2P4h22!Z3CTOlB4g)It zGQbag15ZOB{mJAx3ydy&AZj!)5%iq}^xS7%!$DM8J`g`}HNZ%XjvlK(K_`;l@G|xG z7>jGFq|pi)_yW_lSI14KdsnNpo@N`5PF7ct?zCkF5?oXBmDB8^5{m@uh=*rBaAEde z7x5&=uoeOb!AZL;+vk8<=UiY^aC2FCkVC6YX~9ae05!#5&))LuTD51cyi2`C+&l5{ zy6y3hNJ6qty=+CWdEV;CM)G?^jhlcjFt&NLzPQl7Y_5ubbaJN!P#rmQkpzu=7OY2L1FpT)is+4H;o z2{JR@BO}S!uj44gVoW}no{yBsDvvYyJnTA5%fDL1WqUz@=lMPSapXt#ZJ$ranc)u4 zf)g>hxe(tJ#V9?pd9MF5^RvY{z4aZ!n`Tt08J6rR_zf!TihI(|X0(nKqQev(EPOrIvr5W|CXi8Cmym~h2|yQuOhv)^!#QngR{Q4?KSoj^`>-Sq78$l@lu0+bL!{9_74qV z9%n^MFFRVkIfuvGIvBatSViiqP1cwi=1KfV4quTwr7RScSLVm5_tnNk+e4` z8Zg^POigW^TyzjXj5FP=RUZf$Dvxl0IQyP=2=!D6^Y2y(T&6@)*#2ohS_)>LSI zSTq7G`h!b9e;mTg^JD)Cs7Bu*UUX}0gsO`Evf|=CdIk{@Ph~iUbneJc$u6*|rFXfw zS|YKkCUxTDyXBz zn2k6JDDLYci7tHl_)#77Ut_lqPLEI<16f*Lfd|Im$oo3vuf3jT>JM9#@ML3xkU^XI zv-yLSp7TuqU&)^&7BfeYsIm6vjZs=KiUMx_KW#*G4x{y#3NH?5fF~=3Di3fW`F(4OKKAkD2QBuX-+2-7Djf{ zO!kXXYPm$%=G+&-!0tnAsf$k1LS*2(x)g&4g$e1kk(9 zzF=5X?+G9-;$|N2uiXrlq}mhEwV_*#u{;a<>EH< zC!o$AJ7`M?xJ9pouGxkkVrh(GQ$2#rx{&+!2I6GJ+pyQFD^TU8(2!XP^4=1bBNN%C z1tbaHFhGS-L}JYv{etn{f*2)DGC&Zc4dZLIBQnhtTL9v`G1Da93zD`;r}7M=pjj9r z*#xLi3N7Tn#rcVZYKsy;j5YEF=b0rjMp{6oAO#HLlm{}aEmIo?nI$n&nq-(DOPd@e zwF8RyRAB(_W6NdqwCY$SN`pLA0tXbSDKcbAx{-YJ-4Qty6FQu=3*vv#6+UyDe zzT4>g!Tm^SY&GQL(!AKO_O;h zZ}0FOrP_grlcp&kDSj6n>kLJ}iKqb&SM5arsanT+H|B{jwtq~_ydv>@_DDFfgA)-K zx8H3zOdvt_|9u#r6EX2WBH@q8n%fnzq=f5%NO!%Dq6Z@X0{gRw?;3<}swWuka(6~i zQyawp;p{}86~42AF_V@?Te$kGLy~`f0M$dd9%;C(cU|)4a7V8vT-0qe3K0=%3*;Bj zVqjtUmkV8&`(uDp2}Fre=}_OMdAhxbm)p^*e(#4R&1x30-nsG(yAO}r+nL)u149yB zKr}3zQG=MK$@FBvvUfc5eM#>+jBq)t8DrnwpkZ}zE9Cze2;`=pMQ}E+=w)31dviAQ zOP2+H^MRrwGj%WuS%4W`2m~q+EL$oe0DzNha%6OegG_#u8^d48S4M4ZQZ&R|z4J*C zC{!2IWZNKP;)*fuiSU5{ikoCEDj6@^oxPE1xtUZF`zuEqqxtt;rN$#F6$)`dbS^T2 z@{c0NWhm~tcPS@KmqJ4uokp7R9bfm~*^CWQgZRoAIFkGXa&TAzv_*jrh(P!vCbEq( z0KX2ma7DrxRUj*hU*+M=K(lDxFeXNbueC6lkXif;a-HJ&^}Kw(5m%NByj7)9P#U`P z?|>F@A!d=x_2*2Q16DELTIw(0%$Sl@|A;AzbIg^}_TbB-+DI}&G0}gH$jtcfh#hT) z22{guu66L+@ZQm51r49RL++KlE0KlG_=LK()xEzn!YFM+fm-^>#J}rGQ=|I$P@$jd ztyT4-^KR9LS3Db=+aE8=jetPUOd#|BUrvYR$6veZN1N>JM!dWNwcL&sZc{fyJ2$!S zn)<8XVU06Qj;uEE`umkma9r+P=|;(ms$2eaZ3|MqAO|gZGKU`cD3BJJIFM5^z+a!1 zj`Zj2-C*`RV&5`QnGd&S)91wa2wNlu)NTV4QBM69OC&`QL!XPb-_tvzl{KpVtieL~ z?=(7+O(`9OVg~U|z87-sGuGl}Qqr2&b8wjwV6QW(3-AXyG{l;`s^hA!F5;Z#*_)#b zoiP+tJmx&jV)}~DNct5;9K{DQJpiJ}hd5q%3({X1dWVoTwMNI@t9Wp_hGM46gRU4C zb_3%f$N7SC#>HrHVFV;86o}SD(Sqrjp69qnRc>_ zHwt@Op>!OpVY;E{qgDk-pdw$13DRK}Vtf6D+Iw3nlBj2q2u2N+odm-*?OlTG#KKO0tF5`fXFz=k4P*w=PGWG&022vj~ zZ9z=_H>G)+sH%AC74-2syP7V$KSWt%2uD>d41Wf|HXg z{}D~^dvy_oxRBBfP3>`~th#rc_mB2R@6#06YYSlCmKc;b$9QExr@M;6Sbn zK)P9FL@|%zg7ag989B) z6wV(_{PQLbictWCTDG39@e})V^+8{Z_}oWNYO%jUt|e`o={Y^Uu|Vm{e%{UeHTTkK zH=E;7eX6;(kIcx%0Zv&r{(D{T5!aRAAl(0jvvE@2C!`|R>8EyirYq%5;TP%;-L!>T zROdvNpF14<{#vQRt~+r@{Mi@XBZUPq#UVg{6_4 zRy~I4D$~PPTzZj5yDB*zJG3*TQ?#hafv*AYN)kyn5d@ic;(*8!-`a$$h>|O{EJIU< z?@t#1Z5va5{*Mb_C6#D;l`KgYR&*C`#4x>QK(HjuFlFH)NrRVJ=3ZsaTc-! zMhi>Fk>MXug3r{|W5kuEGjLmm@kSwQd8MN&hHM0RBwL{h(fi*{Vw zRR^siSeb`A)5P*==}`(J{Y<7&AC}toOK#@7q_&tT|Biail9{Bziu@j#1oiKK7xx#$ z;{GHc__^HfAu`NlyJkx~?xsVo4I$D>>mjML!)++Y^0}=Tw>HKfzxeR1%ICM}(Y8dS z!&DUte;~^OaR{Da=XiQktI<8r(H?%^4h7Ik?(Ar7?kyqL;Gqz>YOxW~y5|~68Qb^Q znHz=rYX8r)Pn9Vv^Z;J%sCs{YL&wP2MrLbHzoVnuzVed;Z9}OskA)8K$+!km)t*RG zse>d+w8YEKxMS_dPv^hBUlh^t7-YI}OaiI(Waq_6VWX|;B72*a`ls`TV`rb>u3#K( z4K%$OGVLg&_enx!miJgnPP#MmGNkljZKP&t%cjp@swkiYISzZBcA#*a$8d1J0(x<^TCAV5)fknd?xq*M%Tx*r^TevrBdwH4cS|D;5+Wr?N_U5VbO;L42uKQufOLs~ba!`4cm4A`@Ar?va0nhb zXYak%>~&uRn)=Sdq^UyqnN6ZYnw!6N2L^B-3BQ!G3{S!AO>LB1Rk+W^hyPXF4mk~F z6*V4d);3gzAXy-9Gb$oZ!k_EclbEbN{MwQ&wbYAA_NOF-E-aOI(|UVpt}pqepS*)U z6TGRrHhaa@;awAbPK3$gxcSGFC#WIk5QeF)5?t~+f%6Sn!+DDWJ^i0Ot6!G0%~T>j ziPaxWk`;NH3^4qk({Ycq8k45AOk)=PHtXY6aZ*5y61+yy|2`2gxZDKV-sX=r3hpHh z|6atM?cf`1qm?Dt9vSdM~I&9c#!gcWXipOp~TvjILPe{E^ zU4mpU3-Id+bgXf`eUKE?*~qz6V+g-^j8}ElYzY!981bpVCh@;i7rdx&0TUF_wkfUt zmSADA@>-gsedBXGfU{FpNO(l9_P%vhomx%|`ke@0hUSpezYsUVgq8cdI@L@FhQaY#nCY7?#zP^3Il4()_m%jI%0=DfdL#cb_VU{9pJZF9}xug`F&4 zn&+=beDMno^C@Ku3yM?sY3J*DM?Q`N#;^HzZ|+u1iYr*5!dbvF<`4|3(0i=N3Zh?j z7*S(=4}4Aigpw1;U#i(>(o-y~(cV2FIT&Tw>1TN4w@xUfs;$>PJFD9hzsW-pJiycB8@oOzlMpq|%Kpc1P2 zM%PKe<%E5CIMep3-1`Zp&lQI%!J@f^^ZqFZOR3qO>Vn}G@4{B?-b{|*Ue^<7`8T5- zE!RFW3JEJIF>kY8nZ2R+z@^NPu5f%QIltIDx;qk-w%0jXyqaCW+-kvgUvx4;C%U>k_j9c=LE{|tkSDkIhfmYX1(%s7CYbIXDrH4Z7gD`{QvfnV*jjOTW zm%?Mey-W&=VbW!LwcNP#QGsA<_I8cz3){Kil0C#KA9L`%lNrSN)_}Xv^BRfe@=8ef z(veHt;e23h>NmxX51hs{ymoTvd-?grPuisuzZw#prmuy{k z%npYx!$??`o0!t*9^H?G!@(Vq8g`>}K1&fNnm>0(U1BF2aAWBERkRj@=6>~DI)ttI zcPG33ZX};`n0qnDWx#~D;&D7+6w8z-|6%XfsU9>=9f`QOu(!rFL9ldhiALYpr~eH( z`D@rNchJPaoOC?NKRffs99Q{~7O#KvWh2o&91UAR&8|vljel!KQw&x`vET`-Noh!K zwh2?1{zbHO+9?HgM7LRf=(P1Wbp>72Z^oaA{ykneVBy)17nzX;=SnY|p$;bBp{PE} z>h^`QdZ4THQ|B6D*v|c-7GU$^b-h?zW}S&$rr2?7y2T1&nxzw5uC6S{YGEha@*4h_ zEIjq}e*E|y9AE6Jxkmn1>BKBjvn~B+tg!5E;P0n5j?d!tFEByzHZgl4IIPLK6pmHr ze$QXt{rCa~HCy7WJWiVzT|VN`$h}mHjg?Njr08YQ`!Klt^>e?OsLER$n!LBGsfOU) zRJ1i@9};E%%#S-p?21ACnYq2$2Iyq98ub0|9>Jkl2he}s^f`YLSFF<(+n_B9?u{xW z{}ptkqH6M&b#+uvj`Gqt#H{#C;HMZO{^Wl545uxch-1&tw45w%F$Zl0Z)4tny)!|23 zvH;Dw(4_M=Q;t4KFUZgPqaWVCIVhhxZd9|JDG4IU!IX`0bvcXQ64B}?_C6+D(dR*z z{5W2|LrbdbLMq_Ay(;5IEW3v@VmZ+>8~==nuc;|xcBSAdz55aKX zP?tk7v(ZJ-)s)+yptt*x+v#NLk67wjozY3Saa@|LF*`ebu3BAqrkw6OG{s?>Gv zcoSy(JgYY-tm{wggwO1$;`JT533j_WZAw-}@8C|^p^o-cA!RoG9i^zSyYN>66Oa@R zM{oVcg+eCk9Yju^RYsOHJFLy-pg+;DF&M4)-x~AsokI4CW(ZjjH>P~}h=H-S)lIE+ ze&5xsck|%j4+jgVMm3f@Fwc!_u0$s$H*W6ubkHd^G$LF%wx-UX4Vz#4Tb(wsAmbbb zZR5-!?2MwXA+tgkcZ+LNdJl^0I@7$Jc!1<3eZPP^&PpbCO!oz*X*NIXIsRR;-G@JJ zW9$kIIH5hxm*t*RT@Thywyqz4DPt%`DN_9)N~IYcp|8J_S$t|Ts0abg-cp*Lq{>^y zaH`**LYnRVOpI&%i2AH=J*VX)MC0}I!x05$0=FA|A)MW&Db{K>QfiP8H{yW!rz@2! z{K&EekC+8<#e+IkEJ`)qp-*$@s07AROL z!-J9O+H-ed0FKnwF446{b}ZMHwhrX)YduqX=8g?V$?2FsB)G-$x}0kbs1kz7{`uxW z@%p#}owA^4vG7EAwRMx(lacGH z7UQZwI+vfo{}E0%QCJbr>i$(Mz?Fp>Md$d_DgQ4MfXNMvoCxKIU7I$=wB>RSjHGbD z`>#zuLBxA||AO>O77lDGuk#a<&DMYRo_Y>5ilR7fP$cGk&u_8hwr?KhxX0y5D;D(i z0CyF&&QAnY9~Jx3X0K7kfQ*%b6)6hIgIC;2L}swQ<23H$&ll1>c8AJEb(bgusxiSX ze+CMq83j(LmIe-TKJr*m{(S1tv(*3gs4FY$@F8_^)EJonp%iwLCHvl@8!5g~#X!D( z8ui;OMm3;9fOh`oL5Z^(GKd`tUiED+DL}y=M&bl*+pxb+>WB`_(|z_V+kYp8Mcrt{ zqw`$*1wo;SJVo0UX3qjHOKs1{Z?EusasJtP$+`6Sao!fBk6T(mVB-WUg1|Phr z-k@$nlXvra8bvdWWAl7n4c0i3V`SRhf*u3~BR0!tn(-xGqr&ILPD~a(U8Pm4Iht?! zIJI@1+Oc5cvVjj>j=85+GWp%|(O9!z3|%sh6Xi6!>bTlVF%QY0NT68YxS#x8s(xY; zGhIMYXp$n3aMcpTr)(dPsXqK&r_K+KSyNXoqCvolBm4felFYhx-*Ci+d1-oCe}9a{ zy2YRmkN*=koEm5GW%4`mQxHY&>pIEIp)dGpQP!_A{+;Asr7<@+rtbYxEbmN2Hzc1Q1hKVIV<`vj1@HBXQ6uCTe5^KUss@Q%rvGlPlzwq5si>63= z_Ag`wb+@hg*9k(KPZtcW%J$DL!$`_wA{EfrOCeqG7m^kk|YuY@B z9ext4KjrnT8SR8_<9lAF3Q)%;+;vjPV<|NhMRZzIvID0>=rr2od z2eL$pkP~NO{(e#piJEs=qOJ!HyVt%9r^2@-WxmhqPG#g=hO`adb0!0-u^;KL+xP}s zA?RrE-k0=HHeBTT*ygeDYGd&upqpjKmYkkL&$HDUf&HGjdpx9gidN?*_4v zb7n`|@?{u7{ zl11NAYSIiSl6U=1DcFmg7spHthNQCnj^jrGqbG=%u14VHoynQGJNU$+XimuM0<}$M=Qf?pJxSC4@s_ z4MA1#x&4vJ$oE;r0y*Zy4z2@cxC@{*i|V#2pT2u;NPT(aK>3i_xOpW7tGzwDszuVv zhyxMWa+OD%ULuF1MQCV*g-{gNeF2+NwE5Q53Dps5KTg=)Wp{6Zj2QRa!EIJ@E^Roo zR@C6m_+MG%kFU27;`}u|6aSAd5nc6W{BRIehTuj-U7f72m`MdY;+cfkPkW(@uovb| z<1MCk2S@P)r0v)zx}EJ|#T+xB?1H|~z8ws?Xg39Jb-b6Vsotvvmmmzyv43F7O2 z#;g0|niQ{o9umT6gj`b(>i=KYuMq3{+Ka7`cMi>fjP{_R-4~w~*<_Av?lys+31WTs zC?bDv#Gu9f3qqY9-%CpN7$!ivs1vjSQ!`Qku_-B>X2XmB^QnNEnp$rLRmG&2H^4!! z%3`CVZ7bT}YD>dy(3X^J87%`8r)Fzgn}~$K(a}-Ek*lDZ8IzEZ*VA;u`8`+0;n7hM zMFE_kMf|XqpYNTa=U+eIG%XR({&pf7eiiyQ%!|%!+Eeqvm&4W-EZHd=8ygIMqUlWd`T5kqJ5LZ7fDs=N3v2881m~ETm?91`ERPTv zhk69qznwu)`hN{$6Bp7G&=)-mo86(24|v*%4c~B>wJqCs1O1~m$eDnU(1wFNA}P4f zXaXY%6x=ucD7O z92R4u+92ZaYkMrHca@3leq4ANJzTxaE8>dT1EPbK%vK9g&E4qLBkTAi8>FLMYQWm|Z{XL+%=5Ybqg|-6Q z2x&*e3=(KSBXsap6$SLY&lnIg^4l;h>m?V2O(FG?7H3icyr1hN3`m z-7z^rf__`*po@b7A>H~CwDe)*hjhC;fn!w@6j-l`x2J%@1LB?mxu6W$1(SPmgFP_Y z*A~MdXSNf1UU4SR_4GsMP>Xr|dB?xz} zjl*Pa(dz45ME=X`Sot1O8X4J0(F>2 zF-}<372A&iLS+Ge3XG4Np+VW39sl)qIN$*V7`3djf(%L(Hrt>Ygj#OZQJx*RNy=0krj{_^oTluENrM;%|P&!?4qQIMw=_{1}#-?Yy%x`2Dir>qLc-7d*$x_?*L z{TWkv8`7FhlBk{hr)A81`2=07)Er%XxEzZ*I%3-7+c3oLO%Q!cz+NOzN;$psW><|} zm#I~_NEos@+7e!`t$jnY$$Nl>!hCxMEfp!Bj|*pg-hg+_M&)I zc9ElVfmh3uDK>2VgBFTMbx|o~e)W14DH;CKxdrbse${_G|G)_4C39xi8zUj*&JGn*J=iVG3ySGA7z_K-A?vOc{~bk%@0T$0rTWeOSyJI>B%Iz~Y%kSO&_%cM?1zGnH`+jK4$)ZvwXJ6k}qO zy;7sfR%~`1euYL)jh2U}*@MLBf!g`;b|>HvP9GumtBCQ&P#ywVK{HtRnkVkr>sF;# zKMdf^7WAAdZHRgrxPqA0zCHh%xBNV$ldE(?F2% z3N*e&mYdLNmRc!nvNAH>gFxkAUtgaS;5G(91^2ua2rv6*IG~r=gPBzE?^|Wod%b(Q zl_x!$tPrNZWxqF8=EUMr&;J4#ryudZ%2>VN-bLPJk49S>XwCW)bMOAhiKBo`OZrJp zd0y^M+sl{2DFllN_KR95sF!A{uT70lG=uO%(?golnHbsMOC_bG=uggp<+0ntscK<^ol1O5u5cM_xju%O>R%e&U z^z*k84@E%ya-f<%c7iV?N@Q9h;#=z9T~_9`*vRKLa~Jf#qFM}CmGxeUD|)lt%S$o0 zUy{I4YB)s;WLWmR-t(LY4D8EFuM96rAuxB|dJBgdPkleMN}(NIKUpQ{-fY~o#<&$; zCge^_sO)>Ao?uK?eeE2w1R@9efdoi=}cw=5Z$sXGe*&ANED|gP} z(?chCHih!b>x@ewH1x^C$3^L~z6~sUb{};(Odh%@)kV(H`Xq94Pb8d-NO$>c(!nqM zF;3~Jsbz-uS)D}t=Q!n%*2b;{xk)L$h(3qNAA`Qk!S%-F?1awcmD6_5#6sAeY0W03 z4n=k~R<2XdiURFEro_?Mlt>n5ZciH@%;C%t7(DCq1xcTwd#*?SqGAqmQPDcN?`U5G{?K7GWV{5#KAD-x)>28)HiQUrnM|LqtTW>(wlo2jUA41Ejww{C+LjoY94`d z=@o22+IrpGHon`!tjhl@r>r*ARNos@E`_*sTn3(G-C(tSOP}9&tt&nscXyoYdSS=l zD3e*grzV0oy~`tw$XBXipx#VUEO%}ny$l+#6IsSHCfuuZ0j#n0w4dtLX7F)Np= z`*ZUnpXvWIS_#8k8>ynBS^!n0K$_TOGNlH+>G7lCHyYosllL+|#Efv=yML-zSDnMU zAA@;%@l(mo?)qZN?SN~Ih{|F1I=Zmtn8W;%_>WZi;>Hu^H`602PcPfD={Cw90U3)r4+2DU z9$qGbu>9Yi`PpmRRqQ{l61!^r^nVQ4Da;L_Y3FF>^Fe1SC^qNr*zOs#@aqcyc6;p>CZVyKs{!5YBz!Q(}p&F|Z z342mp6BUPrCM1>V$H*K{ku%th`{;2|<3wMuVoqj4#=~2;&yDg5bJ|OlY$b92#= z@u{dH=u?V`B%1(@mB`1)X!5(-X;U6@9VHSXqPM6xifSL}*OOf?-wJyio8l8tHug0o zc+$Y6xs)DvbOXzI096eRtC<1IEs$P%pC(P4GWyt8%;4bw-}7DUCLav%Pk5ekyEZ6z zqCu5ptx@D1vn0$5=Qkb!&huZjHom{ja$d=PC-w8~vNpBn3bCqM%-ZOa&*;}SC*JJiyuya(~ofMAtR~eyLR|J{7zY(d+XWz<@kNd78oihC5 zX2v`q=vKcxt*9?j0Mh+iK8f5R?_=!ayncRE-)s;q&gY!i!lD|BN6%5nXqQa5C!Zvk z+EBNLYkl1uD^ZS_S)GleuKtDBn!aM16|GJWR_rNiLZe{1U+G$2l}A+gy@@xvM!MI$ z9^!l^HxIRIJ}FX-)tz7{Fw+S8A}X<7$;>Eqe|?|| zRhQpouIJHcw5D=Ls+aWX+ilX?lPD$I>Z$MJy9^ad^>@1q5G37fw>y}3G92tZ_?aqJ zdnQ{y!s};>BX`#)tZFOP2z8iwD1>KQs48?KMi-5{KTv5|Z0;R#Ih;_9?gt4JGYLNZ z5;tN!)$Wp)AG!9|_NBVCRm$fM3(VQJdE1yI{qXLU#@^l_6Zg6oZGW`v55mmt7t7-b zX(-xH%+1QwR9-2>{^&F4VW2ie9myv&)9Wp|$Fe=s)SH8@b7{jl_ihe{7KaysMi|nj zmj^Cga9Y|drzov#VHmIX?*opV)@|q5)R8;uwS!1YNGhUu2G7rRv1x|V zf_L$%a~!i|0sl)=%O6(8+>UYIjQc3tF1&V0Y@o(DGj3FSqoIS?3c7~m@C5m2D2JY z1Uejj5?NSX^v~Zg8z}X-%Oc2@f-lazwAN0!g~PPh|1rFB35B1DA$_#qGCFly;GiOB z$8>uog@`ELu#~TIe;{nVzmS9UWG5u87+2x^Xsbs~X}UM~nE>C%*r>tf3~?Q-c-(uU zZ8&yo>co;5MRoSPd5>hXvg!A{&QBDLW)5%{?Q7PP`myid>Nd=z`*Jl_m&;FuR+q|{ z5=-#qCSH21{^DI^T8A|L`>bP_B>5~A6hM|p7J>G!+D$MoC3fuK2tTy*@h(@=`e=aw z>7gu>Amiea^rZ(m8FhHxzXqLXMqo;`N%sp2hc=NOy;)6d zrW8)>&YOB*$t}R0oE8kNK;S+1uIl5M!60jeTc` zY)MPj(SBuBu(HaAQtI2%sCUhia~ee^ii{9xlU)@MVl}zGJc~lOf4}SNYZ=vdnWLM4 z{M^KzLfS&H{BJcIX3&xO46}N_s9=_L+>DVZEwgUVa*|tFB@koXAHe+B^O{^4ixY*) zx7bX4i+SYyPx$P2@@AG5%Kk@esfrIPEy?%Avj#hM!>r+zEz2~J7lgG+hIz7wc z%#EX#*&%rEaqRl1JhvZGDl+L6e}IGCeVb>GLNlUg$_aI23GJ$NfWmX=E`(MW3h)C$ zvFIV;xMMpN-el%lJ?zZqQF_G#@;QSaXSfm$i`p-vMSVny$hf0-2-kMnh{e$dV({Mo zIVGEDlLGR!nPFc6kOAb-{8QhM8r$35=z$JD_^G$-85m7ff9ELLVuaSyd_LES#Q(}c zRYrZeFECbHvs92bIw^wR;!(tYCcSa_p3uA-4R=dH)R)D{3Hzz*g;BOVxty3MJ*Xx8 z**sExsnhg*+y>^F-n>V-!ob*-99}oG-+zgGThbULL5pe^4B^-izGtT;_kz8f8No0! zrN7>{_Pi;v{kt#Be~ed|Y$;z}c>=bw&60h=xyGDeoEZyh@!cy(U-xVhKTIw@ALIab=t(VXcAcuBz|1mqIBtGY!YwgksDaS#tjk2DKxT_ZH_Jj9) zn!HwApH{8+xVa~sw*{5!OPW@k_Xcm-60_p}y?wmjP=fMJ^l3Ue<$sH?#<_79C~{J_ z(9!8+3snH8LMIijJPTzHZ_&p;UL&+eEfn&R(W5auB`JOy3gon=uEn{i;nHYxq+}pM zbY1?G_lh~`=HxgzVV?_lN;eCdU8kroue=^TD_GhaKL`oOFkWk{zjb;s5Tlrb_#d-< zo9c(|pVKl1*LT@Ac_)b^4cN&~*~Bm)dXn-_bx2jtwHm6TWUeg}3h^b*U3IZ#FaC8| zY~yp)Md)%Ang4g1y>X2&e=WkVOhagZ-=S@(;B4~UO?q{9T!FQxFM5vB18~%#c*%n5 zjRiJOjdwI(tsOnq)N#hIW3LAAsrUt1wacc=8#_MGjy7%W8YL5PCc~e<&zr&=8ru2l zmp$rP;I}o$yQt^i<~QFl-e-=XHcx41dAcYw@`&1ffw+jq<Drlmz6bV;7E1Y<18{yl3l4|2okJ z{edVpO)SGb>33%vq9n1|!tJ^hQ8`|oMAlL9&C);J?Eay@e5uWv5H4Rme|KRqgO{{2 z1^eZaRny|HRP8D@)0M=++d*e;4}4$yJutX^BY>`q{Ri2-(GT3JV?8DczrO2y;OW`f zec8f~c<<$ys{#^6_Q2N6waNa6%P zzjN^7p1E43#={?FL}>C%rd|0Cy>0 zYkeyy%B0kLkolNKuxI;3P~1}o_GGla@(Tzb7`@z`SBnH%390T$#a4cC*5Uc5>bxT$ zCTWa)K;r#}si1+DJDZ9-DP@uAAg{1=fWG`4`)E`RmOc%KunFO4M9rrZ4;&a%X`$G8 zYI;>o1I3^N=TvFNp=C?xpa-Qq;LTr_4y(j!f> zSktaX2&~_O-~J&{VGFzcY8avz&r2cBm9KJp%I7AzHRcqJowj2*dz}d%Xu-Y;me#kg zO&{_%$oO+vDip~58`5wac%lkEZe-|cy|aH<{HyNxtY`sl7%B3!Zv(RA z6=SkS8gB(a+XNiP$E*Y=o|LIb*pmVX(AC=|o_V?adHHmEl?- zrBH(Z^gnv;YN|UWD)cVF%^>|FyPD^kj&C;SVt-oMa?zO9a5i{uO!s%MV%J9n01gfRT*)S0GM*yHf52t2 z`TeEFtK9ED`{+2+GR34TRnT(c> z4}Bh2=fw4cFionYo1v*XHNF}GKOYnUzt`sC@2@aC7tcIq>Nh$lTJK6qp5rPE_Hp8c ziqe*&{YorupFOa>R_Kk>FW)5dFn|3EMFVE50%lcgef!H!ZNzs%!uCD>GSPDL@}Jos zW{Zm2a#u}LLt2UgsFW1N`5H<^(nL-1hbaDfk{HkBsEbJ}IwP|Ez|4;pc*aj|f6c8- zMh#N3=Y2vU33cONbWoavml(gTU(D#{w;#F5x*oV6ll0&B zo2AR;93OV9y)f`)0gpXqwmQ}>csHygIPz}J>v($b8+yfPW@hBD?Vgq&!9u{;Jz;pt ziBYGwTtzzjW<-C6{$XE~=FK^traoW5jy-Sk5^0?M?yhX)+vT}UqX*2~F(BB0PemM= zzj0dUrW5gL*;CP~W&T6x;o<@6L5{O}ly+1a05E>3F*I1pB)}Ipc8TGTrTt&m3 zyA=*1+wi&Q*9HG0wv{s~HllukCuQN)Eh~6Q{3iQ$k+ABxSZW#ohot){d!M{gLIEuTc=BBFV9P!dAz6~{iL;hY0oH%gwxnr z+=j`EM3;_`&o4*g&lxm#teMWu52Y{HfOEY7jLf02<=ag_uuz5)$B=Oiq++r(WOL3l z6Mti-XGH#L!>U9gU5)(NyyikN{3N-ySx%;=|N|%x&6+Gdzj(EcEX}# z%IRNVZ~ev5I~wme-7cRU6;KC2_wkN^~87I+GjV5Qk#;ad^y#?{^8H@4e5<9_Cut6 zBwW?kBVPR(k}eVCT%m88KS1Q8Ze&C^Sz!}}&X1b*A1Jsm1alV!Z6ei3`2Ia6;^AVt zn4!=Flp5Py_uZKH9k{st$7D#*J*6zOvN1n0D&F$0(jV6tke9q}oYS=Hb_FFrn_V+}1e!kmz14(6W$q*nCgE27g*Lkov2OUCDGbnxVQF5ms zGX-{>phYL{fI)kOMWX5EPcFWjzXDokQMf!Su4g;9QR0yqd_U{QliDJqiPD}S3HkLm ziqcv`vjgZ76OMJA%k29RKtE2Fjb*G;!UW9*@)7{P=#}l}h+M5piik9z)qhM&qp7f+ z|D9w$57@!iJO+)vEavkd&rsWDSaNGWVr^|LCQmJgIV8QxTiP~#x)w=F+!S9j%8VJ~FVFgUY-(Mkq zSjGy{Gs~dK9Ax$e@T|Bs)7GDD95;s6mjdn(lJx%u#!&GXkS5Wd2_oLI+oUwbW@%}u z4AHJ;8u$(5ZB0`XlYiNPT?nmz7B&`t?(r@0}f2QlX*Kz^?`b5C4f%1ovTnfJvd0ccfYd8rS zE_FC?r(TJQioP^4SurX@KnUR^zqlMK8dl}t=xDqNun)tjg%aan90r|{fdM-!5H_L; zbniJzp+PerNGdFa3F04=BHqby&>9rrV`Yst!3pw*SM{sv=;)+<`t(V-jMeZDMt(to zdG_e00)$!c=|Fz$3%3>dGvl{!b!bRPq&NV73I16|4ipXY6wKZL!~9cx{NTWkH_gX?S+8Z-<7h`~-Eh$5g_iFW6xM%8J6BbpX?cCPy2?z;*MM2gMQu>Tz?lSQ_IwEd4t;IN9anKaDW~ z1Pq!81Vlsuht0QQ9OG`?CKIb*p;o(>${PKn5>Y^kE5DK?Ddy+Cfx>Z){qgpk$oXJv zt-xxYi%qr3l7fut6Q0H1;%?XD<%-mA+j%G&$g#Y*YwD70Ys8B+y}0#!TzroCj1uVn zkH%Pd^K*7FWhHV>o~wX*xzAT~>Zu+nODD;pPup!8np43m-7I-#r?vzBNqWr1KVMNQ zMg{4&$s3TfQjT>N?|9GMrp0-f6+tfK?wA9&?x+{kAgK^AS9}_pD8(G{oWc7LD)kua z6_dic0<#Oc4%&{6jxE4U+(`mr=00p4sfnUCKM|yMQDUn$B!fu=-IT|ga+Lyzz3s?4or^51 zeDo>bkYC@j0Y@nf5bPRTY9J=&42DES;Q}g}Nmw{_Cl`8bD23@YO0 z%^teAbM1WSyYJt`N-1lhn;{iJel+*~ zHf9FEElH4C%7l)5(OCz7y~U*wzHA1C4v{9er;LmrKzj(TtUW4$>m7*#FgItvL3t?% zRb3sCWdgt(xR4L;iU9Ez7@z=XqoJYk4$d|Qlq|q*AwTF^wtrJG39?48IQGOj6*BAV zsZ$h#;J%4rT$3`#W5PEX4=>50+WZIC0X%y@V9pZ~=#a?BB6JJh|J^BVzs?KREYrVj zq}5bc$`M~-Z!KH?Wn!BvxR3AW$g3_UJm6-0b)0c&bv4Ab4{C^eEwg)o`49PS~v8vjLm0@0^^D`~i?jmzj}4govUOwTDM$BIEr$oNCsr#Ip`%us4%1JSqw@HC#`p@Or0iXR6 z24Iy9*ZRH=jgFRD3_k{CHZf;sZs}wyfRJmihEUR0zxj-FzzG^0#$(OUWnkCF#tw7BxAsA1)i}f zm_B+2hWi2OA%p90L4&Clb^u^Z=>ljO7y_lK-2(LL+!31?u+d{+T_P+04~`?Z&iiD`eNxt(UN?~;;_1Yd=Lgl;zBuc(Z%b>0m<}wS&`>T8WF_}pQ^B#RrvPpTUIudWeOF* zIDzr1fA#7W&%4B5WW17-s<$istQ(QD4njvaTY!2!Hue`<4In6&-VWI0a9F&+>ScKF z05Pvp>V4%7XlOZrB0+~1<32o+)ftZ+Q;G z`bMa5MNtd&Z5{#Zco&{&snsMmyrWP|Y;4K6W$pP!fglXlot~c$d=*Cr(H^M++2C1_EyfU=MF^VM z25V6$WlUoUZQZB2Q6aB1cb(=|bU@Y1(wOfo?#!>htlpv_Y^C=mTi6+7dB*To^oIJ- ziSwn&(5Ed0&hvmbhY^i!B50X5WN2y1M3PZM)noFQ~&0! zbItUXuUgr&HYAEZLahKty+_ZUEtik#(LH_~8W(^EF)V_$>@5Q#Z6>7&h@BN4j>Ye^ z@n@p^og3(J-eZ4@0s4@<5W+64tn80ATz`gdzrg&+9z5jIccaWC%sMhKz~l-%=X)Tc zModf$=~3k)!rkpH&%`oOohCME-2i+exZk1VKN z=aAt+D+wJDjq18kC1#fZ(;eZ8RaFUCYIjk*&JVv96-CDDxre~VE-`3A0doRSirD7p z`{G~Njqj!ixG>Vv`W~o=i=zS*S{A7qNm`wLeQOU%(%Mv9y0|Swkqc3s9<)a0tU)t| zXMH=@LEE8O-tQ6E=PWlN@NImBlxCt9>NEOw9oU}u6zdnpL1VtV(V|9|T{Rck9vQDz z<^KOHz!ZbO<Fj(+oHMC`0IB`dBL##8hunFhw&W~79K{tXQx zQ06<@8i#&mmnfE%(jDbnhe(%5Pprg$BWx6{HbpBC)l4R zy{X+?qdNZMmf2uYAkR&Io-w-q955LTh2?1BsF0MXMP$gvP>&n)}lUC1CQ0n#mn z(R7xdo&5nOg?Ugb$$0Z0$eKnGHpL%**pGx!0RG_s`f@4kIGw&S{#*KVDGF@?J!Nf+ z`BZmvA(Evh`rJ77#CT^7Wod z(}{ImU3xmuLn1{D# zkl^uir;J+R9hFCR z1pgVdb1gL5aB|3z{SGJhs;MfcBOtz= zDw3LtbWe81gl#WvM9f^%^3{AsH9fs-Ag=CU^QV7e>*z4aMLp>ZzjAk#IbJmS(t|4! z3l0vBv}D#dzcE;1<^R_8_e5RnPeOs#lh{FZ-ZPPaX>BQ4_kZ)V+|o{Z&g zrm&!+58~pg z8&}LZ4$V7r&)IP;ApV@B6lIQn(igeIr{m-}oLy^4TU?pJX0!Q^Kwz~V(@#xQc)+G< zadlObjGP?ms)Z9c=>(OOTDOU4%lV!^f1aEFLPiGTKLdr1la&CCiP~C9kWYn9IouyG zuX=ja&p=b~C-}Mtbab+Ea*G#3!^3+JBsnbnxq}Fr5g7H1%*;F3*v75l_iCZ5%87}I z*}S-?tfLd&xhz&ZI{3$;{HIy}1BlEj*D}J)2#ASW!tZg&C@2^&{r%{9xz~n8&KV3U zf<~O=cd)Rs+QF9eFvRPKaX=0O>juVFX=s!nj4y=JaEhb{uKEh|8Rtd!JLXY-O%ltn z&^6XuDHdWeJuZnW-oj=69g)k9wzWvG5u&z z6csz-rcZQ{6FaK&L$gNUwDyP05`vsU@~FvwNA5a$#iQTMspCiI4?kHXIxC9{1Y}c9 zxt5)-{SJI}B#LTKJ|sXP_k>(1S^u$LrYOUJn>lzy#CD(-a*zJ_5n5K6uZ)cuo@jh; zwB&XeH;%_ACx5|FG1(0>54>Q}*4Nh$|4mcezqLxBF!=Dvlf}|rVWXStE5xsun=^1f zTx&wW>!Bh8LKm}$wp0kTZEiLE@k2!)tMlL?)pf)N^;iiEzaa}^WEvzFQRQv_hm}aA* zj&Z5eDZM%tf1wApzBSyB5`%D@pTN_U=kiahr2Vxxfv9CIi)e3BIMba^ zA@ruhD9khp=cICj88etXbcXUaAnVs7_k(IR0Ggjy(L z*XN>%y!2ckhLi);_o@c6{X0-D8xAhM9kaQr<_=*Rr3CkT6W*tv;Ae*;2%sh3O+NX zy1rLP;x4hB7jr-P*IkwM^{XG8$k^=@$NdPUQdd`A7!Ko2QA8xD46E}Ap`GD|~9gAlHa%Wh~< zL_$(28b*kQkSH1=yRt&+dw+6`_vd%}{c*WCJ3G(U>-l;s zXrIR6B40O^->TU4hVlsYU^1U1M4GwSfZ|uj&Y%BFMxmoC@nqSmQKhA&v#hO0ta<*2 zNF5Ijoi}CKvVr5bwvEEu0V4nbh~w0E-*zZ_fhth@sT2r>YHAiW#}7GF4fFe^U^CBC zjrOP)AN*RzkO{|;mf9xF7BMJrbQ)pqh|s$i^n87N+iDu>3!Gw`fWCa?P8kiewX}ce znDKo^!B~sXK0|wC47w8FR%<@4_S4cp%RR^3kG$}#yiii?qQgiLOW*^380{=#FFl`jk1Mqnetc0s8@6@amMpOh3O6 z(Mv~$U^;%h%f(gY1|ciUFC7@M=tkikydCp!OifJ%&TT56ucon_HxU+ZOc-pRDg?d>k!U=wK14Jlm-5Va444WO z-FKS4jg?AS@3IzMFyE3Dn$^}(sjiKy{7u<`&KuQx_UyT1^QKMu-CJul_93q6&9u7g zwTie`G z4qdw3B6FKYO4V_vj8-diSbiVBetmgx?kX+x#X)YuMm_c3eCd(_CjKxTRN}C>zs&D5 zawG+(jK2i>6&K+flB>Dl&rb7w+J)Yw>a`TW>`y9^eZewk*xGg$ceYKb+rrA6E%qBf zKOp1QW}(EMXSV3%#fy6s8{oF~W&imY=marWA?J}WOSD)02oJ$z`v ztv87k7jJJ`IevMbbTFRS%*mvO*!i*RT+>&icJoXvzi7IA^T!W62(ic`r6D>j!@ z*FRa3(e!-QzX^ibzk`^|Oah3Ij z`){)gR=hLwzBggoxcbKASsx-VdXJovR)0^o7oXQ&b#z-}r#!c1J8g;|&e%HN@|tDN zhWR^VLmU0}Z{wVG5HM3^S z&DilUc45=t1)JqRjBQumIdFdYxwS6wO>K=Ane^s<`}F87V?1e|Bsb?aCbTZ;bC}5= zZ@)FB&0Q`&EG9}m(`2rC<>U?{4zFw+x!-TX_;2f*+V<@#F3)*tweBd(1&O8Qwi}%% zuTDu?@N@o^#Wf#hpS>7;P&Tkp)0!T{kB8n3lAPjS5R)+5sOi}KN6TCkO#UPRnLT7`J zSV{g1i2d~sb#?JG@ehMY$u|4tj;r=pZnI^Ere&MshLNgml>3XnYwn-n|LZ4H2JBx` zS)Xdy-&%T>)B)$mt@p>bO1PI$rZ!jUpmlIfL?uC4o*pOFLJ#r;! z?KPy2A|opvQpVPCtuXPG$35b!+$*vqEy4}V2DGO`Er{6zI_z?N7S?{1(p z7%@@pq{I*vU~Y>b~SuBF!e~ zt3NgW=MDls!1Z18LDWDp1;hs`ax`yf`kyy%{$YTcz>Qy@wIkNWr|WT53LZPJ=NO+# zOY4%zB~iwR!8q-$NYq>P!pNqT@SNn{Q@RlnWI&wQuXI~+reNssw6W+yx%5)9|oOy17AQz zMI{SgJGJlhqto!v-DFy|s;k(nti7sC|I6Fb@e~BcuK(mnO$&6m9mU9J`038N&MH5? zysHMyKsG@XPYDPn0v;_`us~GQ%RGwBVFOe<>4hHAck2mOv6ylJQO&*L;#+s_9D@ZQ z%D^bB?>Ta0GQW3do>v#lxhOMKRV`%Z1QlaPk|Df(hNfSW8{!hR#c|3G%z~xOdQZuG zzi_N92^3I9Q`1Z1e!Y5a2FjXgXNQK|0${N}JCE4JI%iN+Sl4i?pUd*)F@x@@R83NK z)a!hD+>OX}%16ic)*AYJ&3^Uxaho6SsIBeombWv zkJp@^7aOY3I%ki@)W&_IblS#@_#SmU+i$c(!kD^smG2Lx59yoR+eg=YSV8&H@TAu| z#Y#S_E@h2=os)gH_&~#i%9-UW<`}+Pmp8-b`}GYAms%`48inXreUV4;#b*f@@|$0$ z`nHewHuK#mGnpr=uTCod^g8&*rWY9(vYSR5m4v;~_+sods=)Gghuyocsx^Ke;@5V) zY(%5{^hf(fjhb$w`LeT*Zh0y--_!11-SjlBzN{axE26JivY~%O?8ud4ox^{=RsY!6 zXK1(g>K;EIy?8Yx?MFpSSE;9&3yykToD`*hcd7NkEo*fPERSAPs4McQ4tQzQHss@+ zfOGN@T{_Gi)}=6Movh>YmZHZEQ#Zdbte===;B!N1&F6^qR(T#ZZpSNQx}X@$WzeYO^FCV*%y8+Uif^YZC@IwNAMbK|SB|8u|N%u2{oOdaRGx#5CA`TG}3J5{wfwHoZ-&9(k& zdiTR}pQLWr8b$lh6wsm0(mNH~=w;4Adf_hNMb5_p;{sZ@>mQ$8AYJ0HVF;zQ_o55)k9Ou(M{#b&X>+=FY%&0qrJdbvNNO(5&J_>mu3kcUK&Ul{LJp|54)#L? zackV&PsYbDWRi_YL#aAyY|hSlvO4qbwe20c=&k5X`6rGNN>egd>qa4gTJ8~JKO_Ei zz0y_SjNBE&DL$RGA{iVdY_s4sBEDs9m(MkvJN*P8o?69HSbr=X;0PPf~ye~r` z$oO<`Y8gmSm|yd^6_3sakh6?f`B-Cw@((Ue>yGYv8^l$|fBL+h&mq+NOIA5t+^2Nt%@eomXkZ>t{g8O17o;|Gvo=KvfI?+i@K^Ltk9l}^6%|HsIISAlg7MP2E^h`PI0k7jpoI_p|53+}5ke}0*4qGKwrbt%lShq6{+?fUqDCGuMN`xj*Cm~?cy@zY;w{k_W*BwyF( zcDFHBjVnIc+?Z=Ta9+s++`|4txz$$6V?XU_qpWPB&}rx__S&qaNpprcWZ13)jr3%D zoC1MJCXc23d!MHf1>afeR%qC4b zb@uG=k5{K2w{@S5!q4k;yNfH1 zoG)E5aKs3!dA{FjUZZh>aEQy45>I#<$tIm;X}RZie*SD~YBmHMua*58H@8tzn$75sW#k#J5lXQH z$g%0T(`@Xg!twD53Fhm5Hh;~u@wYp_csmo7mT{`uSpNL}c_vk%*EtHqW|26Y6YE=4 zRP-9OYWC1WQ`5>v+rZGfvA!sloX8uNX%8Ff#GXAWBgr3-DmrdI^iar{ zwP&R60|U(R_D)qd^rn#az~(?+%?u|yN5@F1j#`%#o;=uDit?N*hlX=$xb$PR@u+uhP+`lUntO~y7iR=?hMfz(f}Yu8g` zK>OJZn6}r^JBQBTYL`(WUIt7Oj}`f~w5w9)TF;&xPl4e;NJzZVkGDECBdJZQwQCpE zafscqkkC+TclRW_lvUxcVL^!h4AwjkYFJ+zwGAaJtIThASI=6xGGUFrhWJ=9ENh^a z*0|B(7S^`5CkRMl?<`HaL>=rt@ZGE>OO9W{uBk;1?znS3Tiu#lyIoLV;}q_BkXy7| z{_wPw!@j=eGyKIGtw}F`oepGpSY3N@kl1?bUmZUrzHHgDWiz__Hb&jPb7z)bsQE^4 zv)gaT|Ew;ZImPs@69)#_#NWO>MO*=@HYX%LzfCF_cqs39xkYjW{Eg6gl`(#~Q`K15Afzb&G!JMS_UAyiq0YB9DuIm9ycQJW@ zhJi;AA80jq?o`freLAZsNDRVHvYD3(nU?OxN3p)zqYQ@LQ8Y~ieT4Sq zs~4(BxqA8XK@MEf>3Qh~zkgr9e}?dJGp(!wu2%NY*8VFe$CT2y;KuatFK--Ebd%y| zYk7G`Hc%tj``)8Rcimc?GqqS3kQ-^b3l}dom`h1Z3mW0xF>NOP!v%n#s3w?G>0a-m zAqhrD_k^ADZRl2|Zaet(VMa!p!+vFUQ5P=kzj0&Yl&MqK)n&^8HZ~qObf~)Co7%zE zO4o|D`J-Ptp0jNnF(+ZRa)k*_bNIgte(jloY?y?7P|JGCN^Lt2_h(-`Oq@TR3MsoV zjA@aL^E`^o1eu1}c2*=Vu`c99>cc2k0KD5?@fi(S7NSTIWGrf}6kd>b`M51Xw}e086I2&jzbACwdsL<JkpKQ=TdtqE!O z+g{*Bl9o9dIkvJ78d`ZeDZi(oVOVh2QSCc*n#5GW$>!$QS3{v8QPf*i#!*&aAD%uv zID`7l4V?n|1$_)Nf4*CND?h)lws!m7xmT&Go;-0P>jS|gTq|wC3yzQPKXm9))Wk9x zt9A~ae{1{A+hx<7cN{_ zH^su@1@d7%1`cdvoL+7LnMsuD7oO{JZTATeGSlvgorhQS46Mx|JV+(ZTw7C9v$9IJ z+rJB>Gs;p)Lt3?J+p(17)&;hbj-Kymk6|WBixEC9OAYe^x zp_Z^o9A^hG^5Z8@Zohwj*3iqJX27|TS5|i7ScVy{Yj1og?>gg53s|lJg9rB>Fkm}| z((S|Lui6ZM1n9`!-TmdWk=}KVXVPA%3szax_I@Yg<8xklMDOR1HFXu^@XQmy1fY{d z5w^3jCFYfsl5lVr{g2nhpE%(Nyx<+~o{K7aOA;z3o?7{rCm0VF*v0HeW=Mhq{IC)8Z+xu;*}|L=5*n73CTNHi?DX( zflY2wHOylbcRyIRgnOm>-l1K)aZ8u>g)HGWi2{yjq?J?BaHQ%ZDvQF_A3offUFFS6 zKO$ox`V;31e;`6N*N1bQ_8vNvyW00-*E3hXFY}n^1ff!JZm4P;hND?m@EXK>%-Z+Q zDFpW=qk~N+B_;I;zXba|8kstA<;SNUTA1?W(Nh$FK2fckkzW zC#8}hQBPh>3rAD^Zol53((vkONBK7WJQOX<9vzD2dBX#kTjCj5Yw_pbE>M^_`tyrKw{jg63mUiQB4ho4qmIiR2ec?}l6lr@sS%oUBce z4#b%=ui`nsm4EP^1PR;W(iyNw4(wMR{<8x%4!1-a{H4?h-`w;iC9~%7^2D0EqnhgyDzByNYwtH6JgpbxnCHe@?L3ms@ha9Z~ghb?@$}?Ra^BN~+8wg%#3v zqjlcPy^y@@uYGBE|BCz7yN&Zn92=uX>gqzeGO@H&A*KLjE9UMxh*;{Km)N=x!|P6+ zJh|Y_!vvpiHHYXiAT)dRBxTsdn7s*eQ|=)?i1!ZKv7?G}ArMe$Ii(5@gUM5-EQDE( zQG>SQrmvx@rdGKotus8tkl4xxg@q>L#z{gOhT0qT#=wE}SM*byvKdBnRrk@`Q1H^$ zesCbInr3dU2-=D{CJ;28HA|J>M`KVJWlX$=zN^2#zXWE9zKb3sMzot(QcqMbPxK?O zNCB8yTdOgQa)Oyz5%FIJSwLx;dq2zzoPL4j5;b2ZClu9j3~TSPW5a!aelNOrF9+Hx zTEZAKMl}90d2(x3XTqdOGp((!lNZ&Nr;nnT-bmCd7*&)}*kS^>-5<>?Sw842v}V3* zvV{f1bw~8>-J20ER#QaiN{vdyvviU)9p;Im**Y3al;Yar^3h^^(9qK`+Gx4z#(~}) zD-}vqTvYR1avhJhWTxWa@$&WK73AUr;sSc)hy9S0`E+cp^tegb@>78vdWZnb+1* zN@ILjYfa{EIwjkQ*#4Sd_jx5y3sy;MgvI{%k#l9rBz(4WhxLie;S}qutEp|_f(3Xo z+WkdDz$kt5N&j55vyF_ZlqVncwU$E@+-hR4OFN^95x_fmN3-KE3?<+rp#Ar-#d6!JW(DsJR;OQza4--SDG58V&na7Fk8fpVzbwKi-gAD{F8Sa zim{S^RQ~6ZtCk;&oatiwmFh*aDFuWTi)=6wLNh_09t`cs2({SiJ7A;&@n;#Ow z=!6B*^!^+t(j;^eWGFajbxBX&5#{3LU?!>&)bS(HJVw327oVs}s}oK&bkim&4rf4K z4U=Jy3_6o45d5aMUSnVfFTi$?9>HilYdhGdlexLMNLs`rg@TCtrj)RljZUtW(WXrs z(O3XK79qBf=|JtJ(CJFv?NR;=um>*T#ffka3uLu%c#R4_n=L|1Un3vXI&> zscsrJju`h&_?Sbap7a{(Lik=Iq&9X)GB;>i7Kl zV)A%Gm}|=+=Zma}SHLbuyH&NCGJvWEDL5q}iKNm$XY7C}aTXE?VF4dbN~Y4a>r?_O zz0ly*_W*S}OgGGiLOWpCu-sQ3ETJh2AjrXjoUxqPMR57Zs6uDD%BWgaW&ZJ^nEe^3IZal716|bWV`hk=ge(51GIwgJ;|Wn@UjL zfJdz$RfYy#&h)b?lI<84l!)1lC?wxhp&#_zfeR9m5gyC8Wjw6k}(d|WcaRY z%iQGX&dtpu^281sHPwwLWUu1^+ z=+@Q9r%@v^sQAAyX8z0|ox45_&u^XPD=;yW)_T&<_c2Ra*)gfK_F1Me+pH4u{IRoV zRT9biZtUuoM}aF192jaq&%uNLjA(k^kBD?To2}4_s+HggBKRu-4nU_KaCF33dTfLb zBz>QHaBwy(G*nLApR7#5X@c{`9Y4N>h@{>enC4q?MDql?Wxd~?%k4aN{jZVaouKiH zE(*bp@M(0-*gr!wx|r?kxHlwZ%Gxh^^P9D0QWXwwuT`?6i7F%R{P~9IS*0eOO+CiO zM@Ls}+1A}RRjT{{M#HT>JW2U*CV6>@HrVy0Ui)rc6B#qvL*m*GPay%jyX!d6J0+x+ zu^u14y^LT#-(X`2h8mz~|Kqj&4Gcn(@(mPrUf*=6$hkK@n;Ig&C*mQf7!xK=)b?%i z_L!H8VA8(uaC3Wmjc3n^Ey4^ucd#sChr+I?Uk-S~PGc3SUG?i2=}tYdC}s8hOX;NZ zG7@7|%5UdvQD`wl;q3u2_sn+DJL|j+F?iGZ7RRz^mo4Nb{}xV^6fXIfBzt0FM~F1 z+tMYme+EP)A_&o+M~vWsH`qcF9jYo%3NXmptkSYTu+{%YzSvF5@MTHRMNgUVW~U!LvfZ?KGi@+Sz-sJq zhTaqeFxC2=uT=quo5qqjpb%U5`0*6XIjvHS8zI7q)8<|<_7EVz1Sd@2iFI!hQW{js z_jd5^dbX(sS6m7JK7f7q95beCCY%CZ8cmqz z%L(Ym_mHPFk-yx88L&0t?$TboL~h(fwmgBZ9I&6pU?_vs^txwajQHgN0|(|L((%gu zh&cHfO(nLVKfaI(!@@McV(I}IwP@ve{d!pwh*2trt6n(X7cbVpduUq!`L&sag%XS1 zD9DQ3N)wwWm72<}n>VvQVCuv>A@4P9ZQI3^ce3Ma)}#SEiAUCmQ56J5PPZ?R-y%{} zV44gtPcB?*{8rZxDRqh;bJy~KCU!IONp0(`Y};1jMPlcFz~Z9vIsW>Q63*kpCf|jb zI<){r^@v_qgdqiI*|DPyC(|Lqr=hiwZ^!iT@y%VR#tG~&?&$g*^fv7|cyJ5e$AEc-+H+84n3ymq)vI>XgztV(ZqezTdum z`>Lm>7h3^n3p0ums>{5R@UR#4-hYZ;0vOz#3;z53CZ$;IXG@d1U5ks3UT9D6=H*o> zWA-GT=r^o+H&*Yq#~~f3${6wVZTzSerJ(5+`o`vr{b=L9efug$K+G5?R_5Y`3%x7$ z9X_1emoXlaMiy~8f`4;mA5LQPbX(g;&A=1+%1TN`Tc^*NQ+i#jSQYg4cgR$PbZ2_T z03*s{C&kOG@t17+otvxidt(z^LPCoTHFuBZ9=I*}>eK|69V^wny^5gtFvP zUyZNQZE7cv(m|kUP}!=K*Sng(`nA{~?|%ZEc9Q3(?{5-M;V(RCt{jx;`R3sOs%#XO zL=lLcJkY?vE_~yc(t&zVg5Y-y5kWmoh{y~*F(Or_KwuA7fmI4g&*;^ z%#!6_Dke?_jxxVKZMI2FlF9EEr8>X5BOfTC$G`kd42mTN{rZZtQU>8NzhAV>n&33C zM=L%zOV+k_n)If>zdEw`NS9wP21@lc(2@B4)mb^u+DQHSifsRY{Anire|_5U;v=%P zP164p0Vrn4G1qsievGcU`iY#MzyhCk2a9RyHF zHQl6FHP7x^cq7GK+PrCzS5|%s!{e`n%Q;tdPm=b!65heJ;hmw0PRn6w*SdH1CcU0n zNZxH)lwdOGS-y;J!#mZ&9OH+w7wX@YnRIBeRcJXZ-me$}k_YhA!QIxU^>Pxdd-uPQeZ2cHC6 z!CANGl_S}X0OC7nQE#>*H)x3`+r=MU&+JL<0FQfZt!P*7BQ?|Ch6?l0DgVLJGH!}AKVe?Fp482W{|CST3HTFtxsf3AH+EtCFGx^c-o8RJD@ecC&}1VZ8Maft z$eNKJ(fb=D62uk%3vNV@(UCBgo*~((YMQ&`wy9I5&}BU-=#0P(f`WoNWJ&%XXrU;) zkJPG2_4-B5$#w~epG6cbFw=MnNVjLw5j(d+@&EJw>k49UHtyZ`)i zg$W#uQ-Z(W>BPAIb4TTKR!6q$KfbB+ECwlgGB9Zh0u>Tkm-?m@zp}4`Yx98kYhKu{ zzY}qyIJH7KZ{_-!b)dXES|DUReL}!tZ*6ThcWzhHrWEkNN|G)3lj9NY#w-h(h*kN7G;HhH; zM|Ao%6!&w0r$$AZ;OsCoHPuEHHY&M2`$2KxGsHWvV0^?Jro92Fl5G2zh5PH4gqk&X~k zHJoYDx57mm7!}8wpGtdw>5Y2bXcBNY5JfA>sBNyj40F#vS8#Yl%)p*KbAk(^&35L@ zUst2n)dHS?LD6EM~PYSwID`>)a;{T)oYQ$6AtH)-q4gS|nhP*o?dnS6G;@QrRrlybM@-Vs?)}ue^Zd3vVQ+$;cM2>QpCXIVt^({~%D)!LnD5l{-2PR2i3^5-`fy zNj5rQV5Hjb>R@wT&Hp0H8*fWA1Q^#c6OQ?V0F+9u=K`VRco{{xfvx zZRDf^?0&ubTIRoC;k#b-_i+uS|OHk$B`UZK4}Iu=S^?Bu|q; z?|S+dc^p*Mh)t^Zmv(NoOH!cc%GJZ`G%Pp$UKWR6xzwdIe^ktHl-jdO`$+V#)QU~h zg>2fJ+6vfe_Jj>e%-+=XRR!m=S+G^+7iQzZMkZscr6L|ZdNh;sKG#$Z*;2!=lr4-7DbbPkWyXpX zEBYAG_gzV$Jv~?ntn9~icEGf#SWY-U?GrUXpq|&!Oh?+!1Z>IChiKyzkwgDW? zh9^+N^@DcLm=a2J(RdG1`^X$2c{f!qF|&DnJ~6XgVfN`cNk?@%w_W8!5&Or`xfkCq zGY~Y)5+&fx?%hQ3_3LPjT@pTMjLmJ@35(OyKon&xsL=)IV;-ksiGy zAYUupjY$NR1tRa$GdK(+a-Y3y!Qc@Bro?W@AUC844s5Rn^H_+TiAg7;GB z7H=SY`dozpGBfyu>{CeE@l#M1vSa#DyI6UO8sSZDf|AuX6N2tq<^?X z+s~Wl2@ID$wfKTgh)RCVO>)JeeX$AhwwqO|N+P9v{R3_+(uf!6J-@UuCZN+94(-=VR4`#>`ZB|% z)cj?B-F^KjOW`)cHg~Ehjkg{Y=sJ7-<)eUKbH=`1fU?1j1#Ehs9p227Bt)-HSi2tsW;9P5gDj%or(yzgKd~?-$E- z!2xgO->ran4qRjc8$b-%q@)3RCOtuIwFLD)Njl#G*i@_K%=9jv43XVUM)x{Z?d zmUsQV3!0YMi*E=UpC}RmQGU4gODqU*BMKYB>_H`QI5oh2Wsc(#qL zMC!~Bo$FLfLRy#INWLL`{Qh$3j62CsWsP3goeFS#9WJxfEn$1sSo`Gz_e{O~hoel% zcM0!9%YVuQMLq3PA$ewGo5OdKHRUX37AV{1ChJMBc1>6uDgAk%|2g~R@^$hh>t*#P zjFqlyUGi9Q@C&=?3=n_ldgeh_?;XpB$a=XaOqY=UtRz|BnlQ;<%YOM_%W;>#WGUFc zQFPQPv6E}G8m4&oZgOAgZUH`ya;{P$vkB?k>DP)>*jZUsE=wx?wYt;qNl(m@vTYY^ zrQ^86A}+M*&6_tLfW}2AehcL7OZ>@`R;dtr7Zl?(*+%bY7Ix#;cfMkAsTwL6n)PaVEWz8s}1Ro24&{L4QqEvg7 zx;F^Wj|-=|NR(r_$* zpO^7I6$QoKefxe7xw~}BDU#=rwJ=w{^-`o@baNwa_>3+%^cM}jIQSGoFI5fP>4u`L zx1~uxf;GoJo+eF2v<{;>D=nV7>E>q;a3<@X<+f4~1bel7*|N86xaoEY4?9qP*$05E z&0g(0fd?3vR9cYv>6ptA*$lTAce=nqP|2AC;N>K`to+pF-$uI(kTR0)+cU6nx~`m; zRD^Vrv*A+dVbbkPhMMocwRlZ8QogGI<^>m>i*kPM8uLC9gyh1C()RQzs07a;3gL&s zXF#U1F{V#sn8gw8ns;30JQBkY`Ct1->za8C>*(?3;k!`$$P4hiQL&v5%Y{wSHSqhE zH-F-8^^wb7Bdb_`_3G97h6dB1P&1arg;8sm0r)64-bK7XsdmCN^va4zrs~JS)oAtG z&Q+dbFMUuzO+ntNBW%P>-=AJm&&1J=)p#VKF+o>-{bdITtxH(pq#qC)4h+z#ogJKUxItIitg_(Bz!|y|J-JppLtn+qlG% zP|2?1*l&3Wo9PE746p6+@eAZD?f;CAxA^kuQ(2o&Kp9CC^Ji%_n>D^Hp!nkYfq zuNq5qBmeKZuhlWBAu7QEgIuR#o5gCI>h}t_$feGRP@j>>1{PrZFPD|S;F=(SP%$2w zUje}Of}VvJ(6aO?zn$ZhIQZ0?Q4V*0&B4qCt)4h#N*i-?ct_9tS~gD2q8lZ3gqw*( z-Pq_6mI3|{#taP&*@&KRAMM2{%nfMs-#2A{#atNCr0wL&)jE)e64$hA$plK$L2B@Q zW{WTD)YNlgmW2H{-HrzlK3+sTmV5Q8G`3ezWoboe@GoHb`@Q%TeK*H}=7D=r+bJtA zudk~!v9@L~4Pq>GEV-S1@=l1k-dvhwHE$kbpQv)pvNp3)Rfi&mp3n7#X2W7D@nis8sKw8fdw zvpD4mGJf}g!AjonhJhk4p-r3fQaH@O4l{otYb}Du zi8=|@SYRQcx(I6$=wUZp>b8hap}iuQm2e;Q4L{$XiCx^h_2TMh`49*u68t@V`cw$p zKtfx1&h`XF`y6O@qSM~U$f)th57FBtj3v>DYpTbq_QD|wS_c3JV-x~@$EM##utGv3 zva!({_!a2~sE$GxgTloLc;eY)C!GyE`iqbJ_Dyi7XD2$`r7OertX%>O>D}~%3Fe?+ z%K}8wI21#8b0=h^0vcQ0Lhs(d*$5pbs`r&t#Qy9$YM1ENH^4LwcE~sk`xT9|X>jk* z^VK3?g|MT+fr*rSOs(T=AAp>jBB4;p9pE6*Z|$+RDNwu`>M z^I+XhoemA$oSn5gn`VQ`92pW79!k@GbPk5jzcOv*$|1pKAg!H-+V_B`iM#@ z-PadJofCyV1P7>gvWxYVf4kQ`rIb)VIyg93>CS-HB9a`z3&qXQLL-imV7&{vg5c!C zmGR)*K-s4MZ3?_S}K8?lFK%-WxW29FIC6+!A11jndbCctNk5R}{Bx0l>+Cc5c{ma`q!Z`6lYShAel|NdLA^otw5t{T#}@5JRr(ObM4>lP$r zWZ1J2i*O2Y>(;Md#vXkL;XnE9+Z8KelB!3FcDx@fUosLHF0|+Uh@vx!bK{~Oc^S;s z&f5CG(sQ;K$Sj?&T)FZXnqY#s!XMzF$wTK|a)wa*m==@^L?Q{NPfvT2v z&m6{h*F}ooC0biXna_IAN<95zAmL=r&k*|epnw!2ntJL1ha%-+#ya-|sy!j4D;~n! zOGsW(I%DdzY4Zaj{Tfy5n+@d z`M4r`vsjsTacH)>0Lk1VM2r6UkcRkVx?$&i{PgL*kB`p<@%IgQ-emC@SFKtVLM-}) zcKl@+k;i^hEi88F;lC0fP(KdtJZe>#SP+(U^74V8#(U+;0K)V~{0=(qd#_t}gyA#6 z(5FQ^4V{4K>6>uY*I(B3^Yeq;Y=m?q{rJ6gbUKV0H3~Mj5}E=SF!l8yFD~VmMjkzS zl-7xRYy^5m1`Hh<3)z(*)9roPXu%g_TjQIy0p|b3WM;OS8N?^gS=LJ`bGC?fTk1V@ zM;W6@hr2`X3bDb>^b8eVTsc>?bYN>~-l-r2fNJw6hoF#7&EIYo#9{=p0)p&|DdDW@ zqYy;+bDOY&5}y_X9s)6|S+lM|F=D7`*1~hb(?gT^B@5KKZBrpJL7Ho&6x(tsWjXKl z>#N?qyNgX1?ZuQ%RaI5bf{{Q@lI926$^9SEL zeKk$MH&)>>0GGDnetiFob)h@SFBHWAIN^dy&E<+fh7%#i$lE3_Q{$$y7Hf-!9Y~v- z%U8YJ_WS-K#tRyF^)`s1jXa1)u$fEH{K5)$|447dd8{RH{?*TXK>^|t%t{tc-**s~ zx@!1vkXP4*%(Ud!+7;a&2U`u#?F5w| zl%e54F*M>DOM4O#F(Ju7?1x1Yy=0rw4it7iAhzh!q?d zYVv|k`Fr_=tu>wnGZ#|7P)&ogg?8oZ3(iHG0CK8??SsK?rMd+YNSm0VKf!8 z1Qnr)yY#HcHiZPBTE@o4m1H}*Uc`N7R)xBXp?4=za0QN{DK&&%d5~{LQPLOv?0|iN6l?|C5{9MFz=V(>_CE%O=uHVQ4t{)9W#u@~Gj^>UIfTrR zxfh?KR~3zC(!I~va-q}%f_hyv&T%zX)@s{?@+MRF&<@-Yte5xBU4mJR3zFZp(oL& zM0`f%2i8a^sYGr#gouZ-oojQ`I#R(}=0XgyyHJs0iKUYeJFkPgNDNh4JXv1Bsn`FPfvoI=5T_sMcPM%>X#rB zvZBOToma139Xstv4j|Y#xFJUcI`-`4@tEqND!X)$Yyy(f#!W84@WtF}VUMwA(O}kHBLk?|V+W^>H}0z_%Y^ z`!s+!j6gbp$T%0Ns2!y%`i>Z}m)+Aa9()X7t)iyJ4D&=Uj%!n+o5T87eSq=6+XpDo zgepYcMYzglm;sFwey(6bgWxZsgIu(B=!GH4tMbt#IL!JOIej*l-x72x#z%%eEON|Lzf{ zXK8EuhX{kYu_^+s8lk91%liTf`T}>!KomnC(Z^r5ij##l_YF9xi=cSWc_(+m{aeEl znD2LxIJ+Z4Wy0;)}ZESf0#p<6d|qADfMB;CNc!zO#un+q$Sbj0Y9|L%e{%^VxRYq3osi->>&T4l{4sHK&E z%geC`RZ#7ul-^lqXES&K)J6jCG%%6Ov8PIw#A9_jb~ZcY$Kr(nGy#BUj=V}H|pIlc!GGo(WD^@w2#!5 zrRjhqVb`iYe3(v(M*1u?Pbj8|sS!M$;3E%~U0

E^zRwL-kS%19T&0i) z(?MH3xC~-rYHF%w9PUw+f@jU1-7r<4eYn~4w9Xv$X_(B~aAwvTR?hycOlehmr*ng+f7dS&;PMme+$h(>W2 zrRb@80#uPw3w=!bZ7Z3(a8)#6C@55?kh-2))&rj2KjFMzGzqvNL{5P=(LKZ?5-mXG z<>AC9I>aZhJzm}0LSBW7zrmgpXW~Q*prWc7L6Q0n`e#8^c~A};KU)**Fb|2IC?TOn zOtmQ`Wt1pLS*cta|DJkm4^~v1w0w`WTRr9&|!sxK?HU^up-=zA(^Ak+Kob&zLjkmM{`@ zFA~7Rpx0jg)KKun91@)nnX^w@vxRN5i@Ih7aWfyBj(2aC!}N%@@&tPyBVSkp6OHB%r6y$Sp43 z#~~Fk^y*acKYh6*lKnYx#cB8cV8EKNY=8!%QI)4*JBBa`5jaiVbro^vCk^l}ygtjxiD`5JzzdeeH%{9xd{r7RD_@MVQJV5d{}3 zRnh=&!U@q!O{VcYS8yFjz-de;;fapol7)HUnmmPZ!f^jV2FC3PoqKj5y45?lJte#a z!HDQi7Y(@d(_qG9=_(|NrQyc;?Io{wxACLTTIji*K@nnpY-jH_mi{vW8`aqDZ0?sl zrmaKmQ~jAor^Qhv(5Iopj4AY?p?Oq#UZF%wVzD{fgzS_aGbb1iLVbh%t{jeE5mK%o zNn#8}H*%Qqv@8PrYjBViUwvNhdy;;-9ay0mR#xNq%_GAXNYPT!3rJQByYZV8S6o?D zl~Ygz&LPZ9bpQJT!4N360VRO=fjEKYQNIhnSK|mkdxef%?j;v|_+OT+FEV2az^rsZ zTlK?nLznfvNnx1WNPUL28qf8&R>JdKQf0WhjcJ^)#o8n2|MulP5*!yMFSilPwA)Hm zhPSgP(=>^A2zZT&>=3TFp^VZaq+S!`&QmqPDM>6vaT1)PvhBA@VA^`w8v)uiPZl(V1Q=MM z-{{d@E;}%DSmYcuoYyTUwH|7+ZjT{9^ z<{$WIzm^}*NYp<8nzIT+KaFVGkv;!*>!f@)$PYR{G5~&E(8nV%rlbU1KJ8BW@1)jX3G!b=;&y2-NiMx4+s@RsA>X$z>-jD7J3+>w-6n6#gih=xVphFcIAACH;fkjCHLbzd`ENE+8| zqpb7s$Fr8B^a4_IuKWD?x)vkO2!5ILHfcOH0AV4CZ8TKP7Q8P@!}!%>2|l8=f?Nka zLCEW}IaFZ{h>0b%M3FmFRIBnE|355+?MLB)+~$-$dsbZBK`ZP5UI+W&VddslFYjn< zKDAw4m*EI)1bfRjb{O97(d25KuHIdOB-Xev?0b_C>BZMp-of>8ZxgJoiDbd6YK$36 zNEAo+9i5S_m6Zj+U~T?V!)cntnTY=34) zCKVATos5Ye$o`{&_2YPOqb57zo5k8>f~j+;?2Twa{G^Pk@IR%5R6^pk(iM8bPSnRp z1!zaMFFL$`?W=n=*HJc6&;AP=)H~6 zKnqzV|1_l>gohlvx*m)*lXEs10t+Eo?UBr^d-|mkVNDQo)Qv4Sb z^ORWFXb{0Tjxj~HincCxfBJXpKS{pwbi9O4#l+%hDr0SiFk@8SX%rCS!K@Yk&+bv-a=e8s}1PMBzb42uRC2ttDDf?jhXGLg8-0%m+kD zonb9Ly2`vY8O>$3YkN&XQ`g~vf0yC+F4SVguI}8q^H**C?sP#cOBx{Q;XlZH=M~s0 z%mwTfsKdGJjJJGc8#%;$XIDo#zB#qsW`9o;8}-D2k+QCnbi42tM+U}%O%&RSss<{f zLSj%u$d+Mqzll)Wj&_$o6=SpCUn2X-zn^*DJS=+ISuz_|g*tNu5#t+)R>c<@rMxQI zaIr=0>ZIGaWUxGrrn+4Oct<$=oOi{X6Y(-YD7c-*Hg>-lN+-}uNEVw@-B`VoQu%Oa zET;YXx?55i{>~vCGx?@{r7I@NSMGl-;f1*9wP1Oi3=k&YE6n4?g|QeRLYDL zwu!j1b&h@hVYTRoF7j7anN&PO@jf^y^gmQO08Hwpcg&hte6QUJ8H--5N~M37{>SQX zOSbWE6>@L~^^Id9FOHg3Lsh~Ih?+CEB&oFj;K8RqH#8WuRxS2uB@yo5wnCc_^tM`K*R$=; z7I`0#jPsvuqHn&9@x*!MWo6Fsq$CtJga`s@qNpwq=Kxq!`JeHU3?+XJA9;!113gMO@U`7=Jrn;Vhnm~@`F60QHKJzP8MFNT?F_4L-8gRh& zAq1cz*VMpc5pvw+kU*p=FF}~KS48AF6yC!kBy><+QbK?-9=(@6k;cJK=pPuVE_rkF z&Yo6@M4o>+mldE9J1gZ->y&Hok_)kiR+yF@UR)oBoBvSG_O$dk7E1)vjvs*M^blfn z*j$V+b5Ar}6di z^5ep8F<*T^1=SH$5CIpF0j9$0%p+E6GKi`J2!)1@Z6Ods9;$J9Onr7@kmUd`$_gQ60C}>$+1UbM;6ksPEF(Z^Ip*i*sY=aIC2$oOJ@6(%m<%YuZ}8;a zet^%Z!q8A(4-n+er||AFz(XL62H*ua?$HoeLJUZKF*L(JWj;M!3PsScVO@hjx=sqb zCSa#r0pt@3dH2B54Gs;x2VbXZXn+N14tRY9!og91o`<1|ii&wc$e?To7#tgIAeCyOtDqy28Z)gY} z1xc-c{508mjuay>oM_nBB%*2gi?0sWGdMB?(8O>S(xzhr>mEOTi~t)csOYul_%#6Z zVF5$Dx(vBc*pDHT&Ik?zXf!RMAHddMR}s0=Y8 z#O<)4AVMZVlQ_6n2oIpluE~Eg9YV%BYCr>b4z!^cK%P&aA_vr<7$`&lrUMxR1>OnZ zEBVI`-fn}>A%nqQ|2qrTk|bsfT?7LL(Hk@{CGOI7Y#AffNO7e#aB2;A^3s&{Z~B zIybsq1{58X`KUK;Y;0VC|FZPAsitK=WOIC|3Sb8hE-Wvn?^R0x{Ne$iD^S~KVQ+8G zaFxwv@T&<8vwpaTY%mu^4=)ZkN1)xH30(YOP2a(>2f>Mm?{UMUuQE^XwFC|z^hK_q z0&IYhVj?&Es8I#ATo`)%f;pZf))O9cY9AR5*C9grVOH zgqoBGnQU>e^_q}%1+Xd&9`fAW9K5&V8&nLb@6@gMocN3SN*MYGrZHNUSKAzzl!5|! z3XS^hqM}_eg1!r2K^!)rUtJypJ^fc2>7N^U-8+(aNC&84LABWN{K=F5ya&hK9y-Ki7)A2RuA!d=*3vt`IhmfJCC*S_L+B z5&)4rXn`*0@(5t~yAUKZVnJOYPX}yeH|$$T7m$ngKL*A`H~dp!d^G@#TQXpBMjUwQ zQ(zT`j+dGkxW|0G$M5hc@J*X91ig9Q-rmn$9s%FrGj#~SuS$vq1sW$XpSu9Xjy#su z(Rrgn&BZ|+UFQ-C-r`LuNl8~#p{%gor-3ap(4LKD=7CR!8dVg)5$y68AP6|)?4S;4 zgF2GBni|{3{t&a|eTJq&N_8L^r9_>gCCDBGx=hx)cc;S?xo&N3HDf0MF`-inn8uIT z0VBQoxVic8XDGc?aaeoP{z?J{0hqVrkR9x5=nhB}nYRKmapEpGPY@mL53gH>lr3}* zJsAUn-MQ6Z<3Ha0>#v0&JFsO_)N6bKQir4my?o$obFCMoezCBh#3H5;XXCO8-VW)z z*PWmW08khrxS!CiGVKpcl)|%~zOdea7>d8&Bhk~@gVtxPba7YQJM?@gP+2GgHh>+j zSE1n9LQWG%C7h6B1eyulAqBeEfvg|o7-P~vY8?!&dN7^_92-wqD`3}P^-e>b60jzK zL-EBM)Ya1ffCd}r(iMFRf-nXHUDu|1e&B+a3oL(8Ew2N7Kp--StsHh1El_I$3~ zmb*&FYyFwSLwR#Jju)@KIYYxqV=ytx&~ZXX!9!{6R^%D@XYNe~I-dIu^ncT&RPwVP zW9Ylz!gz#Md3OMr&$dxKud$E*Nx7RCp0+0`_CCEm?OUsQ4UAxF>3qyS`UaAMG)8dA zqSFOF^tYRuhZ5ejbFwFJIf<5&20Nie_csl4qJU=3yffMZdDXPjvgdS9LF7uj+ypC( z_5A&C?jyA3mEmfbzak~|S%UwH?241Qnsf0MYZKvHdK}w6o@q{~9UTuXmZtFxf1D9qD%#;sgtjSdlox_m9J`WVLc~Y`k zzV@XDV!Nai5OBV{Y;*Sl$ilQiQ_YU;&d(}Pt^{`BA}E9T`TKvV4Z1vXw&Ko*LE)f{ zYdO*VfK;wP%o0`;MG>sfGXmfG_aUZ?_?f6@auk&-A7zToY+J6XPKS4jA0)jk$SYmz zcRuq%diIbmDV~$}1<0|VB@a7vJ-GYg?$Qu9F7O z&fW#~vH?h^WJG;>TtNxFySq`50Vy2*yWi&))6W1K90{qTp$uUDAwxq$2f(FB2TmMs zo_owCMet!4ChKED$u*Ee;e(A{P99zyFYu%X9s=duY$2~!F4UnnR-Ow%?n0N^wX(FT(*>q-YzH`<)N5>c9fb`N}Ak08T zXELOrKyg?JSR0_$K-Ijjs~ZKQ4Ey4tbgIOsoMS@|@Bsb6Lx6m9$>Ybb!Nh_}UDOot z(*PM6gi%n-T^c_T&J^yladG`{r1mPI9w@~;9U|k4z2h(5fKKZPSL^_?9w!8!u+0S! zf_kW*T!tBqA^`Vje`$C7hui1M7k-F~e`}NmPmOwD|5m2faeclEJpxj9KS8Rk4u_^* zBkT7GmU4thQPWAY4GyR*^R>0L1VB>np;_*oJZc;ZePVs?22wx#@8#gb zs7*u<$u*v1m-c%QyMCBt51yHU@v2aG&D}dzAB@C+#_AFH0!+?j3Q&@q=VZi7#j1bb z6xWaOJ=&S?7yPCk1AaaGs@B;7CR7b^-DCrl`5|k+SYl&rtY}V+3=ud1Z-b%# zI8=Mxggzk<$?%QyhlGZL0CfUz*T21!(mMs=)CI0xXn3&leTkcE{6+5Xhi}|J?uFjv zb%2VCXVJ5>^Y-7C%=Bh<(;YfWIVOu5p=$-y(}DuhTKMpx3Uxvv&uR-O!a@x~vHOo@ zhb>{Pu$m7ae4+NZ;VsZv0Ez60fxnE+%>@99d}|g40OD#u8f|kxg(+x`T+J*+l7s4uO9;|*YXyYF7%6;etfaw2mJ9v=Hbx> zcbHE&BUrq~w@_xLtfnRmttftg0BOm+d*D?T+1c2X!RiO#ZM6j)8yG$HZS9843>4Vd z+u6N8_2$hR5QT&J_$LN}Z|FDy7RqzWxij(+ocPY2J4JTVxp3YB=oRq;vLWVT}ua|Ddv}VFb{u;)kWynlfa{I(`7DdJRZr zP2jJ62jxghXxjlz8opCIf`DMQUpG_?SiFBTssWXR!KlL zdVC+w(S8}yPhA!>CM(-s{hF zp+0%*4&cPQIctC-6!lUYFT96o(i+zV2ib|Lyzv=8?U09;f#?<6>l*%M)Au_SuzUgK zx?Nfz-88gDWtE+P{r3|R{3Su3yHwSfU0@|`{yclPxw$zRRZxSjoH7{F$*ohZ!Gw>HVCn2AD=Qm4Y<};gBK4b= zx?kThu090Mvhmk_g7GN`xC^;Y-Cy@P_3Bm2%px#HB37VF?S;C_um^QvfmrnR4)jL9 z@Ll%QgLm(!H`4`CfWg*d*qV1inqXaF6$F%;xv1Ckf{o_o8UPkeQcw-I{svy`wWb%a z#s3&&1l@O)Eue}#nF6G0P8qtAm}~)KXrSiTDdF1(zZb%8+Hx1t3oRx3E09ADn*%no zO-s<*F7ECg7A~%#tAG7#n}g(WQe?GOlMTB~f#kL6YAhzNQ_(f7WvZ zsnxhA$~Ln%^J=3sf)|L&&HgVrE%TBs# zEKW7#k4a2jBG&c3jCe%7u_cv1dqnrGOganTy&43gSYKof2-N>~C|IbB z;#Pir>|e|H|MFY^h_&3muV98#=8*oX0c33cJKP=+Nv=B*RIV*tv+B<^2RsV11AmXz z&F;^pk>Q!2nKB02Fyi8oc7HZNg;89614q)Y+pwGbecsY;zdrN1tTpQ2N*#V#@BWK| z={CW12fb0I9yC=Z#<8!GI;YbgIVKH6>~Tijsj|um`*XQpV|Y=8V>;F=jHR_Sj#q6e z42KO?^8;Qs0QZ6vsvW1=d5%r>_fn)jHpxkWYy}U=W}-}l++eAtaeszG z!g%Z6pPAR4IUe4k$f_>L@2DoHr>bux4zo%zP?G9H=Mx^<3?=_=4)FB$ zp6%1~keinfUs&4otr^%9`1oEptAIRP$@gd3V^cd21y~7&j8yf=Ny5eH^zE{lk#*MBLT(MGH)xCE;6y(3`jN)u4Miam0 zrp)x$Yu~lP9ngdD<9~5RRX6$iSfc!@shU3qB{BYP+B(ToAluaT(pi5+l!T)3szZVW zR;O<0n9ii<9@A3m)mLdL4J>()fs+wx{tbKYQ15-A(pmz3EB)L?bFSe?$JO!tl2ztt z28#xyQ3Kl>N67^>n=Bi$rO(e~DZABvFSfx*@`}BOpgd_V8}}KX5^<6`uP8fz>Tesz zjj-aMaQ@_Ib3fxp4`#_Kyulm!cyWFU~~)a1>$RZRM4 zvV`M}pArA6*%=ZUg$t zr6Oz78Ax=AjR`ZtD#@~`l;OCaZ8Pia*u9!W1~wCsFmJ{goo?=B#XX3zEZ6DezH2Yj zq{h--Vk;1rxz3RhyuY8!$zM+<=yrL)eTg!TpuU0pY^>jqrgrAfJ{)?(HsRF*_pt%v z8x7ne!K|OmT8if5SR0ssrj_I;5ETX6$-JUHwbLavf_eB5h|~0Caj{GEnW;o zeYw)*-W;QSdop`01A2K{5pvUOkF)y32*#-T0KuXjImg64i^B}vDYx>|d#B?~C>o#i ztMfjnM94K04=OL@U|5AoQUVFvrHNW(1mpHk&wYG^g6Wdga4cWZ;5LS~ekcT$QtO!c z9mC(A+sRQs^jLa6;}ziyMX?-z-mj;#)$XPbq2U5=p%x6psu;OVK*NMYcV&0NKf>j! zgGGcomG2tL+$4MsR%^5NpdBPWhZ7zyn&sf792&e&5o&9oaqlbp{qb`oU1V&ID)o%| z@h{-Xr;4CSCP;}!X8KQkc@2WpyKWE^>lgFZXtRcos75m$MpZJ44N3pL*=FK*_2<_v z3f;tiroP+8cWr*JMxnBR&iTnWz1~T`9#Ppr9f2$Bip+}yZ^~Z#5{}r+?!mpQJvd5` zGC})U&mqdW)!vq|dlg415))n`qQ+{s6=}UmN9QswIi`zw@>x1G?>#}VUftZ0@6GZv zWNh{jY{LkjSwL2r72t>m3gZ}7^|hL%&-J}s2t_yWhI=WRdU06xAB(12$GnZVG((0h z9M#T#sxE94EZ-IPb}&+5^>j3Iy)m;4X%@4;ULba!OxjT$Fc912@FKFg{Hnd&R1lR+xky z^i~ogYdL0i4+Z+Dv~tA8(N4$z9F%_61Dn7IhMwP}eSTXe#G~T@tvWMn!+$P9zb6S< zGq|s_BGJKXKg(*5*;sK+;zNuWz3KvsPkoO!x`9i~urqx>g?FpC=ZCuW0RLD#-T7Ab ziu?I-KTYH5#B8kloUMy{#8y5k%NoQx+p+ashG*`M;j$YZ9e4pBE+BCr9ZK|N>cWDJ z;Dg)a#S63W%zIpRILbX!ANXpg({V1a-LPfre+55ox1%mLbc0Zk*_{W{F6s2>2A-b^ z{<*)`8}bna#llO`^{o;})WaR*jbK}hAnPXC^P)-U`LtUC$4?)#9~=y09eU?VOwiF62sZ zS-N1%Mg9dBpY4U0jtPN2+r-ZeeS~c$-Av~OHmw_3eKf26YY}^r(sd{&kUS_Csjj%s z#jdB+V$y{hM6%SyytTUpBU&CLQ7+1!Y!r4MR2z@~sLuGvtxb}G?n*M5P7h3(>ZGv3 z5BI#aweJf_nb#xwHc2~YMvc|3dPeh|yRkM9-gx}Ax-wcgd)srLDnWB6ujm88sm6%8 zJHxcI9{DDMepf`SKRmep*G{D?Q4OO)hNd`{dSV;>Htk)cCTC8&x9yzb?q4}tNq9O z2$=@w*Gwt%YF3p?{q{{V&WQt&{zC_S60FMtSbR%aScCazHYbw4ldnGbsP6Vz=EkI_ zqHf%|Mi;8JKyCi(5v&dSkJp7ycr-syam3?q;uUTnEBl@fiw=mts5Kb~xUtbBVlht8 z*J|KOu>PzT{@{U+W1KwWOx6(TV9E=7S^Yf3`Ch)v{)ob?`}y#uBY8R-B&S#LH@(Up zToWLCy;I|*q?_0mf7_}oK%BuVgFi@(&#MJ@=J3t3m+{SBtPG{2C68EUQI+hjemL_G zb*qX8(l>4#@4@&Sr?@t0Ss_dh2`4f~9Qj;j!n{p^;}OcD!{e$d}1{I>Y`&y@3g^3KN>ZK?bBQo+sAM8U`tTA~iBBG*OcOd|GJU{(CdsSjS~FFo!&H6YHL zok^E*BMWD1t1))qnDNv2wy`Y#%&_2k@)!1@w8}vSm!!LdvjM&GAqyylRb)OMZ;2D| zDQI}Q?p*A}((K8j`s>$iQc5fLV)0DL%Pe^xB5LkyB{sk@<@|7Rsc1n1GxgV4mPN2+ z3@2i<>%L!)ICa7NH%300f_vK`ghlE31R)*v!Ox!i!o2ox;dOlwWKmnKG35?}*G|vO zS|rYNJ)mu+!)+)tr{@v)bHTB zK9-!)*t=ek!{HM&gl6>bv7@Nh_1@>G?VK(NP-R#-D?Oj&l;WNx5hOWP*7H$Yz6lROQ6sgk zjLG5oc+O(Cfjur6_lIMy453cx^@#h_B#-svQdrr-gM}LeC6$RR6D154c%u57 zkrrX$8!Cj_N(cLdha`={s?zyhJ7NwgQ29{xfEHPottWCv%1Ni&jEwLZBF1`VruS0z z&Br^*4qWyNUkVs-^09#r=_4(B+!wrTH8$EB9R>!_8V{xj&VB<-E?KM_8~!dC*x|rI zzymPZLzc3?z?sXIzcL72_Yr!ybBe6nT01P-sj)+5ib6ky9f*gH4X{mku1-WGqT08e#g1gZUeQ! zQa)Ii>~3wFt-^~X%&&+q8fl&jq-fUQFl;?`0+}m~b_4Rtc!p?(YDESKtBgC8@Mr6N zVFb;>xKotN9Sm~QNry4)D$|Jy3%V&&7v|Cg{rW;FtW93p{*65J44O@v4SQ$D6Mv>{ z6Ax25B+6LbdxHok3lp8OdQ%?Uqy01?czc)n?c=y(JE-5t&Wm3V)p+=u_@nBAc9BD( zJ3-NO)z%qjBM`nNEGHJ$DVlA$YrX6gtr}By%bxcU2-6a93J!gmvf&5U59JbXc&ep{ z1({UD!aLk%%iD@{AaDd2Ntk-=RfYTDIlpAwp3Y7=5Oc+H!i)Q08A}fF=a9{C+ommA z1~umvjdULpo$ykfPe0^0;L^e0QYwri&k7_M4&-2Y+AydJPg#Gm@-Ba!QAvg2sk|c5 zJ~NVch&M_eH>=2_;+P%V?=bd$V8!0Pc((Waz#>`9&y0Q0Tgbmr;MKx@A9@y~>oC~0 zR=T7Ih9^!CL_H$t^q19^v87+>A15l%|yh~D!v*}X)Rf8kCm z4Wd!xZSPOxlzF>8=KOdqs^v7kYx~QM2tVHcrWN7fT|IA;k`c^O?tNg&TXI);ee8oi zd#k{rhfp@aX>2288(D1eVXQb*3)F-abG7a@A}I$tC<^k+v@==&qBtOM#je%ngk65} zbu5a(o1f21cb6_UIg~;ZQ(8g9?l1RH3iD$AG~gW}XX+st6EE+>?;#YnusHn=4=%ep z4{yn|M*3TnsX( zU-p<0yQ&A*C!CJo`ad0X6~%~c@1#XluKEGxW2Ra+@p*o|UjWjsg`L4$dC)^(d9w(9#CyEb=;-IlUVY_a&_iNDVCcLRxwFt3 z7s?l~(d@YYAXpS_J&wU8<1h6a@<#D=?gJh=lMBZkGXv++uZb_br|<*}LMrgz1GS%r zbXj)b2o{}Xq{R|2c-KqFlxW<+D2Hy~7Xw(Z!xo1T3-_v`dSky1iNA$D5BWx0q0H?l z;w-$=>6gI6^45X)kEhJIe}pU=(0{kg>Cfr=xs=&jh(|jNQRWh4X6>Zf=?u#zPc3@_ zLnaTG!cba%$|ZqAzg&!_U#R*nA$l$|h(Me^$dSukPNYMna7`!Wwu+r7Jq|-jAm_Dm z$0myX8B40V`Ht2U@lGXp#v4_A&J7Gd;~A*-{MTW2%_dmS2}1?n&Y?dI8L2Z${`nD? z*)0vnWHFD-)YP&U=xD?_Qag1JzmDva38cK~D+*pU@WY`a(L&B-X65uhmsC$@$MH9c z+#SSFu|m4jx)01*r$_M6ptDAUpp-D%2T*29sB3R;@_?|QnbVH499r5 z#=mFd78?+1c=bPD<+?oP7)_@|a&KjN1dwaDbYaS853OA;p8p=zhGzm+lM=*wf=pJUv!*@5(>kYpC-xx}mDK^^ik0{de zy>JT(?mY5?S&msI6d9D%pASJIckmJ*zMg`CePUiPNBlwR^GQM8ip_+(!L-ZNYxOJM z`xmG-)XBzuCmmES=~-*#;q+jg8Oyc|Db?REo=tC8X)5P$ol~L?NcM_#hDfW(&rF|x z5y9+3za9p;rm=O;MGPyC;@*tLJ=Kk(&H!Qocc9u3k?bXV&{N2_1bM=nIp)f2!!Mr? z^%8(gKBYg&gSlU4E3KBueVOMO?KT&Z-_cW8_gKfdR0SEj!W>6ni``8#E! zz}a^`X#4wOq37uFeXcoQb#ewA<1`|;6o|-<#K~XP2!<4Q4Ilv;GI<*LkFmRs#c56y zv%tT^hA~SvdCN_Csn{30QaSR!KmGsLKlN?hch~RubYS=L|J1$uX2%ZBf92h|w(@e<(N({t z%SWDS-Vcol9G^JnpI&QmVe0H^RB+?>Ix%PTd(p4>q`a8VUyr^ly8LxgV+~fA@YBWh zE$AWqiAqe$`8hqL8#YfBN8f_P|KC%>zqmSX!M*e%^LStjR^QUTbYWf-=-C$3=H{~< z)QquR=$p`2c2Nn?&8YD*2jW~jR5UE1%$Dvp#!+jNz5ET1;cFhchHRc*2E!_tNB_in0IVW|Ox-w~9E&Q|C zJ@2#$eN`wTOdrk(x$Fw<9yzDK7cAdS{w}2&^-p4j!tP-1;?K=`o@WIu#z&d<2lzm; z95fK>cNuh(MR%W4J2;H!eH+h_KX%A9j?tRCk3(GH&%<=AD6#x;%S^MPc?3_mYfD*1ZYER`*S!!{s$Zm->a`U6{2M;g~T4#yc4&ic>^-fK4!W8C2xj7l;DrM3EzI ze_lu{dxgBHaHIOm%3E^fRsJN(G{kCiLN} zZjfSN_GJ8v|Idq2bA4p8DNHPQ$Gx3X+s*EL_|Me~kEOFZw>P_!OF;j6`kQHu+qH)F z@;_XF&bH6M!B|CKcwA`2pwlIvZbRMd{>0~xKtxI7LqJXF0VcuRi^=#;K@Lr)G zm)~J{)jW0G1|6>A*z>QvD48Gp%E8h)y5*1*xD@Z?V-&1bj#n?JSLjMPL{Sq~$cd*M zIOwUpn;JzzeISaWt}w*v9bZb*bS|g8i{qV>-MQPYyLtZvGBNf5j%0DH*LoNGZVaJg zEwR%%QNnopYD?OeKeL+{nmt`z*6j28?@#xpN{H-DD5VM1B!SDZnC4N1hFw zer6*2x%Ul)scmN0AFHaHt&8?(d}ftt5c}?Yq1!Zb=lNI!>6ALRPcSzVT zkPxFf??$1m>qWmK2^}nplkN}>C!& zOnHN0c=Ll;IiS+YM@;O_YauvOP~WDF|J@VdIfagDHTyj4yZ@^2P58!Fp(wF^0nIV3 z0z(oxW+a}DD?Urvon+=r$Xz77m3SSqj4}OiDw?leoS@*CMqVQ2)bm^RAK5{kcv{o1 zlOK1gxn9#Ho_Jbx;Cz1C`=$6F!J?*wjQo_hrGDSP?fd2>{>kjYB~6TIcMoeH2e4=c zKqe)!MA)81xQt^ZNs9kl{%JAvn(-IbD$pg7x{BC>| z7PSsL?BBq*Hl?6o^ML&2Nm=4wS$Srnk~czbK8(pA9k=z&-5im~L7AiXzgE)d;2$bZ zyi2T(KuY?Yz-#QVC6MYPxase_x0bXq{&V6W`7=MDswsur{D|};f*M6)rt`7(7j((H z+V@sT+~%s59GBjWmB^-Oe>s8{XLf#mDq8wY)>(mo949?x$XB-igER2ztiwe@{`w zF=~YFjHo)HIBFZSdX3q4`hqJ7IYG3G7Q7fTl`IvkB0sc$TCiSvB^xV{bj`0 zExnS}X_eCLvDeg7b_P=W7zyzI@&S8q**3x@SzG&@ z@o5pGarz7lkAerEx$22nmrQ@{jB1S0nc4_}SHugbOj-o#rTOl~e9+r{HOJmX;iWL62U}Pl zOGoPc2*I`Hx-Q_-)QW(hq{z{8=c=LhNVR0Vd6LuHs_(#RG6FCyGI!D71>p+Dupvjb zo2TN==Euy$s z=Ie+ofh+_@K=N6S^06N2Td1wReX>1=rQegoLV7JC=tBikit^(Er^O`3lMm&`y+%t% z3Af=T-;u)D%(!gG+HNM+_q4-$)iaG_c`@uSN{eN4Tor}ZNh}ATo>LTqA{SxDkgX2NPfyJS&f(M?W4hlL!$SAz&`D#o~|b{ z-&vf8*@;o~lqF`GIh8sqUm+H0wp`Xn4DaXooMT z^Dv*lSLY`sK2W_Ir{zII&G4&`0%yCI%2v}Z?N4iZE;_C}KR%QzZ3l2#hW00%_fl54XLz*88s(7r za21(-qca5Qp( zyD%+shvOxNNeX{iAU?+<{g|b%-f99WCQcfaL)*GnVZk9{@1{EU@d!2>MMe$Vk$1US zgYN~>&tMg9da+p!H2U?1bojnxH|3Ybt;5)!_qW(Cr@tWZpciztvvt*lqVUy@iCT8c z+4aTDoZgxoJq8h+zQmUiENTRtUk^`&sEtoDQE4?Tje_9wIAGaJ@EcHmcH`;O3F(oi zsPvXHcC|7-sbW3dtV691<*f2Jv#UeyB;_!iX9Q;?;d+GqL8-$G-eV#eitMhWdWxDY zf#a9~VstRRmCpt`xM7x#Z7XlQ3~MQ9BYlsY);ox8S94xd4*Plqp^|LIkw3E?PG3-BBw#`Yw2$?p zP-%!p?rm>rr}KaN|A|NYG{vBx?)9w`yQ{O0Z_lhGJy$XH-G6ghdx)saidmmVoe%~@#i8=#5ilv-Gu7ThioL?|-F_Ow_v#k?WKpb|~g8DkB&E@L%z z?aqAeOBqjR#cHKLA>N?`q2f64BhsP0-V|*&eU2#N_EJg$$!urO^#&PJeArxC*sAZ3 zU1S1TUA~9+fGvSJTFkqV)$QTg3)4u~hQ0I!@;TnYZOj3xLAu&N zPB&ApROCmn#C!rL@g+s#)6hr1o;R(c=atTQ>?y(xk?GPa2fbUFB`~PNosR+|nC=c( zJn@X3OSyxk5j7N^&#)QGt|#)<7^q|qK2qPi=qWyv8#;uFMs1fMFJlk1;SWYLh&&+# z&Si6{bK4}z{R7JH;QGJF0Dkpd@9&j^UM`6#R)#@yao2;TcQf^zyR}F4*<-FKfZLhX z%FXYA`sUSa?n;)MOMEp>pNBWh9CV{(Y_9aYATbt7U6lD`s?|oku$pOx6s6r{dtODcD7;Fv$jfnG ziTD3=18V}GBiAQgoi0B1*GhNkGVN-^syLk&L0rQr)jasPOxE2PSf}XTEEYK*A?R+% z!RT?|!mxQOs~(P}$?IT|fz0wak4aMr`enR*h-0^bI{l@&^r%LL+6Z1%!1 zUqYLu>dJoGD#>2<(Z{xhI|7M=;MbInoH1o zvYPcKWb612UA;H^@|aCISErSsmRnye-uxJU^EBw|T*f2BS{YBg)^#H5MbOCknb78W zNG5x_PdIi9We$n7vaXNEb2JfD4k#D9ojF0&d}bzdb3wr|)oFwLK@)r~#CbG|L%zHL}V18MPlX6rKsV z^1qjxqo!^J^FmI0q3C1%4{T|zch)iF7gg628zy}ZBaWZB-mv!vX742*lewHy^M?&A zc9oAle)CR!78J52Uhd851Gxu@`V%F?A0E@|8OwzY z=(F`(p%R=5>QgyxX;+3(PnYI7Bd#BcObXKHdhgmgC4Zb|{A?zsolrq$ zi%D{S{p}zsa;ZdS>_C7;HHpp1E|QsbOhl_>^xLBarFYgjSb;=Zfdrnb4?T0dZ<((* zvSd}Zd<`>k=t%G0f8Vy>NSzmF<&B$B6ciCiDj**y8jR4+sK~XORm$mbRtjpFZniJwd>&Qk16XoLi z7W)irW#2Jnui|GJQ$dF`t+M7T?anri&Wy)Pzw3OIC_xrvGtz-CIoZxCS2sAFkyFyR zb&&0<2AA1V4HwXq%gk9DP47vz(&8GW=CiyT*fK~5Vp7kHd9Hi~&c4e9-9eVwf1I9; zpXxi1Y3x4nx~tAhYL^cPo$!_KX`Y+Q2cY#ml`%z@G=xSO{<>j{ERfm?COA9haBTY1roYT zPvf?FucHYFfgICYd@Bw)+T2cKB_|8yY{1PKc~Taqk)LtYF?}T9HV1F#BcTa^6rzQj zc{A*!#bFJVsk3|@NSkKAj1h`jkS$DT573bTvC2tL#+Oiy267pjl?LtS>r->5>d_BT zYg6?Kfw-s!mWvqShZs==`IK<;6a=0s5Qx6{7Mgo1`7bp!MzOx*>a5?3)xLSFbs&Cb zv08okAY|o(b8Cr_ycJesdsby8)ZUm&1zp0RaiGbmyH2Tktmq9Z_povLSKc>2s_yjjhnocXxs zO1ixCa|33{XwH`6zfVd;N)S|Lhd!y(o6KbhlTbIGqLTlTTi13Qs#E9aHtGHLo}_#0 zYksmyeu}k#?~mkS4mron(%N%-EDcmX4TVN}84;?Kb9CorPkio!#wjJrt2tbH1Jfp=feXJ9&h+Vd`5a52H7I zAWkGd^O4fst!?YR=%!Mx%NXZ-DAT9U4BOd9%Jiy8Y@%ignf?uU;Xc2lJjd#{3 zr#K&}1~}Y#k6~hW%=#+5X@szssWx<<@Mh=pX0|UCLXb;GO`pp4NY5Jlhx#=jWRT7s z7hl}_u_aJdQ0pzad*C9kb#F;o>9evZ z4VBrSNhx8rz`UGN+jDvRiWZsXCVEzQvpFj_U*RHKRcILBr=D1k;+AHym~639PECyP zHbH5}(3i|6Zo z%`xR6wOaAesl(xeWD{Tu6y2EYmJwOtrqVi;?K{jXdE>OuxX~$&l+p@zEv8GkVSR3u zxRbIDXAeYHPQ-;jQT9gic^+C0ZW^t^K6n-&`%vWnh8Z7?lrN7*ZolpNf+LQpZg49TpfhgO~OvQ>uZYJx&Z;&;sJEk)sO6CkGU4KdMS zAiAv9o#fKf=lVRJK%>( z*)Eo;vdSo0l01w!-&}M(sO@9SsP*$=8Cb2Y zp}_Z>Xp>Dh^TTfPgc3y*wZh~Tdybcx(uA2^)fsb(3*-R@YzJ+kwC96yknSj*} z8#p}|r}E^|#fQNPqy3xH!U`qA98h80n{zEdXNcOb{8RYL<7&*awj4x~JCzZ2U~5Qv zF7wR!X`cKPSFBDO-NKNV2}epKec8iUTwYMf7mR=GFY(uh4K zLu^Gh#{$D~xPfOrMO(DT^@w9)L8IE%fEL** zgppSFQ-*&X`Y5~q=>eak<$s_23nY>v*zy)GWcDQ~)lv{&)cGWR)`Jyw6Snc!ho|rr zt?V|r(0Kf6i2Ywjhi@*pt!5agGlqLHm@6Ii<*PG@%^ZBJR2#we7WY`4`HYuPOxyvR zCM@@-KEr4xDR2JO(i(P7d^_dvOzv}B%XoSYo~^KCmEI*jC7>gbyUOb1);UxPZG_(L zm2N`RwBYvzBc-(nx}Ry+$p(e#uOozxmRHvNE&H+N!1WuQ$v=CXIyl8nP9=8}q?KhX zlqB!S4YDsKhZ7BRKM*+P4h1xR`xXi5ms8CVU?hK%`|^IURL`GnZo{!NCahP>2m|Q ztw@W`$6<3>T0?V_LTTES<77bE^=yO9N6uuAAF!B{O!50e$I@m7OOmgn3=+g(Gz0b^JzK9f^%wRQep%$Tot(`J5mMn(;gqu94S#4 z^bBV+qqQN4mzVurBb;2yVAEiKuSnpExUUC$6^2*Gsz=)gH!i|zrGKUjtf zLzRZ^9oY`K@5lM0JoQ#HUxB;Zd-BGOwahz>+=s_Y{_)troqT4Ha!f6=7hTg|=9omN z9AfK`x5-jeXERddf8oXV!5eBK>L@bh-b|4m@*id+E^YLpGY5m;VYm)sSm|%nxQngk zGzuiY!XOpAAiz}dsCmZQR5`(8bPcyBp+qA>ji?%Gne z6oyO7TdJlfwO$mZ5VY3AEqryTy@+WB+T3|BC{B(~B788DN(1YEHQrK`a zWnIHB(b9*RzZ&*C?RCE`laj*5ovU%US8~Uoj3?`0meA1xNkhXaKNoM0{8JN>tlZsQKm;&&Hz%+eXacqXmB0`%3|s?Z++_pS0PBDh5ahqv zfI46+uo9R6E&_R$z=nv3h=_=Yi0Q%V6^_{8pIWgOyD1ad)EIA{Kk~P)|6uUol`z*3 z5qWQQ1_FWagu~%wl$jT9`|fR&*3-aoAPejUhLHE0ALs#^fv*BTV`)Z!&A?&cM?isl zkR1^b5fKp)k$-2SKpClcU}N#k@MFKf66P8rBJaId+S}X5C_hD@h)9{SBO)RqA|fJU zDi{mJso}0vWYk8sl2RuLoNU$^;0T&PjX1PZ>5fKp) z5fKqFEu<-Qyv&S}Og!aJp)+EY1#+oPGm~&0ay(+FiJ+>apY&r%6t|K|T}1;$FFh zD*rC9+4z@YAx~num&{xci_^V^sGoks&kd27`U6HbK_)puCK;Z+8FpPjM5KhW z*=(+@t?lH%z(9-B>0IB~*ofQhX6MeG?Afyim&=8vyStmAp&=}Ozn|9DR#vTAh2`SK zi<~=mj&wSW<>0}CG&MD0iAJNGI(3R@G>T=%jvegXyBCkggXQ$;)AaZEW2vsL=Fp)- z_BQhDurd=zJ0W`v|vdj5*#~rjNaZ}g27-B_$lx!z@W6|fqCSEBoEx+ z|2M{cz=nv3h=_=Y$bYDoFPj z2Y>UGpRDb|RJ+os`r(0z4kMS?iOKHtUOoQ1_0RW*U1OMz?!WcqhLw|tvDoaXx5w9a zo_Kx5A2B`g(S92M2Cx4yWpGheV7+*au>xKBPi*=}UxyD-@up5Sn{ zcc(Tk{65>_5td@H7jEh?G_p@MT~> z@G+pWwzl?cU0q$b!{J!K6buHt;_>(=FnPV+zt`8-A9lOl6PRXZW}X@!AOD777!P2o zudnZ@si`@SDV%8rnP_ z&lsjyEVg@ebhIs>&#%A~2n0HPKHqOK<#M^2>({S8x45{N1bz*?3d9j9Gyen;!pkwP S<@u%n0000J~ibV}N^K}JCZL;)2U z1rY=U6hTl#z!6af6`4VC0!2hc24%h{@csAu{Q2))_pa5x&2WZ&_SyT{(^>E6px<@v zRqd{7(xl0?B;gD}IR}$8?Xuh$JqyXZdvDUjxDS7{CBMd~7OPP>Qn9&4hw`vY@0z@CzP;zX1yTp}`k2 z420l;mb<{Ey;Ks@@pRIYRk|5**krlKgn-9AJVeCj?Phd2B^%HYz;gV^>fBxA!$)ms}@fZbSzR!<+3SC%_ZDus-$FUbu>)E zxj0SM!R%;Oa_|IT0;+;9<_y|Mrwv>Yh?F8=3tEG)bRl6k0WNWN)oL?Z5>+}1?yL=W z-JdV`a#4-UF@Av-G9DieUZ~!3s)05{h$pOZoG;ZWwdS%)PDzL|A*trJ6D~GTVJcY3 zor)JU$z{)x4cbA4B?}L_%T`!K9Hw*#1~-5ZJmo~a1*7VR+=fu87>k8-4QEg^DG8Dw zh>B1UgWkMH)G}nli&7@sjpR&9HK9N`kBfwm0&h)Ym2{?LLB!gH5pg2tNM)Q>na?4) zl5CL8;e3&U8CxPzi}E}}M?+xReA(bjOAdo1xdL=Gp^{bbuqya*+*E83VNXa-MGCm! zuSY^9kqm1gN{A+;FhplWvrt$37I#zt0*QxnVXWYSFpQ>YDu&0rUJv1^m^{gZCnlv$ zjB0_yQp{N=eK}SuV-&8I6+7(EDm-a!G-L-1M|ijt;UPhg6&_0pU`&^#Za3E5;Ur_R zXTmIR!lZ_WEbvk`kx1JSX9zFmsTok$;l~# zcR2(qm=DOt29MA(7vk(TBkD8PT>dzvgfno$6OL1cvS{&fzF1m!VDsLySW~F@a2l9VCoJvmQalNwUZn^8QB2)gX+Sq&uh- zYdN#M9+%mAIaW%uVIUc3yx_CSa5m|x z9B9K(t|VB*Sw$L3NQLZ*iIyZv;rVh<*Zo4ER92O;2lFOk5~m?jv07$r7#%TIQgzjY z!!^3-F$Y-O#%W1Y)&sfe5`qLFRMRvE9dXD>0e}=;0)m0p6LiXe8%t>~8&H|11FAo=Y6 zs6|IBJ`{}Tc92XsUA#f4xEnTG0a#j~r-4e=C&dIVTu~clhq+p;Mr$T#PKxH#awuCS zDuJMhEm#9667uT64?ZT1a`B}cWvvuhCSqe?M)1InV8xVFYK4mA@Ptq)4B7#tjTmGw ziw(CKG%jAkz%}9|YjBWAWT^n^W9kKg@<*9SNpjmOJe5n<0%plnBjX9Dr>5kMqK!fP z{xTTM2jiY%3`;clWLcn{vWv^tWZgiqoXZPUS=i<=xhrAB$g1VE1#?#7E=;0)0&n)S zN*owYlgXZUm2=sG)DTjMgx{r)D%eA|n!!^ycxw)8qvmZ`ef11a69Dw^WZqhjsbtP& ziV?vK#fykF<%|hw&f;w()A>@(W)KocEQy;*S*_qEdr$*r+o1*x4YwgsguU}I{PHCpi)Rimqc1n$M+D%o&L2#4unQBJeftXtwu z?s_0*!L&RqrHr}?ktL~;CB!NwCWCpLEi_zoCB%46fOdR3h zpukZ=Bx1>+WixNKL}HpL9w1pK2N`pfJb{`RC>5~Cl*^M&;fRti;Q`9X@v>%12i#$u zPx*Saps6SXF6(NY!^}ZWlj}HEk2pQre$wBz?Jb#92%L5>O_U zEE=nF&PWIR7Na{_6g*;8_YHhuKcq#12^op`NlVhkhLo^P?}0>0nUqg%G!(B#30jpD zUS*VEkkienHR(-IE(@+Pxw3`glJ-WDlufv;71$YoK87Dd7qSHk6Ji7H4&x2YO7n{uE* zzz<_qXHXX%D-lqmF~s7jGZil^ik!Y^*x_V@aViR#(hW2aH>UthpcTqrs^@EQQ^FL? zW=pPo(ubKNxKcsgc9JB0_97M0IEUh@VWca>TccQ%uj;b_ONe@%kd@9fFve=d^%-~q zwPz9{V8|L* zsv0A}q+5`RD^~ZLI452zQnX^XRa5?`TH%aQ*hTy6LX~CcT*ymusE`qD8b`!xO1TzN z=$tPrOQgmp(iSO5*buW)6^%lwWUkt1B@iZ9A6ju|Og2LCiJ^j-*WI)hJv z56eU-A5MAw0Sg&OKoN`Pi)7MXi?isbZBi_cTPRDz<91L<(i-^BIFb&Fgqv zQHthR$>!sAc9s|~hv&?xRK%Bv*YZIEOh+hzki&ul2I88IixH@#BSpcguUS*#UbD%O zF_Qr{63l3nnpo@Sga{qJrK-i{+|qo18B=Yv$MC;fQAD` z-pmygpsN2g9ED=dL5A8yQ-K=ec~iB}0KZ|YEvQB{f`KAT#uK*+Y9nv6_)()T6VfOEfwdH6Gn8T}mDGF& zHK&DG6%FDxA5$#V5{+Em(m)tZb0$NunkQjTDCZWbyyUKuS;-ax{)#zn(KxLx5)zsO zB8g&v6TnpCRW5=Po@zc+EFcX6Wz~9-^JBF@D1*n%SjNx9V`g5rd%B~tSm z>QdapR=8ZD?#KpQ7-LqlzM{8LETotKFJn;I$-BjHBa})(jR5V?r9a&u;u%$rTm5z! z&9Oc@P$wd}6z>hGY=y|>P|@X)O<_f*psE%ZT|@y_Sdfp>-3bA<(4@g3L7@a1X=I92 zz!Y~RSVu4?pfFfNlnOe{X~j?=AkaE*uN#~MqE-WCE8&K-qKMdX4u@C65U*Xf6ijpq zxL}|gu|FkJ|gF#x|0vv$P5XE9U76KGv*W)0GG=gdvZ|EQEn}={yLkH9tYhwWP}zg+qm+ zlCT;$pK1m{zc)~V(rLFRomE}1#H$>_^O2O@66ehk7ok+0JVLnex~pK45@Eo(O5Rl0 zG}Ua3`7$O_O(3uyz=S>CM$DG++T$Re@QJRPf-<3Mrb1ghDY^in{|ca@NY#=Mt5w8O z!&r4-DK-`_c(Yo9Dxol5^~5t-5-(->q}h_J3sN#ucBg92KvwjoeC}cm2cQb;fqoX1 zK<~QN0HKIK1kv?zL9(Tcxd3Hud@cPA;Q6m|(QL`&ufpdS>J zFcATlEGDfq2_@4sVkr|vZ!wBuD)5U@61Um3d?A&hqlyi&I%{QnEbGM7ibGc2A`4M2 z#8D*bke`ah<0(5*0C||ArV@c*(aze?U_cN`mZ%j22{lIuvO8npjLTnk0nir_+UT?- zD@mwEMBPd{;+2Iw#hArXn6f18)vV@53L!?cCbD!a>Xo#HMhMlEwPJ>xBB92U1}*0a zxO1+0P)_hVPXkI2=u4E#n<7X67K|QmK2b#21}7--cpkXhX%$maLZTkSHH#m(5498r zGL3;Kr?WD_ISiQy*acu1P$3CJkx;{6iva_HR%)mncGeSCo5P#;#u+P?qJ3<(QqEXI zf{hjc>v^`2wpWQ!5@oi0Z9_V!RB4ojKOM(g)+gs zA4Gpa+G;RpgvD6QpeZeqa1b?;juapZq9Ada%EQKR-M|%Vi5lz(6^tfb%uJRbucZ75 z#BNgHhCx7}P%LU?wQG?`1Smb1t<$W&m6-?&vc6$k zP^8s(BErInP{oJZWwF6~tS$$Jk@lJ^19?=VB%+>Z4yzjSA-^$Os2XZu>|&Jnhy^y~ z_9o+K!RzvsG(YD``m805pgB*dXvD~jP39XJj!C#;es3zAj%a=$S7;;*q!nmcO;w|y z0)S`PWX{1^t8qQ1=VE+`;tOt<%PaA^hvN#MVKtn~(3p`X;gTDM^ys`w0q-5>-Qd#a zteFIpt&lhCNN%|xs}{);@XL&~R*lCk0_l}%xCvyUl#JUgS2BnMSwXr-CB1&DIqV^g z4!gIWj)=g2buk68qRL@k-fNOv$wbx_moZN{1;?mz+y~-eJTFOXgU!MbB$c%b4s$V7 ztJ}0dGB2t=D$2w0L=_K}aw^L*RhTuDb1r)^5sYI>05z~t6j4!|>VQ3DQZ56=rYb3& zB_Ly(*F0sjO~|;jctaO*+yxo(P}yF@?0&bSsMevHSC`?mS4TA(P=XsBs1bk*PdKv% z%~_XtqT#eNY)tf+-3cv1ina>ocCvXPBWF2Qaw?e9StJAYFa?*h7-O^xNGuJP9Ue~= z3+i*yYQ!Q~GqfQjDxAkz2zk<)El4UUvIJ|zXr^GrNQ)dO(v+GG0gH@lJY%vk^;{$n zFo%jNT5@60Kn_Bq2puz2abv8+c+!|TN=9;7Tinzr1Aqb=jTdw-lw&ow0(oO}Bq<~?45?Kh zd)R9Z<&9pW8>!kOQVGpPbk_i+l9PBCgs|qK$7VoN22xMn;3d}N#CfLS90-E z$N{`!)KpPJF{?=y^;9irmlktwpOlG5K>4p4bYTfBk7Kb)Oz>$Ep9!#GEo>l|Y%uMw zCrkz%93Vu%++d{zvjHK4X&m-AE0zdCRjmzNmh9G$6GIrx?jmy@Q`qOn3`A11d+?|~ z(11kV$QcqwCvSu8erKkXv)BruNG>UR!VuEn6Ga^Osa~||G6xHUPv!`3y&4SZIG;%5 zQr09SX59(3NL55A4t#5r^%dBRAVo@Xkbq_cB3P7@F45wtM&bdAX8BABWUDEEEnpYr znyM>CP_C5FkTo3(CO zvp=E6-K>(? z7>krj^_qwEVX!fpVj4jz;e^qsJ5n#=q~9FzM*~JSmDXt3rUa`ToJg1;?}ZSg zGZS)|6(bD?b9&VJ|C8l}s{tjHt`($^=!R33Aj=Z~Ex}6uBg^@JiBt_SAHb`B*;_7U zE1Gdo(+zqPxke`IteaWMXbKsnxaRizOp%;KRusXTu*%u28U&@ysM8fEyr?6bqbQcq z%5*$jv0*kFROg6@U#;5;RT?N3&(7&J?5oX>+M$%95x!K}mxnq^MerGCo<&h8<8P znabm3F)gGsQJXG-jN2nJAPoeD%$M=9oMWRNrjkhdg871;22J_moUh_RfK!8pg$SNh z%U&U6LeeIaxokE&VU4ib3n?0$Nr2^`$C$_zOkQ`|E~nwNm#Bbv2(MO*$qa8Y*<6a1 z&Or@D^<{7(J7OGI7*?0;1VAH917qm4n;W7}F8QcRq!g2R6e*BpFB}CX z3N0G!Wh{m0(o6zoYjzvSSwmc5X{gK>k%njvsO}gjRTR;h zBW=r=auF(IQ*urwqRCj)QFFwdA-m)Zp;>3qVL*#^a7?3`urnYP3xY>a4&~f7#BQ+F z)ntie>M9plU_=OnPbsP6aK;(2=l@&?741ww-;lTW4$84fSl1_zk35ra+s+h%jm|p^ghMki&blwbSuEywMvU6HVG;d<`g)5g4A_DZxksp8%Ts**}9K~LYTFv(6)TuVWRvTZi6ds zUHl1<{o+V6P|O=FB#lH0m1+Y-9k_rYB{@jvQ##8%nLJQLAkU~TN0`DXzbod920;K~ z_lH$3tJ=^|0(0^NgZYU#1#&n{O0b&*ip$$Ihov4d`kjaiXar7+u*GbP0RZJ)Mq|To zg)2B-#q*IySq_|X3!K7 zVkp5F^Ctsb94?ty#g@~GkQmK?IE!i2oCy&}(Uc_0QJqtnhRtb=85uFOZQC|#c0oW-sE=2E5I2i`8Of@!Kge`9k#%#edp(CONIDG$4Ht*k&<-aZ3zwFn) zBT6yiOvE%5@x=X^h+Qc9^9}MJ77j#|z{2@Gb+_)4)?IeNZL~;mBwCFKwW?&amy7;z zB9~RYHd_M3)|F(QM`cwjK~kXPXS0^NOHahIa!yGY3@(SUnzyJKqT&n*Zm5XHtAT2^ zVQ*kS*mU^F!$}CLdcAoN1Y$zDOeIAjOjyH-Bo*bNxHDg3q?F8LBUMghtj?@tr>qH| z->sICA{({Jo}3NJWiyx$xbTF#5Kn56eAE*4)U6f8qDovOC|1>UIpRv0C6B>})G0iP z=j$LEbvXqm$oitSL|H+z6s4wWS-w!Knliku#Syo}I1Pc4VkC^9g2U-8Wh8H_8YzQ7 zM)ab3C0@nhs5Rw{QC0}AzyV(2RgjFKJfTS097Gz8f|zhd2tHFccw%ms(G3@IJH*sc zw*XR)z7ko`bIx|)mN+O4&Q>{qmUgIO%X>0l(@`ekOJy5*AyQ5y6@%SZDyZ>tRDsKe zV$Muf>!3-I$(!Sp7@~_j0W4r%7OJYmVs4{S2gyO)odtD|a>WR;5;=DeD`#kx})HBwgjM)on+O3*{8LV2Q+=RcL)$(~N+0a6Hg^lXrwMsP7 z&T1$SafZPig~Wv-sRtS0w1Xv-ME&-XUaJ&h_5k>XvaDUP8||1H_k~l81=uq`uxED8 z1EK%`l8hHrx>9)uTErR*sK*e=n8MhphP@v8{Fk^kd0i(kyK=| z3W6_PZJe@~J!~%Ut~7=VS%J3J)EF8hp1FjmN(HeaI6VgHa^R+nf^~?pqvT230lS?h+{eIVDrZhs zoID%1#VerR6?en}kUbX9@+HMd;sG)VtX5XBr-*<@B`LI?ZU8-Z*`wiz=qpoNDG|+> zfsHH_%6^rHZ)2SL>P?$qXV1< zG57!nO1y{iqIh1k+vIvgE@g5FOi$3EbfA(#{N}Q!Y!6m^qTA;IMg`9{YDKD;flzV5Y!BMZk#R--Y7L|u1}XorQk5jZ87u|Tg2f3=FXH(`MdBe)s<#+)s?sPZK|=)y*P;m~ zs31Zqnlgp5enns-YQn4upjHA-IM`&?mABZ@xEq{j4WWi&M)y;TW`E6+v85vw8YEH- znF8sr0-kANG3qj#LuEL{fKx8Ag&`7T3e<(cAcsLe}IMj-4cBsyFLkh8%sN1b-oAUj=2k%%5cR*PKC*9gOQOQ=fM zxmwsPmu+c5hXe2_BDJ8&%6VC&2Jls{TzLt#RbGr$SJZCZw-B@?8dv4ELKga*&*QIlfPq)I6+ zR5O`^?tli1xmXP-5vsy`-b=ZQ;8M}7w6&5}Dw$$RYS;`WR~gGHaxEp-eQ5(;qeGfz zLJ|s@(&siaX;)1J$8rCslPuu9u!&E$p46mC$0nrH9;rY4>x?U(yAj**_`p`+ z#q7z@(7TgY$Ze0ChP5`m^7*2li<@oy$4}YDw%pkH?(Ww-(o9B2jA1To9r^q7!!uqy zJY&YL8NohZeYZedgIr&Fp>EyYZjfo!W8PV}bTp zp1AyUr~1`w-z6P(UO(>q{6GI1vG(To2fRPz5%8qlzB_GwOGCar^WMqfYo9qYSnO)( z$Zgwq%ioSqJGjpU|9oO%-{;P}^y6b)J9M>OU%KPl2i|K8JKE%@%Qjy={YvZg&R-vB z^-!zx-3JI0+7$;rJiJM>j!iCW(XUCS_|vfuM9I&#sL=CA(yi;HcZ^SyWVd+7SR zr**g){ck-lHi(Sw*QL`!_fg}u|KspYnz=g<8y~&lp8FjCWB09N`hARU|8pWT;mz4| z_ILF?e{A`j7rOT8k{vT=#l#n%dvooa*K_;tI(pxoUyu_Q50ww|*%6DBtR7bMeXKj~yr{5Rd zLT>t^9t`&+_jnS%Evv}cFH{Ca}@VqgkZEAc2cj?Y-eR1Dw6O=~x z4?cf*Sqnp(8Dg>d9)8ODB~v!}CjNBxx{2N0OV3}qCpPiL&5gy|kG}BT9WPwEc@te7 z|K9pR^4(Sawv3+g^k44Xp73i=8m4t^)A_pVu6c5N^qQ}?oZgx6u6Oo+e#r7^Jy%RP zR^J>vL>eb7+nH${f7Upo_XnSKUR;@Wc;UdKEdp)ZPr837baZMDq~p9FmQ7sIt5jb6 z)4l8NJay^BH~2dC;2ND06R#TG|FE(8`lB6V1Al&|sUkk*>DGQmqeG_+XOXF%Ex%9n zPvV-U7EfKgeA~ptU5~sMn)uq??{{3d{KTWLoF6f9&Ap|=3mdb~dJU%U7Co1lFeAfRx*VSKN)56o|-8P*zoUHBgZjq;i^6e+2GxX|S zZ7h!+e+3zT%-#CM{@lLelp!aYd3(P0@ccfVvU&1E$Kf+BpV4di7vXEhW>@@lIQvA~ zZjeNM>^7ppL(NT*Yo_NQn ztv63?^ZuFgV-rT5DDL@WNYB{%SGS$PzMXURrR!a(ulK;_8~0qdWy!pC%*c&5`#KuG zOkYpVw0!x=i&v?$=4|UG_4stm(6wC-4j?BLPS}U%drL=#PF(Tqo2%!vp7I>hbHtiP zX8L!xzVW@09Ez}yjV{FY06k3`ckF)mUXSM3^5MjDcb8UU^V*L;@|~Id$-Q*`g0W2} z^gVHE-ta@78Ly|;?YK=Am$VDnhaW-6eFu)+^+NHk`}gJg@87&?^!<_+>NisFSY{=S~m5ddy8rbMR1k&5;K$UF3ph?+%+6 z9@*3B#%~aI<5iQV1zU;pi2M2;9f5Ub<~%d^ zzJxY;UoZD7%e%I@|4+Ehx>~P}i*BFP*fM+Q*I!S0(%>0)Y)?mWmFJY@ktgQ9wQITk z{>_uj_{8JauW*@KizgDcqsJ$h-+uDz$l^#1kcD^~d>F-m=|I~NCaUYhM z*0TI;w@r6uHPf?Kj_TEBQ}+YMEalRpZBn#d8~FAwcYpfPuF9xIiH!q4-CgQ^>6$h- zYuA$b*t%@|9Lv!M4sUvGM#lwfyS5R>NcQqN=Sxo*Z%%Y-EF5&>jc?XA-1sx{<%X+< zTzLrEm2NxZhp_`j9q4dZwl`_IynV0Ap>L>dq?sI_<_QaRUKYv{p+v%*k;RG z+>y8Uf1EI`txOx(WcA*Y2U^^?vqSgnu8qxNv-X}lx<{RO?$a~D>-Hsg+%vFmn`0xY zUp+PN@RYlnU$J_?mT#s!=j{3LzK5>9ii!R7*^5SDLTSjd9t(e%cFRXM?_OjWymsTW zZ8v}ogNDkF);s*uZUO5vKGHPZlDXpLX>)cQZ9zWR!?XYX`<`vP^Tu~qOo^<*x*qQN zsdL;TrprDGU)_7t7gO&wGFxB3exu!~!m;O%+a73R%X~i(yL?LU260Wt18XxO|C&FP zadWy-#hO7t8=QC-`xM&UTvmV`v27At!~{05BO;0O5MPDJ6^M*@#mcP zcAsAH`59qx)Hs2DzeVpulsorQOJ>`PYg;&#OD%NQiT!q=lkUB0@9nMXf3|Mdl0wAbQ$=qEl4-?{1OvzMP0 zAN1Idj{Wq?ue$ZdwuCQ#?C=LqbvQ8MJ)ux)LQPMbS+>bGs48nNuJbN6<+z0Y0QX`Sx`+wtqtSq7o+%&wnq@AcNn zL$}=6rRPmcU+pt=!sMT?xYP>XK77o`UN5uXKecWA(jKquHTRu4X|$*NgUlPp_9Az+ zkJ{$cT#QysI#(r(fX4oJYV8|Lb-0;cJ!Qb^`D;{W7mo`VzqYm`Gs@Kk5 zGrpMlm2=AZACGjqWt1}bl_Bw$+W(MTlN`CoeR9&OANr?fFMMj%zFxB>;ah%eX4&`W z+m23mo-~^I_{~SrN!ZX^hpyc>9JW|Ka(MO8vU; z|6*wI?CnQB+_G2fHSm^czKP@byPk>B-xNoU>@{fVOXISeI=|Cv>B0rEgO6@%$=vgI zZ0W(?%eTIG!@;-q&R^&=<aZ&l)fxrLwA2968W+MW<7i|rB=9!Kp_dIYp)4og3y}$L{R>H`aO=r)1uxIY( zfn&xmUpVJ<(>6aY{4`yB>+>h$F~i=@W4t3x!#BMCRg_q-K~$R!>w4hp|McCZt~oHW z*UgW520Zh~(1p}-d2^H4of`wE-@Nm_M6(&&49o7kt?izxS~4pG%9*>{*!ZSXZu<1C z-SmpbS6u%qd)W}V8yC%=OWMA?{!-;7n%X;UcqW{6@6A2irup;Nk7>!o`fj*+=$Qxm zliGbpA3J^|KY-c&-R|@ky|MM54L09yuA2y4Yd5;wqV@*X6M4?)#A6Gj&gKm+Ar*DyyEsr z3obw4USEL*%IKu4*K9gc{B7`_%R3xUmadkcoO9H;@h_n;e)?Zmw`8_&8cDPXp3P)O zwrws<7@8T|XVag%rysdUq4Oc-iAyOQ3n-klt?klzCz#a>uOIW?l61$W8{S#l@zY!v zX4!~?Gn=j3>DzL4(^J_UQ(GLXzcLWCp1XYT?YHdv#W-%$8S$RV zp6|5b<9{@Y{UAG~SGye?De zcY-gzBi=J=QEubFt9qWgG(xz+HS6|HhdA=)8|@#w`0!tze)JMz-eKbo=Ihx9heN#% z=MPTbF@oAj{nl&gf$!dHbDcbD(IdYkUu#VBXviP4cV&{#&+R+)@9!*a4t=wE<<7a| z_vbFk<5>Fp%&H&9OgYzl(iOt@vuAGJ{cFD|*8%djf1E4q-!qLT?=yYg^&(-t?mT6` zB=)$0)Cs$i9MJ7-v{~ymqt7Im6J75gJO0R?(-XbKAA8zQd2`vf&$c-G*VE5(fnE;* zX|ZkxcD8-jK8g2UdAX_F|Hl`wVa%R~w{#hv_j*qo`KM-$Ph%gw@R`ZAWap}F+wWUF zto>^TfMFZuoa#Q=_KkM|LoeIX`MXJ*4o z^0#{p*dA_mwwCTbV&VASLq;--58V5B>;J(GB)fjw{KMzEaJ#CLM!Yr9)EZvsX?Igc zKCtEL?cUvqiP+9&)8^iDMwF-uuZLGX zyL*~;`KPpadr#NsXFImN($_j~$(nKb5nsKY_Wq&aA6>QGx}p8L7pG?jtt-a1?i+LZ zr~Hb{`^VP3vVHSwuOB~??)bwEmpbm9TbOTHI`!yhuTR+j*Rn+PO7he_pN~3m?CqbL z1}5B(Z8_OEb#g)SL*`sTlPjePaolv7OGlPP#cWg6CxdB4(TY_I1xee~YFP1hvO8g8=;l%bj9ZdJWXcJ8SS$!Gp_|BbxWVa(utx!5l1s-rTo<_7TN~DO*(b%eGty1PHL*h{J^Qhtm#%DLi%2i-q~X`mL%Syi-BdFmrm-U(OA>n%{DnZ%*!2=DF>#t^J=ZnMtSA0o}K3$UVZ$nK<(RKOvg=WajX) zcVxTn*=L(RJ8fXVZ5-ZX=i~e@gPE!9zNg0D_~`ixdEhN<7Q;>1clVUfJ1=^4&tmK9 z(ag%j%b&XU-1O)FsIJ`A`?F0SA^DTPjECX)C*Hdk&VAvxDF=mvi@W=GPaQJj!=3N! zox%+Az{{We>XBtFR&SoDWll98`(gjSp7QeLgSt?+ui1Wd)k9aV8*vE)ee}HUpBupAGb2XL{b5S!xjjIA;!O_@UOBk;h)~?%c%tjNR}#O!N3}gWWdCMMH@@@O zUibI@W9qJS_R*Dh4ch-_b>$nq(a*Qs-@CfC*_>qmm!`(wKXmVB&UMWP9__Mk`{owm z_y4M_ylLsjCx*>^>D=C@-e5Nu2R(DCe=@8e{#DEE>vPw+U+^BD_sF#ikSSBKSl7X8 zyj!22Ti!(-eDH;1s{Gg$-xR0ab*|9-?RlB-xn8~{U)-`1;I7a3{-?hrm~y#iyJGgS zm7&l{@6y>nc3;u!*#$Gx57N#T_VrpU&hYO3{+VunN9HeFc(NC88|IWJu94Qg7Cw1j zY1#X&COw`zbn7B=zjoca$1dS*zm8Lyxu0G?rrqPfcbVIA%cb$4p2@j^nDyL~i{`I< zX8cXdZ+icZOC%5s+p6>MS>yNbcxKGlx6|2i?|*Q(KhdsT=bn#0e(96|DCc(?ZoQh3 zf4UL*JWj8ET$s}C663Ask>2a3H_IP*V8U+^@~4#}KcxSWSNh*Wf7z*(t5e=YP9t`E zwsf}jI6Qhx2l5@)mW@9@vhGqO(7O$HxBMuy{=R`*w)m{)-kLLH;1@B2!N>1Y2Q{*CX%F{$|HZ_CbHbNz|S*IvK(`nNm$`PZvX|=z<24uJroXS;)_%~jzaDwufj@JrM-s)-R-fW|2)h4C%XW)u;b`kLFBig2txCJM8P!WH{Yw{kU~wMtylp%YV9=9lJLD zx-B@_wfoh*%#-Wye&O~Vy15d@G(p>*zWkr8@7L$Dk-ch{>+5{7O~>Vi@2?)&_y^bf zMO#?Cy^sA`ZM{2s(cFLNyz^^Iu%@~Tb?*EgSa+eIdKLg${J@GknR&UbkG3nqgg< z&S`sJ(@zcmY_X^%;FWmegq!k)TdoI`c62vuwRHBI?RGZ(2iaS}PQ(_kH{GkRyfYw* z?%Ly>e|i)PI`+%4cTc+YPO!~y-_URUm_whjmVdCZ)fJbGTx6JU?5;n30&rSs@$5_g zVBqkbHET+<}{$UYFpdO=WT*RG^kRIi{w{p4PCa&bPWM}v3%gKh_loGM6B4Y>Te>rOk zbL`sz*K6mQJ*o43ls_AHVxQqpT?qAu*Kc^zk?Xr))RDdY>1)BJj0A$_ZrjcRcJ$TO zZy$YSmHCffhM(F$Z{V00%wrJB9&FgdittZ+hxR++XE`NRGhc7!E+W5ofEtCIR zXTGIfrwwrSf*BeAFj(2a0fQdUxBT0u0KJ=o##Ug(U7E6Y%;7grU+~txM_n(xSj~0` z3EN&=s3o`EnR@Wc7iJFL_2kYs=5<&&`1-Hsb?>_NGR{Qo6h_-aKNC0Rpz!< zOLI$a_g+kP!yUghEdOHJgCNV??(Gks)AxKL3AA_0p)qkFx?eT>;KrqArpz3ChG{?e zJ&19fYQHsb=k)5MCw>|@(bw?y@Hd|5piaKdc3}L}d%oL#sPOU|1D`%F z_FMnz!LY%0VgDi%bfAi#ZC_qEX5pt1Fto0Nlh>y|z6+>q=VspwJ~ZI)u0gMlKqsv} z^7+2=^Utq2dwk217jC2b{5bdh#HqfX9UGs$yk_gy)z_W?w+e8(QAjliRgq zmAEqhn3mY^+LA?g4E}Dy4ZO1B?305}9U8?y+;4s5iRUI~3w~<7G`FIu_Zl70 zrT{?e(PZYv8RPUV+13JJ7rfQm?6MZlr~1LyZJFDGxp_wUjo;oqvbopXS2{2K_27ud ze$#@*)u$${IeF@~jW>oVMV9}AZ-4%00VcI_!!XN z6U}CK=MFFXa7{Z~l54y1=~chaT6S;WRhP}5V&Z=}wW0V$^@WFjTK!<)r^Z%kZlT@z zO@IH|=8D6|Q~tjA={uG_lX$bonJ<2C{h{mUVFRu|wvL%|sB9b>+U)<(CO<54*A00MDXKmN%I_3;~WG}#+96oF3lZ(`E9(4^mesITon>~fMo&BZ1 z_FeJ%jJ@Y?y!KY}M}rQ3kiTaua(`jQO$+aM=O=bXdE>3S&h`I&$@0ys|Ga$@^vUhx z&J6u=jjOu*{EP#uznNF;Fk{G(gHz?r+rRm|`0k&NpV)VD=(Vre*T1@}Z8CWQO6D6c zdjekC4&$P`ZhA%c_{Kj3RQ1yV{U6W?^31xW(}vlzuG@RKqwntwx_#$3WFN8Pn}+Z5 zA74Bf3;qAQBsfVWxr=&(*;Nyt3w+5LuZ{v zto3>jS=F}r2mLyK)Xf9vq36%Og|D^XI|kMUKla9=XGYUAe)@g-;K8Sk9ZamD)!CKA7b3yZqgqT~bJ#cP>gTt% zx($t8`<)2}EUPzNiN>hwRA$N8Q#0!$wcq#l@7;fe_(gT+vmYuyo^z!K4my1(G5V5=HQw9w&iC*3T=skW!MnfQeSXQ=cW3UqfEcSDMDxeq>R_I_VXc8)c6FzoNBZA% zrEWeVt8x@>L1Jl#+C41?qQMMm{9P2P$k_khXqrrWHX}bC zW=`~l<~M&ad%(@1X)C0w?)eb9@{Wc49fQ^l+PCGhnS+=8w(sQbzu)*_`ToT5<;xev zwtu{1z>n3{XZH3~Ho$My&p({oHTT%g4b|f-{#@SWulvIvzNmdO2oN-GZKuC4sPT+u zx4+U>Yx2K1d&_{Tx~^Rq5Rnihq`OsG38mT6NQnZQM(OSj5m4z4Nu|3tDIqOLv*}Vq zU{fmHXD+<&=Y8JuonPNSYOgitm?N$+#x*97!^{|%7}Ke5mouv2W0ctk8Bc{rlVUyZ z1_3Nf<%1?nvt>qept34OQ**X&Mwq&+G~m`vPLD6)ln-i9FN#eUZ{cdb-S#XnpgO2J zTwv3+%R;X+)OVl>D*34ydPlz#ibaiF!dI(^}V~! zhXmBGB8-~qhvXMKUBCXw2yIv+Doel>7`Er6Lc=@~=igl`h2i$Tzw-`vmNrst5rt)q z`}cxMUXZ;KzcR?^d;jsfPZ1FgWEj6$UqzEHB08L)15Y4!U^Us-^BTaqKIdy(6z!{d zqaS4q*Yzpv+W%p#YB5m)*Kz#3J*|{N()q5p$YsBQ?g|@yR|KJ_jN|P*;j33980zFz zUlpF>rE>}df@cz1NVg?KBkh}-_YbJ;&3pXM$B0sjm-ViP>{P9kQSB~J?408Z%}P$r z`=%f6a7{eHUB&p93o!STq~-n6xd=sJg8hdN(TG#Orj7VrRkPqLaiL+5l0cfvD%*H* z1}=e2Ur&Q!!Wg&)Yzhc!rii#Yeg$ zg9RckF5$bz=&tjqQV_m!5+I*WA9bNEvOn`EtbQ*4GJ=Nh>FH*Ly<6II@3NvM`X-K@ zQRP$TlnL!Bxr1i%MQ^EH9^Tf&-B)2)Yi~I0IIYq^6)Wp~h&lh2w|XT>^$!jsib;4w z8yH0>PXSzb5Z9e;Al49|g^c~EXYl5Gt@{g}y866Jud68ND*Dl>evfWUTies|t6AGu zxMf$;ecF&p_nd<-Y&30Ghp9CEs$)$yx{mqoV`#$({%yZ9ut69vtx;9_wMN=akji6XOY8UE*HGXx>4(BC_S?3OL-DM! zs4~?r)@5y88k#?3-Qf|%H!>Owl{lNF531-}E~S90UqyaS$=?R1q6 zGShclNWeFk-SzjjD8F_v3Jc@T;SoP>Cuv{sKRw6th5Wtpd(r2*_N@s*g7|;oc(gPI zfkZ1nlumtkjQw|2uz(LIteUz8srgf-rJdHxiE@d{qC>gna*aq~fzui(`rgFmx^Wfn z?SceN-7g&V28yQ^dvd}Sb%Fc+s;wSL??2RZ*ojy~AhaeANm?h7|+`!=cwpP7iA9#!W)+1eLi-0n*)SDR#QO7!8n%7(zpze@sRASe!s0J>O! zNwa|cxRQiX;&#bRfZTGD+f2<3^K{FbO7CD~v-^EWtVVDal`pP0g)FYeGpiy~%Uj+b z3Uh4w{ko*h@mx)KB)T&>j z81-DSDq---#?b=wFp%tVFZTqWrOB#6TuYpyC#(f4KLPMVvYUf>IG6|MS-~%wc0OYixC%DG+e^ZGM zC;#If6;wL8f=W>K1ZF{!I*`FP{_Osbe}sJL-?w~!(xx6!BAZ1D(@b-}Ui+9q1_ihdKBPyoSr6ho0 z??b-1!{H_)$IYVf|6N-60aDQ{S+Z7Apa-3hII?6lJB0{;f|&o2=-r8wFn>30Gz%&wz5m+_CK3@~>YyN~1 zcb?#U(j-k7&v&1crB~SSkKaulpuL-tylaZ?dQ`QUy9&dOm-~RV+WY6`$~^%6ohLN3P(!qrEU7)egU_!EZXsa(=R}mBMdUbCDr9usrJ5 zOoytmre6&Q=951A^VYj)QYsxRs9LcUkG3kG|B&(~1}jkk*dfccbr7W=GA{kv)jbO? zdGD_>rh08>K-QS6a->_wX3LeZEX_s$D?ri{b5rvE_CC>x*HPF4+Ciz#>NPj5{#x5U zYTlve1olCLHn;@Ol!2k%ptucAMG_a=A@a#*rsWr#*w2<}C`Im2+zepC?qoz{j< zT{s}QGTtG}oXxo=o1%m{D~yPgo{$6iQDL(AU#5$D+PurV*o$K~8}R>eF_+~iz z0MetJM6}i3ac;55o>!8|tGyKUQ6YqPpgPr;-am$fZql#mvFw;86q~j_PS6TJtV%Nm zAJW3iQ_U#3wcXT1QmvOqDGMyiFNdsgMkXBeOglp)X$VHU+(>Kl5+t=(du#r5 z+GCH$zSLTXa<$jP zGYUZ{Pk=55pm~U9C1I6Wc=`v0q=hB1bcgip$h-<~rvrfumgHBY+cb_VD+^;cn`pc% zero%~gnp9gEyAWJhcGoLjSR*V>U+^d@j(xz$Zm7N?o5#^3NBfs_gC>3J&h$>AJI@c zoDrSC3;9hbPZGeBL0yEyyf~_-Yf}(8dr;K7Uw{YyNmCe1%Efo}NbF#6RB9FX?_k2% zm;TUDHda1C3!ox==VjFj1Y4=Zwb${v>2Ut}E_bsFel4~Jh=VJHN5^42{9eQ7Dx-M% zi`f0Y^Bb%Qt5TRdPa)GOQU#{7|Na3prXHb#XQYeqt(TGvR=_1O88C0m#&M6g#^qaV zF-QJT2v$Gc0_9|WskRK!-IG_v{JL)sbk<&zaJ(ROWelT&lXCGuYvf~tDmP%e_S{cY z1<;dLd!Ge%dSZ`9zseqmm9uDWGF5fcJErhkCkl$L-$uPeI*{BKz*J;4+Q&C&ToNsm z!Q&EMN_)ySFl2*#xG^LyN@`0e7eDfW2$tKw_#I;f5Bh_(~|dCZq61>v=vcfh7X$*8qR zc{-}5Hg1OXM;^}?^9n0Fvla++x^LJ2kaC1hr^*7^Hj+tT%LwyG;|jGaASW8>V@aoIVbclJqMeB}_7O z5(YIkbgk9i`}wQ)fh1rcExzqt?ttg`2-E5@I29HYEOCStQ6cCZ4u?L+F9T|c=ji{v zR$~lx)h+k)rGB1*>XUC|^N5SUqh(XzdYTN9nXF$tlKq2cPilH~j+UJ}KS z_H(@&A$-RUDx-u!B_3rGnm|*(u!CadnLzhr?D5c7#m*+Bo8|`CIq8zxutbeAV;PoT z$|+-}FL<$^B>;KI1APrt1XGiBWN}vsc_^^^EZz|uj2IShl9URoyto)R>`;Wse#e== zMp%{h{mHpew#21goZkt2qqHUEXWGli@!fYEXkZaua^xY!gp)xU9H?A?JCU#h1Uf5O0d3 z)O)K-8w;XHL^FYdxvOKNr%VHhuQT8?Ber}f@JC$4g~73c!pF}9Z4|Nh9%Z#=_}WTO z%-*8&)b8hA5MW(k?^nK(kI5F^C#d|X#s!{)WEv19bWjEMG1_bzjvX)q@Rm~z7Lbnv znq!S5AJxyIrkxR-B*Sm;PYy}P`|Bi8#Ymg#uHI`;_JM`Gf*_th$N(0B#4WFC$g92l z+loFaxX|iyezn~K(FqDdpYzFtS6)T0+%|u4aozIbd zEZc^Ju&PhCCs>AX$wn752e%RU8BHUMBoI1C2?1Q3^1eRHPL-t7xK#U=_Tx8ZIWH`r z;SsJlv$$F}qCtG>ju~&Sa3@)YB+l&Mbg98_GEAu2E4(@NxSzO&uxhl{x@eWheD)U{ z2rzeS(xbvvUp)sR+#JZha1(Lu}miK?Srj+bN#-tg4IK* zbEV*aywU(B=RNCPI$aZ@cxDSmlU(x~&v&0B{;0N%cMAt*74OqdhFuRR<}wkSTHbfZ z9WRoFpWhM_c{VA*D-YR6Ycn5g%>3&xd{V!>iOydyeRVtM+-*wba&Yf~?8d;l=;7SS zVx)+Pp~p^~>OMFpfhzZY*Gc5P{`36L8?DM2B{wo@i#E3XCF%Y56Y|1+#=KuICG)fl zuei?@t{V71_E)}C(zUM$II*b;sa$xqqZ0HINy6rFgVex;}jz^ z?jAN!z2kql9M!n-o84tO@K(`QnzPkjb`*b=E2_rQ>qmpJL8dyrUk=WYME+p9@E6!8 zo`x+qd-3I(&UU=wokZQWWZ#1cy{#G39?t^=k=IY8*8bPx^7Z^hKcpbF-H+)7@8yUT z{T}1;){psyzR#I^dIjI)>InZeA&?g6xbgb| z6==Y07neU(hky1|A_8YQw>|A|J~+X-c}wk7T~al^8jJpSwfN&U$a%Y!PR%N|x1#iSYoDot*2!?sjfBH`E5-mynsy zW~F`9V8$pLjQIe33UKf0Rv+0N2xFe73UGQ)xJw*M*NF*h3yB@61@d*@qAuEs674%( zrnh06@e%Oz37p(|^?PMZ)wHSkV&SyW^_LO8--TznLBe90_`3AYFQcukDDlzKsGAd9 zQT(@KEcR0bKZ{4S#GYED=&jfZh9bI?i^Zn1}nS#mo$d$w~+SSZ_Q}P|1 zgB134*p)+M*~0Y!4l!1m!G@7T=pILJ8HE(6C(PSb^WfEqt`Y8p!K~eW!mU>l_#xD52iFS`%x;VN^N9w1)VnjM+d~E~4W&;?pBGQNe7`ug zjhUU4=~|4ey#pqcV; z{Vmo8LZjTkYpr3Gs`T5nB>gN~!8tWudME8#y60sSKMeO)j((#i%uhEdw9-9Z6QG)^ z51l`HMlISK>icZi8qkwCalSw`Od1YF8ZQ?39{eCd@lWb4Si7xNc68&}w-jC*5*3-< zsb;4ik~KkCkI6m&n%16b^V7GOd{1EVfUapX&+y`?nJ5bHa_3IEM_0mvkk_TYy%@rF zO!k(v1~L{-51cFB%UkT+n3cG^D2_Ooo+QEMI@#=g zpt8%v`_)4QzO%Iv zd3Dt@3GVY-%^9aFt8H)TCWuxIqbGknAP|#gKezYaX?7YiL5k0wm>VZqN7q&YNIBMx z@6Skh{oiU5iXLEuwCT@J?z}HGRZ4fzU`Mj>HabxX5 ziAOzkqSSRk`I_q>zL3{B;1M2VoU32{PBQ%D2MCs#he!R~lgPulStElmyQ<#Y$gNrL z@>M!fEPXf2?Au~^=Ly6MYmLgil_9v4BJ1}47F!wobg6o1 z+<9`In(ON$IhwMU2ir-cqubu7yIbC<^=ZFu*SgYLD~I%*?2u>n3-0!A1g#p;bXUWaE&#ooF@MuwAeaGs0!@Ui#L7SKxvVUD zvLgoxYKGi=T{_nDP5lSC94v#8QJ;P>?VRTYm8{fl#fkA^26OrwDecZK2a#0oYX2Bh z_uiUrV+^ZD)f_ZVPyPvc*o6(hKk-D7F)4WaebOe8|fS4yY&7T=$M z(|d+T$x%yx#fCdl5p<>&ZmI?)BYZ5wde1b6b3c3tj$SdVl>2m;Avj&mjnO~D!)##t zD>nm~u4B{@QN44E(gAUJ-MV6$Jo-Q;2cbNvqoHcxZp*yP-F!}v?zR7Fb7QgP=+n>j zsgZTaC+i9oN4Ka$$|y$=H73chIqpY(Q-LeKX5>qLqY>pSpMl3Nm#wROv%@a(fvV-l z<5ixuDC9WG|8wxdX%{a1GZp-zP4jm15YaghRYZx;U1#f^mi&^8BmpG}(bHcYR8yK@ zc7@1@3}+&KEAe~Xg28!}Gx)>ntEuHgx0C`!&r=!$pK8zuO8jDI7Dd^XXCkfZe@uCe z{K{a@7{AER{MKwHIy$v4U2rEauRq<^wa?WZD&!$IZr)P*Tx-4e)xhI2m)yEZ znMxe37iEo4|7dh$%zQb*bcNSxmboKia=Rxp+V>dIv@?$4D2YCw?z)O%ipTl;UO^=_ zWukr{6J{)?P?Xuj;b01|A4M&-@;k-XJ}eMdymOdZF~{n}YO;c>793P!By4mk<~G^Y zGma-^rVCiZ_<>laVPD9JGdI74S#Pz-O>DtniKI~yXPpeM#lb8y7R2{1Y9^Uivt&GjPMLzb7e-m9@Dy`eiWA-G-^v7h46?j-szL5i|>vq|FpTk zpBnZ6&IKTK3G9!YE6C);?}`RV?qUC5Iqo;R46Q{^xzzLW1K7lmx`kl~E;$L_F`;c( zrE6SUyR;t8kAKIR1e>spBA8%YP1B8uHhum6vS2;GtW_no1WqqZzg^Vh?*iicq!4V< zq@(e&p*HR}REp?~ksRq-xQ3Cq0}f}Fl>Ttuz3M6MMUwx41C7XGY1N!R|EiTWARVm8 zj@=U7y*Yg?=rz}oLpJ_|w*F)LB5#M7MZL5yfJNmSXf5DfDM0cD9m<6fuY10ka-Em{YW2>V9gIp zY%;BxdFq|vk#+cs+B!}?oVBA?% z?TJM8iEREjpUtX}6?5=E5c4M{H@_dm>F&+683X)D2iQ0WrDq`TzH?A!g$+N>IMWJ~ zI8YDko!O}wa|N*5vo*cnMgPf;_}&Md0YTl*rwPdDhlg*dUB8NZ7yHhhI2`dWEd4Ec zOwYvXBzMzB4uRxm3Nt0N+e)$R&larLsT$TW@@z+(U%sEG!6fi=DoNGZV8h)vP4>+Q zEl$6s$G5!oH@^%`rn?n+l}we!{N^4sK|Gvoga_|MdnP~Gd%hIwDs`==W5SteXF`2@km@eq;1^b#%~<|9_2e8 zRDx&MB4e)a?wbe(HZscmn!U;Ah@!+VU%QEHdnkK()(h9#8!o47%&z~dz$&5N}X^o9qhG(|U*nB^?P0tp5q$L7X%|Aw&MkJA^biw^06+i=62vq_1I zVY{C642@ji?%;;L{o>KIkpBd(OkDH%h@6i5kRNimqMq+4U@M1woM3 z^P9|qRQa$_45@ukGCn{AKx3+ddmz51z!k&FqQnW4(;xjEumI9PqBFr4zeJA?B`(*h zM7GD@PZ}hpZ=Q^)AX{eb^Ec=za>QzE?(wvsih3#5>1y*UC<4r#N z^^UBM6jnJw{X{+*WP}Jt!o=v6@ze??;)k!sxyAw?zqKkSQ5`IMAMB!Q;r{Vpti07K zM%mkRQBZ1a3M8j99OXZ7?vtgQPwur=T^A|MmZ8%3MirhMyrp+J+HUeB(i0-^8YVG6 zJLnoywTj`^r&6Z&3H@42ZCh}Ir?o_ip2U4~RgX6Mt zqFPI$^IHuaX}|t0FqYmb40fE)odU&J z>6lkCFVja$=N*#%EUua@F?gvKNnD#V0#WM&E2un8}Nxp?eItQlQpseLH4lTqKAy|6z5Y^2xDz^X^EG z@X2?jDn!Yq@@a?vz!!FFyRErazxXRvm3n*(=G4kyoc2bXCDH_;v(1(Un>J=YRS{#olhG~!iUS|mfR-R3=y z`=NZw{8ZL?+PmeBc}vUXdj8CE&1?HZ8TwVquxm4GV)?+u*L*3On0bRtK?xPDGw9t$n+P5t^NypMX*j&#k> z$Worr(+K0yJ7l4?{v6)}E4J-g%4|J0i@G(SFJBMp>lXb${r|S%p348HSSBXdTSW0u zNYbu!Vou#&e3!e$4gv} z16$WY{>>AvGjcRrN3H&bimV_@psDM~JEi|Hx!_y(Vg0})DdyH((xER$K|{}PcEKa8 z0pYUdzP`rgPtMKHJ{c7p5jE{>2M3f_jK^jq@BMP_z#a5%QxUN`RTi>kxxHdQSQ2cS zH97H?CXp1>yS{YhZwbwJHrW(j+U17C`Q3TA`f^gQ6ciw*N#bz@!rnBVpE)Bfw_H>8 zwxQg&w51&v<=PgYe9*6)>$AEXC6=d|%wM-vtxpgCd{*l`NZC_L7CHbTJ7^+)JfcjD zrVp)zdi|1D&E`3-gRQ`~YaofUt5Qd%jrh-SWT=W|zj}ym~6Z1+~ zrWAk1TyDKIaNDlo!PEU2?^I#odHUL|J!fZsbgRyLJxJo>BsK5s(Q}P_<*{)~e@j%R ze;=xThC>d$cXT@S^mBy0)}S*&#$K?6oEFN|sd z29dM=QnvmrJ6qULlz<5PZ;Qt4_Qd%RB0U@;S2NgRg2;FtIveQq8lkl4=@6|Ig=jD< z3?}PBs7KLx{p{XI-55N@fw~UwCHd zhT14WB|4+!si%{Rm&32AC&i|EQp6s4w{F!^P`1k}L=@wbhbA52kD3_#Eo2{4GmXb%9DuqVDZ%03U-OB1Sg`z5nz| z25&<7v<43O<{Vs8{$kg@+q7v)qqbaR|29{v$?k~m$S-0JJ*d&)VoSa(#en-i#b@;? z*D17s`2PrA;$OkP`ll_l`UiF6Edg?+Zt=Y@vGK=Bt`{4BmLtdLW^LxzzP#n$GA>zz z9vt41uZu(+a&NKOMR z5A_g0*AZ&7q=>vmCzNMIf19}os4eyW4wM)Itp64-qW-IRaTnC-6SBIxk)2olK^h90 z*OL0@ccVJqE=GlL-g=71YU}c+Fh-x@CSX$if%dczr(9--Da4rg3sY{#xu<4fSsUOK7ig-PipY z2ygsrk>mbQR0VX75C&+`fPsLeXZ8Oj0DV`2y0|Tv6*PdcxOB3stJeD``^NwOp6E&g zAaCX;^njLV+Fz6UDunU#%A|;ZNl^uu>tH_wE23F%yZr`u#6E^-JI0N7V>$5Znr{mE z1u}T-pI@4Cq9=jF%q311uulBH`sos&V@j!HWh|rgwF`2oFh<@CXOla6l((TSxF_g> zYj(c{9U^Hl#3CRf0G(HUIH3*8#ROfPkii$>N?>2J{XIW*fr> zc=`@6D4E9%fuOWw_^xxyf_p~py##+oSP%}nmVHN#0+?1ZZ7YweG{t#&Fxy#PMI(9V zD{Ll=k?)Sb73uPy@0KH*cgj-S*K_-8qws)nqlo|IELb&vB~H2QJ?BUfMZq++4_iY& zIPyXKu6u-luax3!LMn)^A<2M=!95hinA{T$6ocpr@gwG6lE>_yE)T6onH!EWiFsMA z8>Es4gaN*gL z<=FVjRG>i5_poRD>JVKia=bk>G5PNsAiB~WZ6NoL#16iZH<8*{fPL@>YQSDGmVN*} zlvxfvSxcuZb@7JqhA<)(aM92YFl;4sVi<(?qw|$E|oHe8+(~d3$BRA6VZ9jHe z$7bsmzQctU8PEC6>r6Rn`GKgDVSZh|D6TB64+qeWj;8DKNriUEN*(wJfC z^()`NgD)>1ixTyB#o-1D%7AvSt^$`!Wf%pRjC)5N@Q^0QH!qEOxwff7NEx$TR4y-s zxdp~>?V>R(&Pmm^<#N(5f1NTh0(3;RX~CRop3Q5Y$~%Fh2BYVFUwYU6s%P zndB!R{_w*SW8W(5Am7L*WL%#DBz+*>WTobVZk6Gh5qj$rt>wL;LkZO4U5#9Li!6Dj zj7o8?t~mdc^ZEPqS|yIxN5}J7!DiNrDy3w}s-;PKPiz}`${&##vOJRj`ThId9I`i! z*D0=s*Y02f2G47RRiIO336+&Q337>Meb7VpwWQ0xE=M|tOxFC=NFf(9>jF=^&|h!o z&8rRSkJQCfk^d=-KsB@5Y;BKX=YjqLiu3r6KR{BT5$u{TH@;JFi6MJi7xmT2m)QkB znhIVPa-R+qK*=%AXruZSS;tfk4DU~pmIDW3Lg)~$vz9x{Xu|qZm8BbxfFT6lr$#H) zi%ZG_F^dTAq@wxv^n<8IfL(D(BWZ#SMM*{?s5@rGO@yQ3m}kaIz)%QBh0;WTSUMP^ zqdh|G3^^x=xnlzOWJsn(AIkiy8g@w@W< zCzL{riX=cM6vTp@oW(z!j1;9byW)qha=g#O_7}cIR7~AfLcNVP{GUtI(g=YpF_|H9 zJ0A<-+f(|&V7@u+WIdXIhi4dMt2oV&*Cl1s@BU}HvNH~NW{)yzh?tT<@~-3j7Gv=r zSF@mZ#P2$IjHE|$`$|A(zm_70EYC{mJ11BaW80!*EsP7N;?hbXvjOsxx7nbe_p2~i zr6Epd`L?+g)-qabN^V)NI-tykhTnAnyZTGA0WkI=QqDIx)|UR1XL=*d9q8Wwi${d0 zNxLS`wccZH2Ng*(oG+b~%wFJ2!U9rg1T{jM0|3p^JG*8$4rs;? zZvU;g??DA`u)3TuutS$pp#>0#egl|6TE5>tLyYZ!b{AbUv)B9X#LAXxblt05N#05H%jW@ zNS|M<0!jl+Q^(y)`aJRk$gwhakQ3SpT4Bse%AI8ZTpWQMf7h2hyT4^}$?B~C8vr?Z zENC|d3Y?G%nE#tPyR_pmy;eVrj((7N?cGhF_>y2q7(@vbzzg|& zcX0e>H#&=el737!H0bc?DISN;3+{i8I{pc87hihoua*m@fzQq+?lAQiUW15V|D(aK z9?T`Zk$F(?#^<^QsXquPe~6c8zI1~x7zmZ2ULN57~yCaWP-WnjX$+VpeCOdlkM1xAEc#NOZzAK701?QgjVdf|^j#M*_>Hr)epujuuAg)8@g zhJg?Ka3v%3jdT)NhRY;Y|2@DUd;aLL>^n4Qnz{lz`S0@!ArM`((hUb&1*E-0iKlqe zWeO16Y>!kW#2>J;;1+X#N%6e^G~HP=J(R3)k8yE_;h-NFJ|V+>08Zf9)#BH$kM}CBkcHNafpJ*_x^Rv+V1az`&}~U9 zF=#Ltv6g_ic;)5yYE5luz90%v+%!+*zdPQBs8Wv4D+*1(^mSr*INm>7B_#smo&RuV zu_In}jJE?<`H^yw>2ykLx7o)rAX zgzL~dRx=S?FZcP2Q>l&p-;_k15lmt~)5H+DdKrT82+0m;Fq>an3%yLL6QRg0CWL(%vfkan5C>PgW< zxJST2k)8{*LkS)j=EppILRsm7vDx|^J^~F2fl1?_yC#eb&zxD`7Zai%FnTi|fMN0Q zbK>uR{aj}>Siwzle88Oy7{3Z>zj0;32Xf5tWMIhz17?a~{O>=8G4Se*zKIn~G`Uo<23E4s$NqKaVIJAVf6XW0=;F@>NSrcT%_ zk!>aLYW|mfx0T=XE42EZC$L3OmbT_A@1EJT8r*W7L8yNB>1Mkyi_lv|^ugN_Z zB+z;cM)~Y?vIjxX{G&#WT=H=4 z5?mZ{up>Xy3M-S@h3WrK8R& zlsNwp0^g-?@V`&ffrlL=%|Jl^1LidUr2@n%-uv<}Xz0ELe3S6NBxnt~RbhL^2#$mK zJz8fipCsiSV${yN!1XNwtB1T6i0D-RHKOo6+U#AlT7>>*nqUCt?#)tzhx}efnUH{K zIa1dX0u4Mdt-D^t`=qLwzF<;bcURhr#+9)jx9QU3Je}?oq)eIeCDPllG(}wWu&-XH z6S|`AKSgp3bMV@jSy)UbyT)z-b=r7yb%TWpWHWseLQfd+&_7lozF7%jPQA5zcwZ48 z2h3^=tKO|i(bk`Y*S)Dp1~;kPdDAAuY_Mx(7V|>;Cf7+$0?-yyj;Q~7(&Qwd!Fz>|{U?X&8yc*4MS;t=(!<6d0! zdyh3UCF3>l{Qb`!o&Vlc(#eri<(jwNMRgd1NcMpf!%$S{8HsF5ZbR*q8g1Ry^9c$) z%m5Zpmn*Vy{rCA`g&bEQDenCAoA+UAPGoFiG`}D%w|TR4uj2G;cI5mWJh-2Z@nK7o z)k1EP3W|7%9FtgXX%#Rk*Q5^P43g^;x)`-ULLfDmkZBwd7)?qB?^ecuM63xeT)FS( z*l^Mp{?T0VzB-38;4do|r>hxeXXqSlDUhQU!Eo^bk=&o0_MmO4IYl;f@7Lcpu9c12 z(9T$Y#8U~+iNnT;(MuesSlMZHZ}HSE)vZ&BSS+If!!a_kjmB-20q;wX(J>@3`M_<&8g-!x%3&y+BiPrMJhw)k_H$oPfu59r9li~Lf#=gsB3JjA zJb!PhhmS4kTN{kp)E*7RRv76v>5@3H@qsAVj&@+y;GV{fKPoEig zniSQ(G|_O!Twbt?7*?p1n;m7+dA5Egxq0hZJ<`4JQ=CfxBszG?|`KzzDp|W#O$!%_7ZgORy zRFnDk_V|>0$zD63O1UW%m1|`8-z>R?d#TtnRdz@@TVvbO@JvSNb10O)((@>RP`)Xb1)ZZ#5f*rG^ z+7|t9-SUKskJGj-fl41u=`}kfAi)@XxwRL7GvNW57>lp51mdT;OmhtV89y} zHRR+MBTG^P-Q@}g_mI^ zT`Sl+*lHR4yY+4U&cooaW5`IHNY-ST_XEJr+2nG!^Cx z=nfriZfe=b41ccnOvix5mLjRvn{vGKWUFb}k7~dB&cM8-oxb}aLdSDIfn=+dK7rJw z>b4X|5o|&=Q}Lx?yx);p%AQ}k`z|*#*SH3%C3W1ZEH4Qen&E%;Zf~og5o4|NXMCee z(oaKm9S>#87E)Y8iI-J-U25wcH*ySHV=oTP=1I`}k6~VyDwgfkqZ0lK`a;o(Q#sqM zMyY3|dwrgxL206K0t98{F#?lJqfb*O4F#<8_K>{mVdgoD-4oL`s`PqBq?Rpqpd_5@ znRwS`H&a3}Q;gB~$SA$MNp!T{KG$z0bzVE%8)293N5O@7pC7r^E|zQOo10krXNvFn ziB$o=S@d52>gWZtj*8^ozg&PEe{=clvLta|J*zPVRBhO}@8QfP3PjdgEPyVk!37^? zlv1+}?*=?nk|bNz@6X{Ph%499FPI9AWx7u4YfybauATn(aHv9?c8Y)3b_~v7CGYeJz7}3CuMrpG*_62`{AIQyZo+uF+I`; zA#PS4V&wM#-6kp@ie)@8+lzb!SAA`j4XNaYn1X}+|O;D!#>r_ z@!#ibAK#c9P~8&)G!r+hS%O3P^vf~+o{B{2Rb{AJoDeHgHkazK@6#iE8WT?tYWHs&jX~fAd=JVYl3D3pM{Ficw_tgkCl@rFD z)7^nF1tl)4p|(4%zDHe2DWU?GfksINHtNc$;>~%x?%U&qKC4gNSYeudWHjzdOwr}C zvpG9{!y`B7N)KgQRwEEIDaGq;%(sS5QcDa+rySSQc&>&MlVqcpM5+z#Gux`;av|?|O<_ZwLb)JfW!fM1 zu$pG*JoHXIxro~0@y!eot}ZA=UTC>TAg8>xkA&$azOIU87M`{Wq^=GQpeQhH2T7~+ zj#s?z?;2Y6wz#&($>iF4r0|=wf0@|XWzYR)jy#=nay0> zFQn$E8*$DR4g%v_s)QV}3AbC5P6u7ArA(F%ToKrw+ep@kzOQ`cIfk%|sBer*0JKp1^ zgVz6=<4WlI_P1-i`a{@679z~Jc7MFlA5^9@*gDU4yv5P~Eyyq^I^oy#?&eTz{(Y(K zKhEXDT$6LLw3ia|7unaMMAfKm8e6pFIPR7f7Ls!$g9FbL*W@|#U>euN*IFyJ z-nh={ZQdN0L+Vztv%8|UejK;qS6fqqh_d_Cp1qS`(E91Zbz?cd;yvM5mg)TI1kY6B zyr#}7g=OS{Yq{TE#h%sDw`8_Pt`_wUYtF6hyZw*8)c6QgaVl;G(4tP0)sVH$=R=8Q zLLQ4JnRQLdDXre;LoEZE&lbohoq{nVjYHLbo31og*_D2F*AhMBbk%m?IUs)Sy<1D zH$q~`Vr%kqyua6LPu;w-ksIBos7(k7BPTTg`UdvS;y{cOD6zshl($7>q0LV6B@k8^h8Y zTPnRQYt7hxcqux7EYm;O*BLTQbiVuP4g{22h*ZASDn#@TY_$%FpF9!Lca6)S`}5=) zTX9Kcl~>uUMsYc!QFuTkO4!As`~X1kjKST`Sbb-8xK#e@*P73QN?QFAwl_4$9g&s_ zmiea}8w$s@y0m$py(hh^7H~#!MJCT}1QI!gzo^>IuD6^C{H*A)OV{8`V}w^fSLywR$jr(H8zLVK<+tkb7Bc6x4cH ztz|CLA8BlCO_xi#OypUY7P6N&)w0N4?trnk^J}g$EDOFC(VR^8>gM;B9z!g8e1q+l zulKz+F1(RhGM4_eT&50{16~5e-3Beog8Q3vM^r+*y5JvD;8@^_Fq`B$j5sz}U;t+B zh2(Niy*c#Oh*zx-01wTi5hto5P&9`2d7sr?uIyP*@h;5+6 z;r-^$e6WeHt+_Yc3yT*`i7eO%uBRT0^2QlhmLE3OJ34xLi}jXAN3XP(4{^E<15kZS?EA%9n%xW{^EbTK2y3 zPsEGh#ht~n3)_XAKD6w#YYmL`aGz1RKGx?uf2!NMib7eyt1uo8 zXe?wfelB+d>9`@ati$8B4D)YtY18g3a`lvn>I}o2=t|$3#Ypxoh@wpeK%uWf=7K4# zPp5L|;Kb|2`Q1MO{oLxA5n;vR!fxN|bnIpu$Brh4#7n`522c4A^JQl9DOy}s@8ZLY z&_^$SKEwP9|7s3TxlM!{Nks_8~djhl` zAjVIQHB>DEA`@p54F>#Lc1b>d!Ds2U`h)h%BmX)Irz&k*Ej^;x()yd1_4@Ejf_+Vs zhAPCbh2}r@rL$*xmR|;}0nTN&$M z#y%ktSsGc#n(WKi#y;l0rgP5se7^Vhd;cEyKllB|aVos$x?bz^^?E(8>upe<>bF*B z0sXpABVN5ACn9^H4S$)zU_=ZoLfzz)gGEa!LsL0o@>c- zD7>)f7?htCacZ$ZeN05-LfMH0p-v-St}&HCO_dW;s=$|8WOkuactOab%!SoZ zD5*k)()kVVQawG0oR2}nN8SxhA!X6S*GH>KzE^*RlPd>>LA8qJz0U^$?#5#%X(ocahB-R1&j-3!_iA3 zolkA+$;FN(Ez=lJ1lhQzoa-*Cy6 zyi601#mm03!hK)yhN=E&i}DGJ&7&4QA9^-w3;ikPNHbUSd-s+gHJ5G42^?(=B4YI1 z0%LgndViQvAeS;sFf?zabIY&wgeW0%S@*V}6JFOnR2PxRyHWQWP}1j}@IB^wCX*qL z9qGzc>X25 z!8dMkI8py#pMNy+xtce+#2%7?r$blB7Xr1Le(NyC1P>uA4{u(S`z(LzZajLr&NrL6 zXj;MCX>{J@{(hkvqk+47Jg*puPYK_nN;t+dNu1D$=9D{W-#C%Y>5*so=Go>kaO@iP z>G`|~l?n5K2Wj`r#MD-{H9Pwq)+b|9sA@_2J(sZYXOno&g*!cgTnNZ<#eE0&u?mPd zp^4g#f8aLC>)vH~0Y{m%ohde(bNP7{p3`;=HIZ6Plq-`zWnttphcBNI4)MOM#$KzZ z<>u&3+1CQcD9`%3kY@IOuQa=UxjFrl(?V)udp1>m+@~#S??_&8Qd&X_tc11J2X3*=!Q9eHOE6 z|JrBqW$@9VR_SUcjN$$(ZeQZLLHG>s%cj!?Kcd*B)S#x*=B##~R&x??r?n$Cj{Xc) zda1OMmp{h3coDn#_+Lvdg!l6Y*E8IA$@RJn3#)=}*3)3iqpSUE63-M((rLRIhSG_x?PxRk6rbhS z)oa201_3NZIhkfDaZ>%|*xnIi8>g%5MG|b(u*fBAb#%NjpU>(`*ix;^5)XF#zHOb? zl4NrqC64g2gcb)K{CTqI4*Qw6_|k4W0=6$c z4UMOY81eO2;hGx>X97J+Za=+PB|m^YqHxC3U`3%*OQId#pED*vb{Fdv&KBntyCH}y zEzQbqJyWNn7flZOX*Q*kiDfInH7+i$?OUSrv32@$7L1K2j1=#^v7aQ;wQ&`C3bs6( zN7}qEc`IBt6Xtm{j$ig7h? z6X{0P-tV}9BpTQc3b0;NgQ zJ@A^}A!XRt3kQas9`Uy+C*Zl!sR!EB-Go|_Hj-Vf`DS-NYk7!%$B6nv3Zi$JTd<#M zsR5F|f`+_wnhrj8_WnRFBAERq4GGF4i{9!iy}lqjf63E#HGloi*w}`qQ;&BRe*K$E zWi0kXTTuC!?wS6DXSkmw;p_7!X5-N=#Kk3|`i=ngi3LNN)S|~MIXNq4+gze&!oqfY zq_PLuwx^n8*)eq7K{v5r3DY)YkF;9hRK?pL30-f)n-Tm&d0p|Y;00|q^}p7pHAf@j zU+Km8u3*+bD{uGq>~1uw`Pi=OHAbW^!vxPJR#|gcWINuOpNp=Qpj0Qsu4yDa))V~( zwP<56Oeix?pXIU}_%(qOx@WU3RX2%$6x*i?UwEtJjm8AEJFu4Z!)N+0e(icc+ntMy zNZFl9DKSSN@!LZ#WfNQ>$UXm58C@;=x{%GO`ssa_xL)|TYunhOXUe_c5Gk7MPRuM% zuYKvfAQSw@OJzuBf!>POBbehZ7+7p{ zz7g%aZ=$uimc23D z7VaT`NM%oYM33;E^QD%Q%|DoZd3??3a@lVFSxAN7mdHu%dp@}0ju5?=sd?dAkF>JB zIB-ZsTo7B3z=hiGs}syRZT&LW#H2PJQrc={wX-X}9Xwy{c#bTCU+!S%i%UInM#7GG z$pMiao;a@zJ65+}Qr^|{Zg}fDIYCmJ@bRl+)$IKRG27_`C1hm|F7z<}gNs*uNTvIO zrNqnOcFpC7kS)AAJL|=9wY}?dS2s-xcjz6fND91H`0-1lw*(W)#7!dJ+C$e4x)b_mMdM*1Vg^y~e^$QYI6Rh1BZy zBE)i+EhyhUO0m`n!OyJN(sVU64xg4^IW4MbI_DX!rJNu|?LlTG7THw`wmCot2JTLZ zLHz7x(|8X%B{D$WscF5@Id`Lt>#l89-aMIrtt?wMbwf;ALi5#(VHKLK9a-?rWkVyZ zRM)MXs89{AvkXP|?sqzrB%>cKWnxjY8;g+nRnlf-@-JOPlP{(qw8$7yq0_#!(nej6 zSa0_Zags0VB(jvmyC+P|icnUNWfaU#Fyb^cBEbPtzL+s(m00ffOP26r4RX(3MC7fZ zvzKio!~EMXSGfdQSfaH4${K>E)q}X$GUqn&UFl;~DuSy&C%1I{{82i0XF|Y_<0ITt zc1UK-htu!U{I*q%urred*wrFyr>cj@q8goflJtPJN$r(6HzSF$(Lom?UY9(ksoeWW zye`XuO(CnCjo7g@-pFp3@XjR+PERxsE2Czs@25#-GGVyMdwJ2ez!wpDAQ8=;e_`%)OjgL!~tU)?lBa!o|E@mPJG$wf1e9c;BvDyz?r!duytryr5TyGSkJPkd4P( z5J!WFheg*1#wvzdMXMc0)OJ?Ttqvoo2aaUePYUJ&{Gz2*eL;+$)9)_T!z?IW-X`3ulVGs^2(nm!5NXOmI*w>v`mU1ND?g^nGb#_euD} zBmt_PuZTmzC&Db@_sY!)KKvUQ_qOs;lMvlUMr7;~5!kZuY>rCid7BYpNK_272%hS9%G_~dtr_shC&?M!=3&#w|4`^KwDIct$D zHNCkha47KAg-CV2ZsQBT+2i!hz6p_Ad2zPjWEMPolWk{at<#Mh;BQ;xfD`r9{3DD} zG1ONlXL&%{Uhbj@LpUr#cn;_V=R4Pakfi&;1peyr+6HN+VWS$&>1Y4(;6ZwVVAqIU ze(ZrZl8iD8C9EtZe`!;#UT~vd1>*3AI4C{0BwOx-bL@FMIou74CKE|`n{Rj6xwsP64_|IAv~p(KZdhDS!30F|ul%8N~)Bz~`wLS3e zCYk((AFUv*X{i#x4;{L3+#+Nx)3u7hkR4-4-D&P^aHVdidob~jg(_9{=<71*`>io! zhEBDpl|N~e-z2nUj*58=iB6}Wd9RP==UA1Uv=|s))p9J?tru9^KlGK|UvsVf5o%2R zjWTSgvxpfQEPdR<(z$E)Sr1(ym0!dbf~P~;YMlv*sVcazDa3C zyH54=op?Op-zo3f5kqNH$mx98Y%^257cQyj`|E0v?_{BsA|9TbKLY9m)#&J+ce85v zcmCsF1Nouv`Ler3$57Gn^pqD^E|mHchEaG6@XS*b!u5?e7K$maW6Mz2s+G`3oi#t3 zMCiH%Ol}78w|!c%zhmx4S(FwnC!4I3aidrTQr4l3zDta}`fF=xs1+dxmt_ICb-eES z_lV%O1nO$C(@)EAQHX-IwYeMU%P&w6yq?9OQQz_2KI>iH<&UR3EJ+R_og`!mekAX+MYvh(rq z@DrrjuJlTaA`H5_z^b{$p;Y;|1*dUBM|L!dw^2fQA#5A8+g{}4O90mV0 zUTjfb!O;1m7FQu0A4QZvuyY47EHJ(>NC z{u-2KyU+cPb}anPceTX=XBK7$&y1(YcM=u<#-Uyg9Hs`5Vs zt9Z#R#!WP(X>$z3p1q4Hjuv7`#@kT_~7qs^p_zO`@!$$&DBAZfayn_ z6$?;I_k|Xg>BJDiq4TUx+wmd$1a)xPuPHLaMdem7$DN3Pd1Wr;TXs3{bFvKpci8_U zoW9`v^E2vYluYO;HCs8c!pJy?H*m+~LM z4sZbDb$Ty21lPx$L)URVWI0IBp_G0*fInO#5GVxXt)>jk6x*ZxEt+k22<-G6cap)c z530S{2ZEoY39?EcTZQR{pi=bY_Xk}Zi~t6lIddZEIiTSlz0*Jw;&qG5%N(?)Dgd4C z3|$O4K(`3eW4Y58-T}@#4eZYbczIEg;KNwXkI(Zd!`@2-R2@}15Xc0ce(K;kUGPER z+CdsjkFkPh>ji8lj+A6ugWcH&GU2#T9030aBSL%ij?G-QJtN1&-$e^>Ur}q4^q6x> z(ed#yIca3DkDg4m(_j_+et|cFMM&!&4Fy|_CCc|_L}!gPq&4UI(ydod(Q`z9>AjV6 zkdEn#5HNgBJ^ZJWfp);G@*e&=OWVRtx_@jiNh0;Bd=Ko2@A+9<4fzQa6Zu?#(x(H; zT%~je>3<#j*RkS5A7-(G)xyB4$|CL8DNhh)>p4GPDCyE^JBjW!Um~o}4sJb{j>Nl@WgXI5?pPY@lEzT?%aR#=$;3C|Zi2x&&Sj z0G{IpsW@Zd`?TTajyX7TwizWTIX?t$nNnu~i!x`((N4`jDORa3(IKD61#Ub4s_;Hn zAR64y?H>(J{oMnW{G6o%U`>8##_@0~!2l(9a1S0hpDIa{z4XIX5(h!=Zn7T|HB1n68O-`bY5pHxS6lp zTF`_bhj7gIqGChheEIJbH+QW2a-~DwgJ~dph5~^Yqv4_VV$;Bob+( z@8(H5rZ>lFzV;f>)%iKXXdiIYqUq}DY+-3yR())!=3L5BAwiaYh{M?znNjrH>1x>( z+(CkSY+Mgkd;01fNl+ zABDfQKMfyEP*T@8B*bdF9l`~?NhJCWKXUNCBJ!6M&IZ_&+)-HxsvyuGu+X9+yhu-;rI%_Nw2Hu9l^+KX$Ronl< zkmmA2`YeEv&y{qrT!%~c96~p)dMA&cgBE?*t`hxJ$z^c6^t;r_`h?>W;c8jBpM%gQ zAi=*8$N%);i|9#os-3M(=@P>p&vE-Xt9OQ$Lq6(Uph+h}={5qQbqgAlZ}T+_4noER zoyHN2eapguExuGwteYD+>lw2o*!B1zB1Eclisxyu_apVdunYHSssF{c-?&|dB;736 zDf#gNSC1=SM}VMTX!S6GICg?gV*svSScH$&RfaBvzVg74E0^sqEsr}d(W2M!%HA8w18|(oSYC{CbIsbwywTkj$ z3sqP}!b6!$bd>W(=Vu>l9Eu~@48Bw%fLI-S{mHDYxMEATwa6?p9n(+Y&wcp8npY%Y zkdf7AUH?>OCIE%_y-8S*f^BMY=M7O^fB}D@vlo4R>dl-T?bDMgDo!7y*EtV<*Ev|@ z@8yCNs`)3S0bFoL5Y=Zy_^2lL!kRxC1c7{!$SPT1?FQ*Di&w0=5g7nT6Is%ZWzSLwQ^hYdz%NzW1EhhKmx zdH_wbhuTljBnv(XNH&MIz}bJQGa9OL`p52<%~fdn2w=jzrT;(winub&Q^gJhVEQMV zrbx5=n#Uud2NYYb%7aa!>3k><>D7O+jmV#*b!FIJVFp|fkp)l&ROryV&yFB{+@;g7 zy++4${&qkSz?iT8wk{wGN1s3CN!5e&%&f)p@}jn9ZqQs5>?ZSRR>BVbmU1B|HC2IW zhJTHL&mA}2@0p%TJomS*0iG06R{T&9G6G%^kK=l#20NyvIC!1yjqCHQa3OLO*3M3a zzZ$sE0I&B+R0sPrvDdfUiJ$k{minu^T!1FHDvFc!FATG^obUzW`cbgh)G3vk#sFlqLSMkdI0svZcmf-IB8 zhiKh|HJ9x5#xSk4$QGPcVs&<;k)f^8Ga^8BjtfAQ+Yf0EcwrP!5K?k{E#Nv3Q{*fF zbOxMb^ClHBG;lhtIVfQAMkF4n@p&@AW$6 z0)c?gauL;p3RRV-W5r2y8%5t3KdSr%i zwA;*&u%DbZAlgeuIaH6TP^0Tkz4JZ{#r!IbL|X;rFJu0}j{jM|NP=01VvAIFiZ#Gp zYYZDCmjE?6`CGweSTv2oeZEHcFnV{dbHz&*xvq@hycWd8z9~#5)-6*S!l6W_v5Be;!_?TTC0cH*a@skFL;5q~9aU+cmm-=#~n()RdP`C$q?lCgoALsBYYMS%KjZ~@ZxW6tVG0jGqWG5Cyk5ro^`6}3ED-bSM_pC6U z`*p66(V)RdSisyD4*UEdnirlshU014Ye4mW1^Y={^tuak*{YL#aP^_K@p46`ZT z+0&g+#q#SnuAdy->PxBqO;?O9RP9UHN2xBc$*Ru>2D@ZL8ugB-!rDWe>rtrDi=s6; z;4RwdVe`duj_-Jp14K&39w~E9uUHX(Ki;@TpW9b`DKiC@Sm!0~!S&YPK=Hc#qq%;e z&6t;@DWB0g^aHET<}CFct=^8Y^2T;TP+R%psZDDTh?)ON7nXsAu?v09(SPKFfHZ>U z;``6=;a)3G3e^V-(bO!e2w8BX+mnA2kW%pls=yr3`N-sIhkV zey8I>tUNxGzqp~~7yp*w5!brU{)hE-+wCg%Y7?C{1jIU3-k<8bt9cv_OBu*3klJXA zFOk5WRlGiJmPq!BQh!D*Z%cLQp1K)qnj9XS|8Zw>(_)a1&{gYFA6KAB+&W>4-9FEQ zzsj!J=umXOO^S4k(1Fw9wJKbw{>??`Z-%rlden9#RypeItf={Y0h96=F0umj@3fSH zejlStdhvh_L~JGy+OP5^T`F#ZBWA)ozs~lEhti*&+8Uxr8yh(Q!pM9OcCYc>S3#)tFX6Se31^1Vx7)2mwCQ*;;Hx0{(| zFxBe2zF3_--O({>mVxL?4Y9N*4y%MsD#Dgtiy(yDciFbRMLs5(*YA-Wsz;)0*YG5H zeYj`*9*J_Dn6c_t0Xt)%IxLPSap^32YKiqjjEjr~YkhS01?SHQyO9zhRo^{+PZE%Q zClg9mj9o=%+mo|FW|AoqcX(PS*`5D`a`RKJa8N$#CrjHGPKywy(|?*jOr9WtSe%p8 z0*vHpn{^(i`Q9+l$wPuS+*&OIQY5g>m0@bb^lD7e@EJ`7JBL7jl@&2vo-NsTmAIyWx{PD(#J@U}Vd{|S7;Ri7Nj*xc$**0-Q(d30 zFOP4J5>Fdor3$yItfPi1@4Lq%U8IzI%SN+gcYk+~Fq}4AYz*S_Yf_>_=;8$8y&kHf z##e4e?^4UF4<;mHZ@vFC+(zJ6O-fs5@>rg9b)({y(S7P#ViqI&x5U!5npe0N*FCOJ zrD=ooeg1z#L6M3Jp2Plqhh!G1+C7ogj2i~&p+G47V+;m-k*ComfhLdq-{pOJl-s&n zL`6?-;;H-I!asYBokSyi1QJc9WLr>EuCv1#H{E4%c9jlDo}dfbEor5aolgj(h`Gyp z;ky*6$$tg3C!?XAl zI{j+^b1tlKH|)nBW?WyJ-V|GR5qWTaW#}f{ec&LO{mh%9jDoI`-s#)Ecv>qAyHQ$x zWzImumzIs5gVcWXiH>eN-Sqyv;jk;&-}MVKc$=%>Mp3041E}R_IsZX-Zv(lPkkNeR zG9qTOwYR~0vlU21oUWnxO0<+R1N9thkc^jkwWE2{y=@8LT1)Tw=nv27{~ z6f?+OyHcdNIc+RCXGCvM-cB9^jGez=2yjqU08j6RSTPT|H!jLhwz|ueJIQWj2T{@) zCz!dX;}m`Da)j&D;Se@!&Gfm(kwXU)*newAPZs%7r!TY~4>MK-+Y( znDB|7xeUkMhUM6S7Qd>V1XbFRgoOnm*_PS_q;pimp`qaV$CaohqF$1zBKZymM7Ll!PWlP*-PamB-XZtc@b@Az?7M4^9c6r z7uVT?OcAYh2*Js_q7!DYw_l;u^?q@jy7?q)sn+2IoBR}qXs=hvjkC%rpoYy!0Jh(;;f2NQBu2mfD$$Zqnt3ouC~HV84i6EH;y@bVAemrW?dHR?wb6;aYu&fM!Rkbd?Z;n92j$PyU96}pPb&!d z5g{c->ui<)(rc5NR+BEvUgqdHYAz_-mIY%79}Y5}+Oa%@Xg!Y}UH{)UL7J<>*G ze$NCmmtmwCW}XHy?DgoA>Gl0^M$DKTitzA&`p$mbUV#2vOT<1GTmYB*KY3g|I7$ZdB9~5ahG7;=Zefg;@nN7*kPCPR8riL!xdA=O{9& zsL8n@_v+n^7{4t!lgnHAwsEFRmgWP z@(+zmZ%u*;bK^kWl!nS5bG-Pu_tsP}gf4T`6XEN<+0&^=$sgVvHoB1N6$EMjz{&>c z&&&6d?O%CYiB45&BRggBPUYG%{S-C*ms*7=fhs4I<8Iij?3$weM85AVW$q;(mX>)4 zHw$Yp=1P;c1aF4cnCqAFAN~J}S4DC!`G3o+lDkU1O4PO`UN)+sP5ULGj*mb~saCM$ zh9Hz#P@4x(GX9zh7Cx5VsM0Fx4zV`;O!Tc| zOB=@)F{VvR)NXFV8)gDXrV7&24TS57Txatg(KYrVJuPi4dKv@o9?zdmMYmxO!&}yW zT=~|JIaWS9TPf=9;@0Z>6DB@f=WAxPPx147RAYy(xR32w?mb#1U z3&J+M5CT|22SF|+ThT89YXb9khMH&pz&K=J<$n$>V6fUg>fBHLGG0c6FvLN`#M?_u zkHlTR9Zzo{und3o^?!ibigU%IV!O0A*e@roeRkPI6A8nHw%m)(55~Y%-aV1YiT+_I z(ZGaYqf8Be)FK0rfCtvsk%0QOwU>7m55YFHVh@Uyy8R%${-6LiwVZbFy^8Uf7X58t zj`#WeYt*vy%r0D8)-j?{`y0v-w|w^D>+O9jOAlt>bwlRM zdJsIRZ;7u?4PCd%=ses!33D89Tf0t(GSPMH>c$-k_5ytVjb~>kzC|| zcRcw*_^61%(A;)njC)m2fg%Q|@vC5i1V$5P5rRAoSUCWWz(I5P?GZ8NEyefx$&oUX zX@k>U;PmHNho^Vbhmt#k?hV<`x0g-td)q9M``q&TPnp86vW(KV7OmrVkjlLo z>U_7-<2RU zWQQN;y}muRC_Vb4$=W1;I<>VHZu`wL;pbO`eZGUoPI7|L(}6pw%0jaoHcp`7kVp@o zJ>`N51d~UXzb4Y-){K9evLB5B!zNm)c<0H`OC>A=7}YMWc+_qtnmAX6D(;u5;T5XN zKS4cA46v5=feh1qdytMIP)G7_j)M2E+n09c*mmdG?v%}f*D}HPsksdMia}BLlNF4* z5p(aai!M`-`pNaNBRAI*{O-odOH)^drfa-w6yj|ky^HdmL?{@qzJA7+W}L$-S|_v^ z{=K%MDEn~1#LzXi^6!?AvW50A_sSjADtxE0mdS!PIYA0)`y-Enu6JIxczcC3t?|7E z8sGM6{+>Czbi+aeMFO$Hm7|967Xh?End9q21vZGW2))s+&dh-{ODs zd2qfUJ4XDsv4>hT=d0KT@Onc)bh)%3CkZZxFc?34xr4t=mXhKO9Xjx*OKOS~aN&RX zr&d((Bpvcy892G~t8|5%y6s7#Hv3;Uek6SVVHg`PFqG3j%|h9>sU(gqDwD!UM&psB zjO?={Dq@awMP-aSzvWrD$ZJ6^+sBi$$rHU%*E*L6b=FrBqBws_sAy3b+_)6bzt)h2 z;(AWW6Aug17&>h~H6lH>cRCcwbi+Qw?Yj3YA;kbDJm_f6(+p4MaMlAO-{F!B%XYxo zU{O5$zcmm6hKpqo_E$7Oi{BTJKddq=t`5a$e*tLe>R~HfPS<_ z`jDN-pB-|Lm%6KmbF22n8y4LIcsN>$0hA79Oe@6E7$-fBjAtqYX#J5Vfnx~BO~JOn z`fO$}ME3!k`5h%7uf!it*Mu&yV=nd)2cwscGdu6Oz_-d?pQpT_vCeAj43%M92@h4c z=&Q82<3l0=|DplVOs>2>Yk5>D{WRUaT|SB~->NLp8cdSQz>0sMbP!d42qe~k)1viP zuqvh9Wb`%mdW;3N6VMho!S~;CgJqmQh=}O%2=g1950FO>-g=u=_X9@H@lhMtnaqWy z)*P^xil?+DyZWp%-65fYgQqD%tjv0xfT&Rf;CQzzco}P88E7#q80Icf6;*aXfz<{|2|a$1?@+r zWBaQ84`q@lnWZ41@`O1JMc)I(d(og}bfg4*ki>pC>q zyZ>)vBw#_bN;_Kepkj+AyxLZrF5(1jLaL@9rrUY0MYG450O%QaV&2*-|HB2yq>1WA z-a7N2&k0-yyUzBopEYo$D?Dsa=(K1JjzPIZC($<04>a>6GyBQk3;Ih+U{g;I$occ< z9iM@I!e!M8-ODRv@(_q-rLkhn-_Htxp*Y|@L`a(dX##-twI6+9gUs+a4_ht>c^}jB ze&z1mBXte&!`?Ikw&%g>mJK21GLD&GW z<=h;YOTG$K3YPJLUjaOpu*+_L{oJ1)4giD7b8>=cFvTQ@*of0Q5cvm`6G~N78vl~$ zpA9Ib2Li)n{WA5CR;bB?c2~Z7#I?T#`R6$gqLlu$0*X^(vX6kLbn(-4ro0A-EPp+L zrX3Dtt~lENmH5kmKvjQ73;*c~@kiVI;nJ|txHR0MUupj+2Gm|D z$z8^9ke(axNP8~YrUuhA7@+iw{-4oEti)-XOLR;XmjLpWZVrLbN`-&cr{qF^PDT)Y zzU*9!U597RGc2rVt^8AduYnw!l$r`6;LEaq*An*c@g?|Pp9KFOO{kL+IvS-0#m~*_s zmsJh~<^#*Kbo)IkO-F0bGXDMe^!G}J%3Ong_Is2W#Fle+@|6Dw`mZm6w;{JZ)nMSX z&F#i6Payx#LVzohs1}U<%N72%gi;`lB%w|h8s+IS$kO^kCvg8wjy8ek?P<0?W0pfl z(;|&VwPpVajHfx61jqa{`EuA=|194^&vgAc?)`$Cazb3=txT}YKnvQs3?@{O0C{r* z&QOeS%PXt*pMXvQ%On7;oG?Feltv;)1I1U3cnl^U30yo+e&cF!ElCS{^o2Tb=d*ft zss|3@7-ElE4Es|ljw%G1Kh~8?zrTUQdo2n80-63l1~JETO0%5hZmeWcmSgPK@gOBG z8t!O%nA-_lpZ=htjUWhQHh%)yXI`+6W%HvyP8$ei0Pew7NycGYB}jvk2AADaf#{py zyWd%F@~Qz211&mPq}vDsD;RqVyl(5@{~ROI#7B$Yb5Alkz|Tc{(Sltcj@Bnz^M8H- zeD^%N0zzrBju8V@6HO+6ABQX5r894i>`7M+ULOzRlMJaf8h^H7q`W7sW` zOW*XM1Lm8sn_dNVf44126dYo-v))^(CYALWdxImb+KDfY0lX0U^3TYhrp10>GN)v( zveTY62Z(fe;SskYA$#lH_vgBEqfUNUs9Ec9!)*Zh_RajrWX8;$c(34Fg&lTB1m(+6 z48^Bt$fehh#{|n&rO;-ZCT@yER%<1(AEU>d2SFzLM@|v2kMlsG=Oefr3SeVcqT?Et zA(R!PeRJ|e1h>>Hel&dR+UB-dnD48C5CMK(5zw*odjNtKkB*8t{a1~-bDxsW@A`|; zUlRlc!S^FlykImR5HL?YcQ?Ap^V&Eh zJLb&vBLFe}i#BVrN)K*kV9<|??Q?~{1_Rqa+!SE<79i{hxeVF3`Osm9?L$xh+K!no z?Nk6)Z*SkH?5%ai&>Q5YxF@Mao;i&G*Wau92wvXkPH1KNjIK;;6m766)pL4Z173AXjv~D z8Bedl)!WZA^{R;6XJQgAUgtR<25|9Zw4!JGb)r4PG(TtGRm;{B@~bU^BZ!3&pS63w zRLUmY?o)Qz?^rF=WR?3Oyrf!C?R8gXl)_wm`7>yKagbi`pzB10^xS)wj+MC*P`6hv zVA@#bFuMi#{qpUpWaR$#bWswnfPcOXP`A48D;kv)F5*UwB|7C?>Zj++Pf1w4KI^@3 zwex9|{Kj$z6%%hQJ>(p0Z9X3DJM?v+aC5}FbIo&arMU;^WU^S+t-S+&d;QDt`(>!4 ziK_WRv~exUE9@7}t%vA{j^jtq%_dav0QAakKxr$j{>RmRqU2B)mx)xMW9f5!fkuz&fp#eAJ~=9(U4J zZg-&+wNyFHa@P0z)#i`tQD+PIN|#&M6mipAi9K%3ydG~HFvA|;k>#PbF5j<)iM(Mp z_Fk&Ybsw-yj9m>r%(@zMs#?UkWO*(^dgS6o|9Xa#HkIH#xIh&jB|2_o8eNCf!Qi&p z?xZy*8)yr+qF;?z?Yd6Uy?SJXRPS;_Ii5-(CAsw)AQW}cAqAI9S3H;N8EcDM@%bZS zMpZLIE}f#DOEUBMg1eRiQx#9QkKc74_4RPT4p_;6wQDUD1Zk(p%zg+LBN)20b8er4 zhtM17-p73zz5^Vj<&T5>cm6o&;EDkQtm=O+VgY@=zY^u|Nu1*{bXIn9i)B!EA9$4{ z7$rx5RJ0laH5FMU*$O)~{d{>LwI2lzxA11C%6liAIDNx8J=JGbgERq#DW(+`PE?|4 zFl){3g?mZoAuoagVNXprzDsn@0FT&Q!td|Qmv`NQcu}1)-ZMynXj{JNt=O_d*&MA! zEy`^Fjzg$aFJg(>wceYK4pg7D_T1Cf?@Q)ZkWR>q({DqJxUjtL{bqChCFU9wvVK2H zKC-B`7u_~`c~-G`4q9iQ{m$QH*$+};>`!qMzgx!P(!{B^zOcBtZ)ujuV9}%}zbIB1 z{I?YWUpE{b?Cfp$a?gwVHDKR>c|hy0kKNXQB5DLGJMz)II^PQ^t~&E`>A~y@`b7KE zd&=WxjiFxX^1U`XKBBP|I@De_!3gXM?GC)U*BcC2u&Rr{!o@e1px|K4Mq@CWObbM7 zXOl!KU}7-_siuHUanISQI^86U0k)sVuc!ofX_vBOEIes~<^BePZi6$;Ywl_Ze6b<& z;It?Ie8@@TE(7b7N%^YxxX~Px=+t``?V?xT;x4oW<&0SpuR4Up7aXphFRZq(4t}?W znlH#D)$Mi=wNKU9zqYT@GqB`zLfAen9;kG&HuBq^WU!ob@K|XT3(B9P9=Win4Wf@A zsKWuJ?f)@t=60h@qX^l?{syo}M+a(9gLAJEM9-5kdK*1)!Rn*4ZlsRhIF zZFCSO)&kBWQn$g_EZ!`bkzI_)gV41^oTan{aUpsv5>qyRhZzDd%3U~92sW@DDZkO5 z1H?A@1FJCT*1&wBrQ;adHKe>4<&bUUT%A5ZnZ56C=WDStJ`a4?e|}CD@=4@~A#iF) zk|%hjq_?&Gc4CTuZ-z=(@2L6@NAzoc^jF32%vHl=ekk*szwqKO8u9T4AU+1t$|WJY z3LBAsYbo6m#ql?>=i2Zh7PGwth;J0EW~(B!t>cl*ds++lBx za?|{(syuLsv3hk4aQ$|!!R_hT0Zm!N!QNiZb#$3S7uyz&(|X^sUUzw+?Bbw<+i{ER zO~^?a?23T4u#8@_Q%c7KPztQ^tJD)gTy&QJh0W|0V`3qo5ItegkBRTkfZ9~xZ+)Yt z7kn;p@M_X?-pz23Si^l~AVGL6D64JagI?1q$k1}p2rjDuOLVbg)of0X%z_e1b}et{ z^y7M-2^P#kn7gbAvL#wzWMZF0-M`-^R&Ae5dEL-(*3|m-fWWT%fB`JNK^cElozJ&y zbLtjtp4j9vHI?ktE4TK2BK~FGFbu4@-T%Zf8l7zD!G93uz-f$)R5L1XbG>fBTY+de z2s=O7M0o*xrw5czAU(q2*K)m--=i_anvG=( z`X}H_h?#xaV-Oseo&&7I(lq^e9E5U}ed5A`eIBx~@~ijp4F7?QaS*f+^Q6V%<#;{g z$}D!X;QT~m)HmWY>%M1g;IN#zZl|U7trXPUCkCg!wfPeMZ!q40jh0{Nqsj^LF7GT8 ze%zdXpJ`zAF8j&m2aL__Xty>|T-n(jKHX}7px$kNyEO4PY_i3230v27>wpS@cBbht zV~QAHrqPbl9lLs0SBenCCwpaB@w|M+vF469RjaK~a+eCbm)ih)I^!8Dem+E zyfI$uTSDNBSIq=0o56e0x}Ev=wNKN^1aN8+Tr)u{Y4mCE5U`O2{}XxVc3AmR$bB;F zT`GuCnTokQG-~?Bz_#YT*|_h{dJHAWYwDdKn83%55g8Y+c*#B<~#V8|dN@k9~2V!tR0u1pr&P>kri)5I_rXBd8H7&W%COg7@i{qk|wlX*mLVWATla z%L+wp*qg?__94&=GoDE z4=&4*zwWo^`K}GblcAlvLHnoGN~>d?9|dq4I7Vv8 z7hrU6sq?zMIa%Dlt)C*}mY2)DSi3n=Fs-u)L^U!_YD#o?So7-@6s~y2FvgPa$l1$I z_)+`PcGnhxus8DeTVan`&&pMdi3Wfui{IayR`fQ5t&rQLM`y-@*&SJ9^GCq;tL&r8 zK_=jO4}n2jY;#j`Vg%JuP4>tP1_Sqj<^`91OU6!iZpk~~wnG1Jf*{m|L@;53!fMs8 zhAP}%F7|+2lpeIngog$0@^aEzrdMd;nfn-Ry8QRC+yCl^fSvKTwLkrot9$71k?0f$ zi;Pwf!TOe3(VCr1bt{e`oUr9N7Gof|8jlvQ<;!f!uY%83_pzztq>qY~|I`@~Sbvmr z_eZpVN{vJr4J(VETI(cW_}e;ms(=AXj{2;{OOgSmiZ5U(ZR=n_fO-f)Vh&ZP#|gu$ zMSnwa4f(N68W&Qsq(?j(ch>=go?x)qw+zc11zENLkJ}%xD*>6gJiV%z3pTLQ%tvxy zFK@e5zfal4-Bb-4H^|1FsRLD&@$#$*c!aq!$4S-d0#E>0q9^U|RuBY0C(rK;fU^Jh zQ+@xUGN?t^2NJ-k42u{g5NxJavFigF^IXOKMZjU2A#vA-T_2Gqni2~lF}gbYJ(=q4 zq!BO{n4Qf{+wkKwaaa2Z0)LSzmq8Z!-i!L4yG@%!_r{mUFJS$)8X)59L1I)O4=nrO zX%nt4hR(GeP&imx>uw`e|?$98*cF(<^f)(w) zb&YzB9YPuLCe`*v8KhhBLH**)PClX%a|RJt(D+&L^{u@ch+{G1>kVK31&+H6EV?>3#a%8>_m5KdErjhe)3ym!3AbX z1&=YMnb-g|%Rt@WON%o>^VowN5o`|Q2XuCFC?1KfhE zqVH389n0&la(CyAkI^y?_TBj!&kJtt?MT`A>FF@)OFH=$sp*Nr_iPG^Hn*XxfXYC% zXO(j+;5et!UJ{*N{zq?|rGagn)0!`84=kh%#aIEe(z_)h_tz_R0WE%f0`UV-${a7) zrjEIe6jLzHv0AM2qyESO?!SMjWP+^X(`N@aYKSOJIgIJlJLPqMn2zdpxQNhjq%qhYiz!Np5x{1d?z@^ojMB~vqd11t88F*D7 zPgb@nX2>W)Jy)9AS{tWiVuL(xpEF=-mh#btAs?3wx~D=FM8^myls9 zl|_f6LQ22A7SjDbVpy#6HReWd&(|)qR>S*=PL;ZRQHA{eR{C8`y*v{&?+W{>N_-nj z*t&YHdGG7$S#jv6HR0um&y>BaN*!DbgMxLUEu+GMdx@F}e@8z)rL0NMkKV{lQ*daIx< z*MW|#>Xd?bo!Zc2J|yd9wkI{gw<<%=<>>0QpQJYa;gSOyXV*s`7W+T++#HHj28|6! zMTQF8 ztBVJY$L9DeDcjKjg{%!iXpI{uQg*@h{WZA0XR)byz6~M2Tw1#L4Kd}tvzTlaXh|sO z&X$~LWoFa+X7RQ2cPCB2YSx|7PsAaxpa|2P8Yt@jS0oKr?wH;#&0SwUIW8r#&l|wI zRVlLe?Z-$hIKa;46^ux0n{TAm{`rPAUj3Ipo@5m3n7d>Rys8BDcu?+n_#ghDKg*~= z9V}oh){m&yA)qVDW?QOkmEp8LP?5POuMFbwSI)hk{@yqWffoWDR8Bdd^IbN?e`u89 zg2od>;DBBo0c3<+7QBUh46$yy!oWrIltEM#SPE^ihF+OA)$2D%`SN`LW7#>)^nKf$ z1^yllo8>5|qX-nKf3F+>dKbhz)&TR@1<+a2jPW#hk>ypL+>dF@yuep2`2St&Y<9L!BWRaJYR7z^Y0tFN6v!*$Vcm)W$vqfxr>V2S77g zCXb3R*eB@tO5u)4K?p#hKHxOPk%KItAbBwDDH&C=yqLc@$(TJLGK~pTYZ9! z=MeFOIoclI0V9de$iElC5Syko90HW%pCk(+SHS_P^TA+~#s`p6F_&28t5WfwL(IQ* z1_uXm6N+Wne;tQ-159H@0!&=j^cqU*3;6Nupp5AOk`o@62e%q5AiBm1R+IlLli(=C zs;m_I9(x5&dEd8pc?$yM>7oav+8hb4Mbxrm;XL?Q0Q7B!D&Ej`!0#+31-k?lvHkuT z0bZ+%%`)SDZtjQ=8lv!n9iEv_1HC~2WP2ZFpoF^?cP_X2dJo_&VyMRKDq$E@qiha^ zGfsY&;n|<9f?%&N9+k&~vmkv>nBmM-uze(8U!R=7(iAaT*udk zl~=5%!K~LJ?(BuPeyP(y}iW`II40zjct zEUMsr0*B*JH-K0dmW9iK!5ZX0FwqmC2M%sULPU^8GsDC!`*fplmmX?Aj>Q~QF<2C34feVfMe?BC!AgKnD+GcSDQLGr3q5f#8`01)aCWp#>wxwmP2fq;djaSi7>eXS z@dGs1fx>!Sf2SE9nS(_o6hQi*{bzp>iDT3y?wrJzhT~A?LfsV%uoVvXf(YAeBOKE} zJ;pW83N>Fyp2}_8xCG!j|M{ij!2ZyfFh$gUE&%q-Y9NmG&?ychSQXJpNk`p<%GbY- z1ujYuyat!P*BT*EQUmk^_5+kigA!xy{FmVkKqy-1ExeT>$3U4}Md9bSAsx3r7}VD9 zg|)KcMjgl@^1!G7ORKZk{SkjgfnFTE%d$5K!HFA(SbBp9<8?3z#C@uOp>A8MLJ8Is z1%N3&rrh~s^MXOk7=@*|b`TT>Ddf7OZ_YO^ojQaAm!O9d+TGO*u)DqPmy@w-C7AU z3?hLkr1(-s_vrYPZvgGjOt?w2zA$e%4ZH%>!qL!OqX&s6fOXangfo_Z_5?!&<(NRj z`|Pv)HMTNnZos?g0e+tj`3sOT60lSpL8}R5s#roYE9cDWA8~H#KjgRB-IX>uKuh2! z-!}yhrXxNb#9OaGfqM%wq57a=+umQYlYd|UFvd0v>U_pDcLpK>K0t}~FVl*kqkKON zlwq;rM{GT+1AytPo_7++w>e~LYhV+%?e*O7fk8QTl%TuEk%j7kYw-d-^oKg|_}Cgw zJ9Lpe;3@MU7!>JS0S$7#Mw|p)P>d!0OszY{J9u>CSPP>DcTW*LIh9YKHal506;%!u%a$ro{nnw*CV{WX;Q)FU!_z< zgyL0kd=t4R`{;8~=m**O-_W2>A%1{&=oN_!2#H=~-CHPy2TF&_;z5I3)8Q$Mcm$OB z*Ehq=yQc?e#jt;mU_=;24D+`mua!k7&Znmz9lrM<%ou=tTIkNDGk-8N7XSJZAhb&A zlKYQptlzi?L#=?CRoYely(28W;Ea7^@^%Z~rxQ5#1wM>;1JR|l7cXG(VxXLBaptPG ze|QJ0`Sy{xb!JRpTo@k7o+DtPN7?)S<(>{BB8eiTE?bPUU+F?`#^orLONNWl64oOYyY-d~PQJtWj+ zl17d5r{eyLL==$eg3TA)=cq$Ip#$kko#!o?4mje-c2MZBtN|-M`Zg3P-bauKrGpN$ z9=sKh$z&`BDT6Y83ec;NLkCGz+Q%fMb3{MDgb1dl+V3L6qK0a|<Ej;Vmh~4uu1DsWw|+LIp%F!j6nO%?r%UxsN4D0a#M7L;7o8R-Vw3;;Ry-jc#TX z^YJT z09<+TQ?SPGTrKGF!AkTMxW0}rpLio;j|mVN;8eY!PJ_Au?C;w}9wXjDJ~R-dsyE2S zE<@V_s{}kSzo9CAbdP@T7Ay)6n8`P^1)csAT8d0arpB1VS^Gkwo;4mWbNKBAFndC- z{w;>&w0(?9LavOQ2t<)FPDASIa1cd(AwVcbPWr7UGiJffoV3uoUI1SbtL!1r=~&we zhPfH=1}Lnt_8kgpE%*JG;%p=7SIg{NBwyQ4dY*fr!^s}l6v;mRS|4;?4?Lc&`sw%T z9#{$3zy#Xbv=$q|2c-6a{W`{1PuIVowOMy<=TyPG37Qhp$VEht?!VQfH(e)uN8?hB zc{kUW9QmGW6zyRoMNs2>p7|Mhb0EfsV{Oy7M$q?-?>gE;zSko^dXC!Ym@Hp4^WY{f z_G72|*X-Mho$9O=GXJgX9}F)YemHGk|DqbS{sG)eqIM+TuWEXD*J3{g!nd~Tu^+4W z!d}uURuSrA#cM?Sc>Td~WWZ9kWVsv;-RzD~61&qGhQL7b5?46F9tY1}QUSjqzDhJ| zBW92{M0>^P_ESEIP-^IY^GAQ9Le~xW67g^5fob=%JX_d_wS0=>pj0-F%pXNm`)kUV zp>g25F-ryP?r+XQRvuVq-<05{DU(rEhJ$=xxU1nf(>@l1aXy}cmm{dObIC}jApzD0 zELD4#hvwQQ1P|exs813h>4(B%pAn&EdgtAu{=qj`Y|WuWxDOewP_o+rn*XoQb+8x- z4$&HX*zOB~I3)rXtb5|bMSal6XP?Y0ck%oCvg0~tVynq8D!T)uUmrdKzQHbmaUeBRQHdWF91T6c}(iE~xh z1rI;uW5LoDpCbGJMI#)%Ot1qA(`}dw`zRhkRv%10PN(d4#3cahIVOPYxQ*$zm_4_5 zZ8!%eFxEIaGfb6WitI~X@ud>_(Kt@Cujbf(Qi%Wt$C!>kbj3XB(gWiBfV$mtN!Z;R z1S!-p57@4{|C6gBTpEFydf@`d_cppOcoUE4D7mr2`+&HyH(@aMcgJ&HxvB|YEbqRg z^T%=!!7f8?2GNOusP`xe zGG1-q&O{P*vgf6tV-f}6N^>+C2OYi+s6b1ZUrKv2#?cQ|CDi3o0l5xLAw?=AdrNLl zsblAoFNUqgbZTwuzR$kn64L?+T5Uki#)YT?`*l&5>h2RMjk zVBnj1k>G$@E5^Sz^qVB9@7=n3ND&r0(142ZtOaFtvSQ!Yb|dg~3h+II5+h)nc=)>3 zv}h5jX%jobeF6=ppwGdx;9}}XPvtr+OGl^xf|FvQ-J#Svz~aBX2YdMbf00H0C?!DUhPz_b%01aVwO$9nVfd#@qJUT{NH{xf z1-#uXbQ+FLX<(rU2E5Pyq1Kuk zbU4eC<2`*c2M-g%guHj z^4uRV@ZLAX6KGRC@Y+0&qPRDV1q#)}X}NojsLXxIW2y(H{6*7f+gOJ;B|5=tn6igcKx4b+3op{ajx_;e!Yos_`ij^f#N_n~u+`=No zB2>qh+ln);L_0<0Z{2fgue5!2EMoBDv<3L0QT(hZY75#jwqsf$X+%*e_q1EN?hQ@a z#)z)ANY-q=U6pM8z*aMmD1N_XvN?HoO;Tv!@0Q4+6ly5Z8)igCKxCTpPBw+~C_wo! zc!3IHp8W1N(jkuHr(%gHP9sn1wPqa@ao7n=8@YAG;t$U+H0mxcp7s+80%}TIoJmQf zKupnq@vS-jf06MSD~#nq2({CxHG4#98mudlLmwA%bnTPFA4ivx_B8lE%mhsMRK^18 zeR8-@5k)RK?^-8310erbn~D!iL-uCeQI(I#p|FjZfFr}0%XIU4q|c_M+KQ2oKnuX| z!GvTK7JppROBB%DoF|nK+4q#YM^uvp0qn!@E!$)4xhKDX};V*|sUvgNRx z=NxQ!vO9m#5epuc9nd2rSG>E1Yo{fBTn9&_Y-fNx83E)M&J)q>RYGjtXtDXZbtCN1 zLziEUC)(g%Da++Q+mu#Ze!7nqpqh^dFy)1%HRMoUVM)~aC9NFJKO;q0kb6_^yba?? z5h$ey3*^J+Fr@`1n@wg8#xk>$eNc~&{L|30chlPWDZgrBu3HqVH+xe!uq7lS5!SS1 zrI66xdoX=GEIP5byr#+)xX}*A}I!?U+?%hJN;e zQJdf(8t_61-driLy(naOz`$JwXyCGKb6b0ygtRusqgDr)AOvtVOfLvfJQEnq@$0`= zra7!BT9AJ_#BvWd79X_0nYmCU!lKG$+lSFzwA`g#0+qKapk2%k1SKfTk4`QbuM*!R zzGr9rW5&ID>8}G2uc6_Cj2&VxMw>L{?Hh@)jE45bBPgVP{ObV`3N<6l`$p?(i|xU^ zxkS;(JeoQX$EsUv&>EyC6@d9arCcg?!ZUb2oa5(yvQIQJI)|RR@BDAdvCMHG?4!Hp zkscV~J&}S7c9Aq?o{E43QD8*PMSan9f9r7x7I^pxOA^i{VHr5z2TXpfZH)wv5icns zp>y6{iO;m`&(i}AxjZ=JZ1Qn3EF3EmnGu5@uLd07UwdC@JO!L5?2Do$K>8?W2EXG2 z^O*ndd5At@!Mbz^*4{UBc5pCHv43V!AsYI>jA%PMXw)^1%!M6r!@&;6&;X%Pd6xg5 z>qD?M0}1iqxT(NiEOh@U?TyT@1k@NNvjFY?{|Ok(=s2LM5Q_uxzkk9(3h=DRmA`*Cas|LN(PMtQs`i-_ ziw*h&;H`VG5lDdli5~cG6DdW#PJaaTk#!E%zDEIs9R5Y{{_=mLuVny$SqLcl_l!g= z=luZHXd}uA{>c&m05k|O7RHF6j{O(aXaUQ7_G#w$FJSh!aEa98B`W?^xL~B?S*=5i z#93`k;F~m>Ua=@afwp*Y#U62wSwjv_zHJX)cqwnsmM_X%Yfx-QAwhYB(}Uko2)veV z#<#S#8@m=Yq)jsz2C`1A(ZEaLRJ%7ZvX<4?K*`x}+z}&qE>Vs$?Gv8tiQ@AL-H}Un za6|fPmjsj{Tzb)pm?01bl-A$dv$P@o$-v@1|GalX==R`?&dz9_*Q;nPM&4*99;1E zqnr=yu1VT4+=y&-LzkcYw;7>bG17)Mp2T`|wMHEy{ys-q0Yfi#j>yz%bZs#Aw?sCQ zJm#f?u?fuGe#>st5MNj|=Z_3KY4yncr$}J(rpArvWy{Aq@|Yir#|t$)zsGH+V0AxC z7R*d`p)@122i65YE9Z-EXyB}LAe11@U_-c7`V(Vtce$l;#?gw`(RRv=3c;sYUWE$f z$#W8d?|xHrX$%42mM%DFU~lkk3P5o1HcVTlUydl&=jr3$XOa9JymUpkM`I(z$gs>Q z=kAiLgmqHdwDlXJaD#SrH74%y<)?VsS(VnGt5mS(J0%q`S->;hjx&gCez&iEJiljp z7=;G0!V7(Wm(uj>V0Q2HnU&|voqz{Y4@~yzj9E&R&&BiP{^H4t99p8%H=6*^8T=Ev zAI=`v=0B@ZG1E---4+;N9mhp(t*;}uZZr?WK=^BSW_Z9vG2g_K`MMsH+>}XZl|=8b zF+5p*w>lQ(o`y+pf5S;mDl%&hD9x5|^*2MtLhK_L~?8XfEH+ovb`D6FG2PjQssT`WHvbaG`D%4&Y1C*a1!x|jX@m9+#^p8V>B`<(q|tKx#t+~Y|J_^xEzNQa&J z<>A=@H4LuBi3&lcz}d(l(|qPc+ku_jjR2^$8U6RNi?I+Z>bXY5_C(LBc_L>9Lect2d98TcDrH6;t}R;>~*PYfbk zQ*onRrOqLjRL0J zKr=ByND!Y{afL}*S{u`<;8u=ep0VhY;V7N;3Bu>x5oOwDsBt#_JE$nHW`M0B_}VYi z-0V#~c&qls6sNZo84jBU0<-Xhb7S{c{MbBQ3VUd%yhna&>J$sLdRvtJ zHXNPslEQcHWatRl@RU+*%sRJz*fJJ~xfn4VNobOv^&m`prN?1F`b5prYeAdf<}}U- z;clk~ADX5-fD1hS^t)i$AiW{gYU*XTbJ51N4hu||D$U<*vD_`sJhXjEf#4$zgAZ9T433-ftz_b`$rWog;-5xcf-W6a z>k^fx+p^FBzL5o_OztNq^wpQL)WxGO-Pwz78eK+g!Z#*ug>3}4HvWzt94@(9b7u2- z7*i}F`|YcoCkEGN7jWS&%;jIyKInb(Tx>MYyVv>~WhS~s^n)nC7f+}sfO^OPr~38_ zPAW@0n8UHcM*0N}37g%D9QGOu-cU^`*R0eg8~F;kxUr#zAa8FOEvq4`fc2qxZr;Mi zNjLf*>FnQKj5^hK!Y19GS8nRWR7XCl#G49wQOKoXaLmaBKG1(EVcxUfeUKnA^_{Ac zk+HtbvF$HKpXbgpb~~MKYHBxORrmIuD6>y#>lHEask3sQa%EUsoRxo<`&Byr(R)aP;=T#%fkIS9+{nsbJTS;Ja{j>tn=W%#$(2JjFGx7X$e}z zmgx=NF!R038R8S(j(dxP?;2~P;8e>@xl(t@`)W}Q+WLjQ`MXFj;i73*ROz})bXDjX znwl65Wo0tc&raXt2Qo7gxLbbfPy^-tb`<742|t0xQ@l*k0`>y!j43BYyM`J zDV&eL)NnX>tJc(jCg&WX_8)k@1o-j<&Z zV{VIAob+7%b@UJ;=tM3`(fc~p{58u<*DfEgOnO#Q%lb;M6$gjnJ^Y%?olm-}uM4>X z4IOiI98~iAZs>9b0ES9fZOSUJpC_rVQ`j(`Z9?d8q4Pu|hNcgBvHDfT{#H(=G{o6z|}m9&AS%qi@!U;3H&6(32o+F=l3Z zrcpFB;FNgd&1+Z;K5vp^6md(HPWDZS8J}(!mqAg(NKO0Ug;z_|=Xa+r%u<<%-hb{b z9W~goytQR-htr((@l{MN@e_Qz@JWlY!6H>2uc>JtyVF<9^rm$g=mswg2H!0T>Erd#iwh&!l-t*q%zr>SD zTT5y4%e6Ap?Yxk>;Nyz9czH^XH5aJybo^UfM0o%`|Fif4F zE4A(-6<6B)Eo!pbP_^ZU`pWL;7q~}xuEX70({z2d_YLzVN?gg8$8o+!0a0S9;p}>G zTu-h13cR8ecj%&-o+zQFgnAqA3jyYGguS{}aLd@qWmy{mRl3idM(4n<}437yK`LcHRC@AqLdPdQdk-gr3x7wS+zcXvGVR&eJa}E z(BRS%$_F$iE4taf9^^Myy2RAPgQu+qJ!JE6-{Mb)ZU;E>6vDZ=M_IUPkXX0kd(Wq-l&xu)3qo$Gl4p_F|E_eC5N z)g{PBckGiZe6&p3k~c=@ zB8EGTOHG}Nt*mX9j&)V~rLbg%k2Pc8m@s|K`6!WpZwazzbAkWqL6XdEdfWP{=cnrz z<{Run^}D$R=9kdjaC_Ft)Iv%`c7azhiD`6s2{kQlUc}PlU_(Bx^9I_(R2n9=TsmS6 zZ$l|##FDQG6#r`M-PzI*itV8?^F;8Cu2ftp8jbY#GM&2NCS};USQg5j95nRhTwO?M zuUO_oS7cgE_v^$0sX5R*9=ailbJ#vZseP!eQW)X-(hBKv95D~nOIbv=eX*-v>Wo@I zA_+?@K6_%Umlm+%8z;LYL`D$bv2vCi9JE0(aK#&;JyVPu*cZ=@;Ilzgu=X8C@L8B& z*En(F`SizQ0X~|KYl#hSQ4SNly4#dV_g+i7ywdv0Xd$+;tn}eJd zNfgR7&ThDQ8Un!hs%>QsUor(-k(U&y^kk)GRE=12KvWG$zGKH-$|yizuQLp|>%@O? zSC5bE%;zW|BMCj&0S?(efNoInT6`9YkMN=|nt;x|FWjAG{cU4uirOc21~nVxQho^i ziv6xjpqo@$8>P;!3mfOKV5x+&@N~3aNlWdB0sPA9;#@M#Xwc?P1fjrc1DCi|+WT5{ zonkdt(bsz+?0i++^jmX^C3*}h3zxUU17l%c2N}jf>u#a7|I`Q zwpyv$vC0N ztxXftUs7BV!VL91i-MI}n8+JRd72Xp^b?N?S6hNjUVx)N*`G^Y>K}9KQ>6RWb3a}w zCqYP6y;CBTba0JAi9G3(m3ZE!ZX-DUVkt3_WBp^D-q!4e6rIb{4f)d!<ouU6_iQd)i;pj=?%lxlI!eE zO5w)}^`u8kmPs!Y<}7%u;K5x4=&VlF>vRJCqx&B?-YM@Hodyi(DwXlX0A)B~z|om!^{NsPeSkA|%MzaVQr1*j<3r=)-B%VGu>Qcb0tUQimxuHQ>Z+~%JdF7I`2waJr zLNN>0#gnUR^8VijSWkzKY&jj5VLvWEjm`W7uLD7$gtb(=9bSSauj{5(|4vCRB6~9Z zSAyj2kXl?(!Z%y%RiO+jxuaGI;4MJ#^$v^(qG5Wir(bml{xTZ5ed)6vecD=oPLKX3Z^g zNNMT4K8y+ST=8&-6O4AMY#yR)dAh(Rr)Qw4VI1T$OMMIcdxE1h$8F;8(Xciub_V> zoDmJ8ofN#7>+!H;R{XeIz{a4uqCsLu>|g!8$30KD=oNQ$C(XXYeQ)n{XL7W?_{GEK zZ(Ow1#Tgwyu{~J)t=J}6Rg$Jgy=p(GSZ-+3*Sk5!MSi^O>6WkgIyTLWB@37sH=hw2 z2*sK)X$k2-&W@Krk=+Q6yK{w$Ag*$w7WePCpRjgfNg2XCuZ?5yele$eUoWm)h4=4J zhibm)_=ViU-y3;~C|1B=COD#PCEpZzo$5G|Xk};Hm+UzP_@@lsmu731( zK!u{UJU0O?nXG!nang`$Jj%G&ID59l<4H+)57Whm>k<;T7jFL~d@3O{Y5}Pjqe&ic zoiZMqQ~!OPq8#gS8(lXyl*-Bti|$=~TSBtoZm_*I{sOmEv+DI*zTsfMBxYu{J3W~| zt22xsfbsl^7&4w!f!~g!pC@yWD4xy@HwS_YCs z{Fb2)t9l!B%de9-Ge*R`5uaNqa1>P9@O1KPak^!cK>E2nj)|v1G{ZVk!$VZ?v7eoO zRz98M@NWFv__hA`V>wTlbX}X!?q3>YlzjV3NwV_D4TgtTPDT}JPSo_yZ+qOkTBj>X z9GA>#>jAT* zh}0X{0KvTxYg>?hz3<(9e(V`x%ZZj@a7lavv?h4*>hMgLoDXsHh6JhG{(MBu+-M0~ zC97PJ2tW_{>3=a_5GmutyJ^%lB-2K=Kbz4#+!2J3p97fw>qB6p#&3I+zg<3Mg!e|O z)i>$D1@Ay|o;*P<{=r!v^e5nh?^%;;Cd&=Rp?oInN&!{Z?7{X-8^j!&2lYU}^|CIw zzuc2HoU!&Kc!thyeUrkq@BT(7;5Flqc8utNP~kk_*~>ep9Q+=z<_DtKIEJMi?kk9S z-`cn#vX3qg);4aQ}DVEn` T;_2Kt;2-?D(zV>HM!x?CTkK=0 literal 0 HcmV?d00001 diff --git a/docs/design/architecture_3.0/images/resourceManager.png b/docs/design/architecture_3.0/images/resourceManager.png new file mode 100644 index 0000000000000000000000000000000000000000..7a8fcae0c7a986d4bc9cee650f71a7786d692718 GIT binary patch literal 142678 zcmb5W3G@WrRTuc>BM3p35F!ED2{8$fkJ=Xw(NuL+b!}aH*CrvUy{fymuIj4ls-Pqz ziX&ObNb)Qt5>hy za^Jnb`@8r3zuM4_ANMhT_+y@U;)#!wM9zBRi5E^!Jn<2K_@n;-DEa7b+P)BcykNAB z>=S?Mhd=#4Jn@9^MR(r6H;=rkvnO7G9-jI33Ir;<@%{>wdj*2s<-n+4U1-(K)HFKn~!0 zU0pNqamp%VX1H@Uj1`!xr)TcDXYV)P*FJlu zG|lEJ0Ug;(doVrYGsV_37pF@28yhRHED3eZqhRiv=JE7 z(+|tRE!oly*Js+n-A8;SPMu(2@Gb4>RWQ$5iWqH7-WO`~V$pYP5OI2 zKUhn`FP<+wz|+~PUagZe=OUI+GLuaVm=AwjHa(=l)$Rga3~{~;VteV3IDluv=P#r+ z^4*9gtjsOyVOA#ZD1h4*Rmn8Qv+}?~kEL|W&Iz~7$0~uU`+*mb)jY-B&F$~@dDsS# ziKUbJI7!^`ZjhW2VlDhA(JMVrCfCG$8zHP-8i%_otq;!VAllOp&W0=;a;Xogh~-_z zn{`%Q?<<<>*$B7dQR=bQ94Vt+Y6qo_JezV6qA;$eg3R2R`GjKV6AUk55TXchG8%9w!|~vV^)sQBB=E5yW9)#$lQCK^_^*9Pxau)yJ?lZ~sAnUA0VOE`+# z%@H_bR7CKrIZCEP_*QhCO!%IH>OEXRp??}6Zi~Rrn7>O+pxz7q7L8Dg_^GZD+F~#j zH@OR}H$IaVY4*4#4CE^zR>DWf2QyYF3acnP#7;G(ZpA z8Q~|{_ND<^dlU!#L0~Ntvk2@dR_9A15n`Mwwqtpy^!uVViwm>_Yu$OQ-6fmGOMWsO zLOch8g^r*^xQMt2whrsiQ?Fy}bOMbU%&{eah5=oP*aciv5~>~%`NWaCZm_SXQ+mDD zh=-EemC<#{KNYJSBv4c#{F%ni;1=xIW(Sl%uL6}HkfEXi<#N|ghxX>ow&`xe!R0qT zH9bVJZ;5;iC|z$F(?ZYwIn>IdE~#F#_d!|7y}8?LBS2BIVL7-%CS0N$q^?EFA)cOG z9~ze3^25ZF&BPBEG+mK650`v`3s$$KeHr`du3KdaHYF=@Zra*b+A!i=xy{T`mR{hG zyM%gSAsbRhLB{F4QLEh9nO!VybdV1wR`J$D>X|ba=$6H@hNRYx45|ApB6c3Ld0uQ9 z_K206=b|^Eg`?y`pp7&J(*u(O+d3xn=^}au%&7dCIr&;0K$Cr&FK%%X)oQxem#aYz z3c_8^44Ot^ND<5-z&rxzNVXm-qU+4EwsBMgvW53CW~aF~ZmpQK$46F>9npy}tfiSU zKxNv--Q(OErxTH$lt(4D&?9NRKJ(!+*~~VqgMI+*J}e`q!$*?3J{ztppJIcA2%End z+QGH?eZGJ^uGl&dG(EL~G8bXqh;7qL*Ycj*K=gzaSS~j995>-FspmZ|V{?lWtj6S$ zco-Nm7pXXPqI+zhBdG2eN1UGv>W-iokX6`Dt`(_UqR#@m#189no2myD*3S76iiub* z__e-IXR2UNOH5RWEC@*5rU(|WND$l^Afm#uu+yZdFk`I&ANoeXA1wRssmUxOIYSY8mrsgP6YCL=<(jyIoj#WSfZo=bpT-iq^!JyVUp1t4# zR1M!%>z3Uo%!X&vino=FYFh0tBLxj7@_JT!FZ305Qtp{v&=7zJ0udoRY4~vlJC!?b z84aJbyT>7l=$$3y1f_@4j$qW*aqgr@eJ7`S5bZvgSwYu=36tq5aBg0S zTwT3Kk2jgu_^FAZ&MJ`jEZo+9@KcArl!eEw_rCM$L~ZUf>z)Ugyfl{m5XQr)3a2*I zj9c3bT@`lPnmBd#IN^HOYE51#;SDSCxRkvM&U!sn)&$h-N^NAl8h#LNw`&Dxhz-@z zg`p9q)^afa+?{t>@|S3mbRB{d8opG=0!`D7z}YoLw3wSozO34T&4*C2oa>zo)Lr}e zm+BnNy%hHp#c27MoP1B`&=eo-dlD%DsDilM`GSHk*fQ_>k)w7jf`V=d&w-f#4@8q) zFS?UxynQ{QzN5R>&H?T0km$pRh8dEwx&!bz4Nt=*lqbVSn&G{|OZx&|!knNKIJ2&Z zLvv;uogWo~?|Mif9!x>3g5?2NwpVrAs+-u6b3#TsWHfO)>62ukVUxD>ZK}B`Gr~oP zbUEjj8dtB;Cf1k!rkwlyL|;Gyqj#WrtFQ&SVj~3RsVlamJ+%l>pB#s)$6Zdw`6^LF zMXY9TV8$ivtu8tg%DP@CM5!(CdRf);@& zX#`4Z2VYau5yu3hFfxg+QJQNcE#7F#A94B~x(3X*R|}^+P_vFOr(lk9&dg$cF79c} zuD&#g2gEt1aDEE<%c=8qqXr;H_fK>T6%;=lRtMbxz~)+!0;)ntx+_k;+Qy5cn78Yp zW^IAn1XVmd(#JF#f$sTB5I_<`VP<-_VU@Jp@c1b4O&-0ipMh%`qb>4`c1BQWleJDC+ffKMJSQ|RC zVJrF<(ka0>t3)+2RUtZBT*Fp{vv^IcL`R?^EBqy-di}10yYPJ(GMrEhb@CwcG=!F5 zxbn5iC+b-9&GDh+678J^^}*204y`v0Oh9{)-HWSQ^mZrM8AB8vj1Y35d#ZK|hQq}u z;XO=&-7QpoTCHN;+3+Bz0uo8Pmb9)DHL*nC5C%-=0Qke{J<|J4lp^2)SnINQRbT7C z@)}8lU#n!xPhwB%O0h|`*{2uhfM3mBDN2R{AQ5Qtp;Hw)i}^L0;{adK!bVO8^C;Un zC`lMV0upRJYHgW5Hw%|^?V(1Qdwhjl``T_95^Ty$m=vWBp+KVpaN>k%Wqsa(LTtTT z`BGxRI3EKTyE48fIsSHSGvZpT+XHp9{2u3rANjB_mGKG|uByn9F-XUugHp9QC5FS~^ZR z{One9SGZxZ2(hWGSCMZs=frP#HRTq8l^#E9Y~3m50N=cm9t2rsuZ><%4j2*b3#7bq zDSu;hsMRR8DNd_cU_#=D7Y}mR(q5p2+=z&;!!EgDofBcJUF%>rp~XRti9E@nEiVmQ z3^cJrlVG@B7D%piA5I70o(hX|gC5pwANg%mkEV7mTwc6+D|~JQHQ{&k9tj(L^@(FW zU+S}dpx`?TkT1}PdU0$hg^pJAct<9c$hcwl>y~Z_1#gZ+!(z#_E+nW+o(Vn7yEEmQ z-8rqZoPa!i?~iSa-i^dLp>gyUa+!+00xXQ#%#jf=ppO)3J*a0$u+-G)`^J3QKk%Hj z>`8!CCRfD|I>~CM`y~SS%rW5uop&j1PwP=tDU&|T+`wdr9IARu-KX{PQsi;6< zo){TNPSegOwdf_7S|cpJ>;^4!0Tz9PuCJnbd}E^WIOv^wW}TBpQ^qh+;R0uV;`8_T zy!zKeuj`zf%aZSumLEBX>~4!@f$FEx%oqSbsWz>~6s1!YtERZ*z%y7n-Pl?{L+X?$ zaT+!xaYQ!&!ArVq3gHw<2ji=O7?y+)p>3R(~0#Dgmf@tlikqy%_)a#EpJFZ z5$2;~&W-Qv@IiGe9`pFP$ptZHA4u|8rA65WtDKvTpTzQX7R9@NN>gBYSRYz!h;?S5 z6IekwA&oky_vU`&y`6P19QddL?@U19rfKNgc{4X3$MhZYtm&fSFke!{80aYkYO$Vw zjJ3RaNJHA@f_BZpox!ds!Asb%ZMx$?hh`9&9*KQKwuJ=yH9|~l;MlPfwySJI!$#f7 z4lBycyH>WQsmf-j7UopL`e`h96zu-ZN`Ut(-0Kr|Q}!L^jo8+I}xh0`3#p@Kf5)p7KOaw4(EJM>IQp=+V?}!={RPM}lzN-R5F958+ zE`=Y%7wW9uS=*soKRaY{>yr7}iFB>NP@`#-V!)UWKh)BsC%~(*BTz~#wQAyc362mN zkpqW1@xcQw3e|Yyql}B4C!2lb!gN?F9E?oFNn5Csk%P|Vybu|s>I@Mz3up+`JRw|G z(>RueSS-eetxt1nf@_nv3AV<3Rb+bU&?Rx%GPOwp04B_`GAV;bs+Jo;#VvqF@nDC* zPd6w$w50|0z*;d(CW6VbpmdbVBPq}_*n=nNUOW70WzFI{SG7s7~9Q|^>I zYp|w>T)h}Y+;Ti|YPMnKmfw=j4xl2vc=9CDS80AoXD#Ew{XcC%rrOLPZMv z+3g^YE94FeI*pf9p!GVPzV!x|@V4DWIO}X-*3lzzi#_Pf^rW1aYKo3))Kzx7nywWc zSLTfPeY)ed+uAmZb!lo6KRz@`PJ@o-?&MG19cFPDgVEbbXqn?T5ZLq9PZ;MG1357R8*%{;0KpBoA zwzwBFq$rAn1f#l3ing9THo<(J8%^kj3{uy@`B2Tk9|pZc0WazG#|fU-%WcVcp1?PC zoQh-OGxNO>Z`dxjS_{{dUqL+*>>kn_;`&@a;*~8)BfimTsZ!!S;^smtlan(& z9DMn-^eamkGBEt|lHTTk;rc1by#?5ULo?DD+?y>P4{%!JoCr(aRS3}eP`jzVMK{p< z0D~U9QR2A>TOz$>7saHEv=k2B!^WT^TwYae;x|h-%7f0TItUGzQjWwC8bQHC2z(TF2Sky% zioR)LqueOx;66;5H-MlD?7a!rnLG~UcRLy6C+u`P-jb3?egkAik=FfXHz0MVOkK^N%r z=`z^!ZJ!t?q9sUE*Y(2n2sQ03t>ep7 zA2Ldj#6=XR{FfnfUpL$Ll82il9xEfQ!o!DJpz z8EB#DZMxcE#lZwfQ^11YnxeL=7E^?r)ZN67-kGma+zkwKQ~Rh^nia^9dUlV1P`w+i z>T%JrL}{ePkLHS}5IH8IDBbKw$BnurP3U^TiUhBomZe7`o75>X-Q}heohILc*(CsX zzz2lPh7@B4oQgJ#O!$B)4H(LHLYWd8SJ_-g#)m8zEqWI-0fQIpegSk#55`1Yj&~p1 zcIYoCc=Sk0qLlj*SEiCv0p=0TnIIT@xR4 zYACsci2HMC*f|y&XZbccFy51n%Rn1!?Y(slrhYNcE5u->EYYZ; z^Bi6^l%{tw9O{U0GJ2OX#nOleRz|)j##>WH3DqJqG<)l zo!$76t95m=S`-nNr&-3|AJKSd+U^Y)_ZkjjEgdPaw7mnE_j{33+r1{9DuAAV)`XVo z4#*uqbr1Kd%fSYD8a9P}n(VMi0t!jE>>4v>Ofyi+x$S5@B#+K5j^ng>0GdG)p2HP)Gj|f7p@Y?gh9S!=hG=)qos;DS5+)L(F*nO_y(N$L^=)28~whK^osD(@C zU1o10Pncm@iYM&|OxmuL4uLbeP9}u=YGLXU@;8+QfrhYVLBuPh4voGJ@`b_8!|8NY zJNmAT+F6p@ZNXJqbZR1-T{;_*$FA=idI8P&l-ll_^VpIe>reQ@HhATDz7KZ@go#ej z;spcAkF)}}r^89rq<}gfN5(00=K}bdt=VCl#@ETCZZe9MPn9IEVbRNWTgkm@E8B%T zA3rL)5!8Aqb}3lw%I4x_&!EUF57* zN*pnFLhU@#qWN+tYBD&8&yv&qwoZDY#WqSNm->$9o8GZSQ(C;ks952n`3WK~ zJ7~II0P}O#Y|dA3!(w+dBiOy#Z8Gm|4L0}6O_?;PXF~s$kipz0yp|Pp!%%RloOUk^ zK-NH+WDGo57e$p(11mjP zB*C@I1?WlEzUhG>hoXEbbv?WsAI6v!``&Mw8m=%!x1E8G&yTp)*3;S5!wF6s52OoN z*$OH6al0r+gQ)e~>KGQa-MvG?<$M(#z<4E7TZwb4>0Io9tN`RrUDYdj*f4_krIV*H z(%C;n)o{xxvrPT_44XxSUa2_3!}OsDGa!Zp2UY?ceEI>m`zHp87~mIp>ELfBfxlqy zKAuQs@vWO%Ot@p3;e!$g+C&oRaw!YtvErf*s-qZm|t=}A=x!F0JoMvuwcot-S6wocty+NE(4UUaG9Hg_O9B~l1a3-aq zlwFsD>{~$vcFI`t{~-dl2oriM-a`v1RjfvB(q2l(^Pr`op$8~1VnJzh8?dUv?GU+Md88Y zuFD{GAZ+h?5N)9?9M>jA0hQvD{=uqkzzgPAYDK`U*TOMs*}wuFU^u}94MyBG-95{M zyKQoLZbPpWWkA<^-M;zNMr46opNnUxE$B`LykgU5F?E@-n%+0-z9p&cxT9N^LOC=M zX>zoNaoq@i0N9BwJFcEnl^#fB}(XBl7thq;$grz$DaroeY_GJ|p1i!Yk z09b?0#={FoCsn*!i5Sf!8c1Rs-UUuKguW3kTA^+S^LVO{XvQb-aJ8550M)>^xu6bm zgE?BNxQ^EusSeAt-Kxo3#Sro=2b#A3&rUZAnrDd4Ti`_K3mG7X1 z=`NE5TOi(%nfr8YMY7Mh5&dN9P6@vot>C&w)`zY=Z()x<6IOo9s{PB z-PNd{$iN1%#*NpnPiKp4w`#sZ(&U@ERYM>V3Ff6Sae>MaP_hJxJt-f~Lkp5|yaMYZ zmtkcBBp`Mvq6+;&IP!nW(Ar6U$5BCmrzUSXerA6IL%omx)dF1f5Hz+#-3%NRU1 zz6@^zaB^@*?1NLp2!QupXxfwZ zy+l8~)SbnoN1#+rdUZgjDH%!UK}%|qatOx&FVH6(MyePY5QS{dX$&N)8*95DtEvXD z_(XKu$zTnhfi$8y<&e*s+KJn$BJAXDm{oQlFvH8kJTmAB=+i?}Z8nafg9!3(Dk~P{ zVT3U^%oc)6TD!B?;Q@SYDDUQQBMzr%M& zN2-W3Tn!l02lR*!`_$H@vvr6*4gPkP;mLD+y1aE^+q+$N=T5gYpCKdHZ}HJ>`)PLL zpbL??3g}EGi7Vr9h3nSou38~o+;qSTfz%0LOUVe2Qo`kG$e#yD=40B_)=A77pxy)! z)MCo22VtDa7UtVk-Yu1*thXToxnGq@ZDgF6IfMc?8?2zWoCXT6hyLV&g*G zT4XD)b1*teh+cps8wkE^u|M5>Vw(ctur?vE27N@~bPaNvfjH;*c3jkU?hlt1jW_z% z0g@afuYqe43M&f43lXkm?m=J8E)3iUGTn4dz?Fb}oXqPwLu_}u6eeM6B?Nn(a8aQqDA|FrK)@H^$-EF$2XKu5F&EbxeH@{sG=33jYS6ck3h@*osK)2TmpXSPZQ zo(18{>)4!@6+bi_OuDy;+XoVKzW=Brw$GuWXc?pwDoms^lPMWF0D~2-t&vQS+#k*< z4+WJLmC)r>&x5>r)QC%^VMHd6_siHoWbx?h>1e{}DxI9$%%M|Z=g4&49JXvuNjEwW z+A9(;?`+rm$_(X{cI9V7V~BB6?Cz*QN!DQ(9VAJT`SD;Jm0e|+lLhwPVuNr<%R4Aa zf}d89?m zw4O`{tK9@QKdbj#SCvsE;;^ctyz$3-*;0797&1|vWg@L`)0&ISmVuLXK7`3JL6%fI z0u{pdw&!@L%$`H$01Y$9Jp)-EN;2?Q=2BB$A4$?0^w^cC~%)8bxp7w<6pg+wzT$iV5zK>`E3%eZ>kZ(ryeb9PM-lK?+hOv6;Zikb`EX=i44 ztH5t<-JvGC9zo7*9AaLHp-p-9>`X0hC#OGNh3KksWHD(Ni)bMGQ^T`c7cmc%UQSrG z;UFBy20@^%?WsCk$_#)60k^SSwB0mHD<#->AP9h66+{OqRi}k_4-1~qmEW|HQ!VEG_GlOJSwV=`3UEeN%%bA<~R1Wo~E&{a?@?qi=# z)syu`;kxy@E@EaZQs9mED+xqT2ul-XqnKfd9fbpe0L1FGBF^b7ZjB459m0}rIr3!9B!PBf-uG#0 zPG#R+$vK12!fv?xZHm0%(qc3*Sq>1_ zel!4AShccUr&*Zmb=$4w*{MB7+{D&?xTOYiHpQ9H?ZscbJBSA&-aQf^o+}Ivmd^9o zS^IELCqrsMRud|54VhiAX_{9jTw!9`9r6=M($vN~mXFhoNpAsq9~;2^9zfn)<~Iq* zvg=}`giq0YKRlh`fuCILKwm5XhtEw^^gS+S4`k*IdpCEx@ejjx0hds;>+q7pb*oQsK+TlWXTL{I|a)&89?Ye@k z{k)9>Lj@Vzz{(k_e6e$~I)MbKI7<7G8XL|BdVOh5@HMET(oo1cBUw85U-buPSZ;;WaY{QA%z^A`+_s(HOSOUEj15U-pe$ zGl4Z*dDpGpSOFcXsfu+1o}}JTQSk^wS!Y~i0qN@1s!xbur5?G>NB$^R?@Z~X5N(V`Q*~KPOE?1~nkNVK z>ibIyJnmapk2XX-4;g~*it~k=(Y%$R6wN&F6vamAJW4LSIcB zsoWzaOFx|bgIyZJ26CbMTestq`|4epa^DSgf1gvQE}Jc-z*~$0tKk z#WP4M(a2uMG(@S(cH8v{>4r`=s)b;9l2;%eeRE@Q&WJJ+Ge@$UN#Ozvj(P;D3lZGB z_O&bJ=*d9+2%!u1W{}klEpTsV7sxw6L+k-CKo<;8@7>ATasDn4**$Jb)ZdgdNScmH z4aaKloGZy$J(V$aghA3MF4f1r-Nis%Gk<4bCiaeVG*_|_p0!gw9{`2?G%d)X{<%ezhm{cV1wP=}+1I_O zAyXj{D5KhtYxp>U4(!ab2JG_xpWtLI6M4#*wWshVKS_b>-+_0Hfbipo?ctD(-84LN zV01qLfm%tw2O^_f?$h$hm6dY&9J<6^B~g3AE`nP_xrLnn_h$UUSPy)5!(D|`Bz)8CI5`>as&QP@Uo1;-CkElN#lca;XcIvy~Z zen@u%+u>8&G6SWacYSdOkwM}>jSmI~H0ym^gIxneAX25dTVPEp&1zP&$1gx-3rv)N z_J{7KPOCpj14-qrM&m#kD?^SMIO>}$MBVQJaAeq=!u?eCfbZ_oNoov>5&~9t)^<{| z@o6lZ;~jeiIhD|ArG~Xi!`)(!qlD_j%58^%Z9=|{^)1Wg=~f(ewZ^x!EczCSO$xgA z>M$9S0}>8t=Yfv+N;1*dq%3Nx#zO$dgtbOF?+IA}sJdI@RC9se#kIDS&G_!^X6tk@ z;sajoS0iqzw>fN&Gf;j)rZA_27{aKlWK1grzP&4^lLPYDZnEn?F2c}mdQMUKdcb!{ zJZOqN?UM?J#g0v(i-LFRSXrS{;zK+J1GrWq#w+0$%z^2NTSLbm`0@+=Jmdv?i?; z(87TJ(vF%8>s^)twIv*y3rE<xkKrG9&tK%IZMZcE^vc9@3k2_{j1c@fs3nq`nb>lN+^vKu#vovZ z8>eOye53`()wSw^T}yDh1-}upp%l&1BEt$&NGwQ06xZcCNo#DB>vjI(i^tq=C=fIs zGJoagkix}u+XgmHOw%{KWNEB@#X)xyO``{}!fYR*^IeW&aOl!YKxXn*?Pu;;DZw~Y zZ^98f0m7%-5J%RK)Tw}+6&^TW0dfGiNuh2j7zTUb>fR=~UB`m>XT{G0IXBvMJ{W2# z72SGF&k;dd_xqsTmtqFzUErFeks~5mFw87fqyQjYI9+lO&(~CY+(F9u?JOQ*O(?b~ zeMnf2oyq1TkwE~s0}IQScrGtaRuN`;9V1fbN|3Y9UayUaqJFa|h_qZPVaOtIe3q3b z(QU2ev8jM5WCIZ4-oapYyzV}>HiqSLz)zhc_2sqk0^H`)2gqaFl#0$F4J^NC4tH?k zg61a`z+;`_tQZ{6tl%pu*K;*J1oy$wn!!aT$OL#)6PSip)`rK~;8RbWd5@}Af^s+$ z+uHz`;pxp(%V zzN`aCzX8;^VDoUmMMP%C1%!-;$c`((Oa#NNzEn%yj=)7xj&%PuRznVkqJ)X_v^ya4vkav4G!sx7Wy;XTXb>PR~N1z`f8+hJOp=L(pe8**!@ZA39T3ed;yif3s3-S z>g;yL4&6l}*dDs=A^7;pZx4{HuEPg6Gu$2pclh7{l$z`btRN^2Vk061tyiGo*$`y1 zmiKmoyjh=!BY6Zn)sSu~)va`35Q$7k1QTkhl-eLuZ9ttKvJY1fsFg~XX$Vhk!fSJ} zob}jn7|3@OO3gj`4%eCcLIw)$X_oq+V5t*eiSup@$*UxV1kkNH%r@af&Mn$A=>*os zV)|~AodLEeIH0_e;kg~m$#B?vC-YJ=WA zLJnjf%u1Pb-G+q*2Lab6s+ zL#XS8rU#(NQ!yWCy7!Q_%f|j&d&@2>$9ShN0ThN0k1dR2ql*wj)x9$LGYf> zD{MU9;KMw6=l~(d5+8LRr0(BA5|9t1z}SN?kbzvPIJHg>l&vcqYTHzG@|wC-eo|{$ zsM)ndEOxXeg&|m#bGCv98ql7imAr@JlAdyn0**EYH=N1fBMZZbI-qsoLc3OtKx7P$ z6fkRGJKPW8kW_LXUBK0XYYkbcijYTlM7}fi=7Qb`GqGbeaMUzKD|8}gkZ=x4(G8G0 zsOktII5Hh!K%&7xUA)`l2(%k(D?n_-8Xk3ax*u=enRXlop_Qnw%F{)TaZtv%OvO~!kYlb1{k+ZYfSrBo2jgw0pv?QAMHw^C8UNjhT5M((M zjsww@;NVuUUWYPE2*VKMU9GnkyAEfVN%^viOYD5ad-m!7#vm?LlUm@U&<52f8kz48 z{}L?0y{j+jdYu@(JR+)HALA;alk}{vJKosu(Ewc_x z)U*E^2IbfP`scyFj`{2-B#v<&U-qlt_K|nz$AA0>mp}Ew<^Ooc_l2+e4)O4#Kl^>( z_U%9T>R)$1k9y&_&))N! zzwj@<`$PXGegXQM-}a^-{*T}KP2cr*$AAB}Px!1q`sJ_tlpp^qhp+12|Fisi{_$6Q zR`=82{i44iee>V`wV(QjFMEajjopvYUU;G1~^<|&;x+kA~y)l3CzkKl54nOn#fBN>DzyA7feeIh* z`71v3f4}SxlkNZf)8F%^m#-iA`CRxXKOw#Oh2qn%mww`RzV(a$-MgB@H~f*)kGzBV zi2E=7$P2y&$=+(c>UU=3~u?-w|wuP`!o_-KahUm>;J(^ z|KQ7>;{Njo{{8De=L>)6`RBj)bx&G9MsL6I&ToCmcmK%`ZO-5N(w}^4dDSof#Gn3J z@*AET`roJCa{5ceUkU&B%Zrtt`h$Py^MC$le-!zWmwfgMo_W3IMWXrF%OCyr^56cw zSN}z_YnL~?boy06{D|kr_>}g>Uw_Yc`QQ1*7yQnLKJ@x8vHr~Qxxv5V6>s^Xchx`n zf$#hAAAec=uCICP+rIs4KIW}|k9qF$dw=c;j4bR{Mc{) zh3CaY{Om7z@}-miY0odg2VeY-cP5|k+MiaQo7&esIqyXu^(8+oKUeg`Tfg`VpM3ry zFX>`$hkWewb$;>XuQT8Mp9kSi zyza@5|6K4K=Dk1pnrGMW?|(Gt+EagC{4>wjdGD8lMyNOZ@Grs7R{4~#euDpumuG+K zA3eXU-~3ikV*k~@^nK4h`ppBFqHlcpUuE{;{FeCMpZT>{e)PvazS8@bf9lV2$&-KQ zXaB*=|H2XZJpG%vpZm+@Uw+|_y#)08?XUcs&#um!e-cav{KtRjQ@`j%U-#x;c;Q!m z^H~1t>Pz1Aflrt0SF-AFe#&3`&FX_6|79QbJ>U0pZ_+>Uk4azp@P7Dsd-tz?X+X2E&J2$cYW#?JUiNVzVdbE@wImNc`tYg z^y(MDe?0xkfBG-}X#jnJwSFG?>W_T)SAE8(yzj*?``Xv@ANuLPKq>$H{r}+|AN7e~ z_5aCBKIb{<{?4xglk=uudjFU0-TG+Zy}$J}-xhX0x;gRiIf zCx;vKmVf+NzwmyPUH3ltkxnW|7zx&vbpKzjLKvo+k}>DMRYsbka}xZzvmiJc zGX4tVUy8*Hv{M#GF73B4IocmdKE|N6ZD zewg8cJ7W!>oH zbT7ykn7RF-S{WakTb52>VgOw+XDv(|?HJdFl@NjaJZ2i9>#_s`8 z$5nWF6BKYnm)Nh*c1c!xW2o%Rb;1PC_ZKCXU;Khh6e2N3WtFIWcy;pFe3T8HgcsX7 z*)DKu3eo1wc&}%IczZKqRI&%^5?1|W@qD=xd5vQ&Y;*WdvantsQmJu2U;bUs#udx< zbYwUNg`6Msf45mNPk8tJdGRn@hUcXk>?eD3e2R`C(CFExp{AMwEsyhZG-=9oYDLC+ zShl4|N$ef&0S(0T8{+uK2L{31WUk3$d>}|$)I>II+}qpRP)cDzLHGSgpPS1dYr%`6HO^p*#eUWg-g-dHAmiM)Y%tW)g0Nqgyso9hu@Qt9}f z5#CT@Y(J6UX|UjTIl0F;oWXpjoJLP_yxtd+rfzLK7_82$TSC4)c`ZKMbVrTt?d?hP z=ETA&RkhY)bbaSD*79&pn`EqrEm8~Go1i{!#+L{F3mX77o}9mh zigl{IBL*X_&s4YmP{8{P-;6Vbrm-uEOl9cMat=!2^k$=)1!);Un!+|4Sw~Y62%j{% zR^o5qF=GJUy)4yy9-Ms~qr>hr7s{*6?g*k#5^lz>f>^Gy`ghC&}!i(ESwDQ}vM)@Mz}h(?uB*jFfNYD3>^AY)#2+KFH!$PQ@JFg4B|NSU+k$GF~b52FPMbgbGTG z6pwxg*Q4b1oi$2)XdtLqKuD?DGa$B(rvBY9j!O)~HuzPES;Z0wQt-(Ckt%Vut$mO-OGFq~Bv z8#dFtC{&DE^4#tU!z1D)(|GrG=rf4|1Zq+2&=x@{MmZ-@(mC$(qh0)3XB zzpxY;G&1VATLs%T`*fci)j+Cj7_6I(nPQ-Bt5!vj_k=w$DdYu3+%6~70|xwftV9)( zx%OyzfhsukN*VY}%2cv5FV?v{kG*W>>Ilk(czZuCC#=PnkEw^Rey!`LF*^=bjv*I~ zY*Mn;>Za;f%!GdL{$O&3R%%EHG5<15_sHq`L(JQu0mJ7j&GFf$q*+CHIN2YhD0GcM zmZh2c(~JW3`S>W#ljeUCN+Noo^@s^yV?n|r!ayyRNx-V{a-u{X(t|g7T*{%}q{XL< z*?MiYQdC@|Q5;;?9;G{#CmB;uEaJdJI&MiHltRW=DH+PN&t$m^-&OB7(%s&jWEXzf$yDX&o2_%qb}fkRR*5hsgfQ`U2Putdl0LiG@9`eryJTEYki_wahT%?H9i-2?}*Y-=H4(V*~FoYU#J zT|+2&eU8r)h&MJP45#)bcA^Jy_>}c*bkEawWlxe>jLmizz^#CWF_fZ-(AeopncBL6 z@u~r4G=*`Tnoubr$Hx=lh}P>bDh?|S&4B(f)fVBU5;JcfGWplQq7PdtH2bgly;KB? zk4zFr0-6r&$Kni;3-+>uGA?8Bw|375Y9xTn|Tg!67EQMRKmxR6^%nJQ5Fx$4e)~Qj3h2 zfr7BY^eXQH4k|DR!p$f;JH|helm(U=N~e*~gT%BI8clt%J5z%#n-WrBwWCN8Vex8a zUMs_FqQX?Rw9!z5T|8S{cn6*E+Hz}C#Nbb7*bsVl1k9F#N?5|RL=dY;( zE4Vf8Eo~gsckF>=G)tj$!`(l0lB`EUZ&TV$FoQ9OgMwYjWSO_MY|zA?p{N>lt_r8$ zbUq!@JHeSr70kwSwTbeFVrF}o(*T{C3_ruoVoTc|+ugJDGl%2ZAP%Qi6&mkTQbcz-=X*Fv*_jB7>%stx zFWI7^WA7D^`S=<_?Qic0*KD{SJ^>XQsJI?s`=1{yuAc49@w;y0*V`?G&(xU1q8*J- z@uyTlq-VtX;7CTIfdKZVNhcmlQ{pB>=v4Wlrsfslr#9Xnp}!cBB9a#xKf%R?$m<%T~?@GrcM#kTa=m$a2uOa zI4;S}fmCj-2D?mKKCc1-X^Li?vA+PAKZ;tV0R_lH-f_DD!)4}j0foMp5d51^L2l1- zu|^qbx;lEX=FmZKBdhvC+cZ_V>;x|#ij4hhczm+Y%5l?@Ot!PNC@3f&tb?Pw1qH}% zAvpnmd~O^ywb?EVI2pviXVeYp>4WWxNeg&V+F43l>mRk$=5hq-$6ovQ&z(M$HfAC}p zdqn3K<2lxcVwy6U$qTog;~yn-1-Y+#$z^ebVYXI3!2OU7HlRX+gor2}DhROD4^)SD zy42?Xg0_I}b#0PKQ`~0=tN`|FAu9%|KU`4%z~84_p7=?<2cKsuPpzTn4umAb6>_@d zg24PFF!S+*p4yyqlSE$UA31=^3L33Y@J<|{nMB>%+#piByqs+id;rWz@OP+Sy?=XEzGOG z1J)HT2q~xd{trG0FasEhSpW-LJjG@?^h+=}3zT;yZ{OGad1o*TntJ{tha3D=2Zj+b z=+833;f6j0E5Ub#>Ik*O{B^F(+aHjC%_+jFAIlemV)7luz+vSUt*Rj^nEwfg=+fYK zxnttyFx!MV6osZ*C4g(F|K5UBo!e`wzYg>-V2~0F^P~A`{_%dQU{{im-eR<*p+5i@ zF%0jLd>=3b6Fp3XNiLS@AQ>z9y@fEVy+e^d@&K#uB!xL446x<`dEWaBaD`x3pypL2 zV9NxMj)N7v^;AW7w(o5i@Rl6_s|`THh(8K#N5&*)z8jpHaGI_*eX!CK6@km}%;E*G zU!G6T@_;~OU^#B6cCHHeGwcu!S$-dYFbX=A2)X%imNbB_Vcx4NoIoMQPH8dZ0<+p~ zLD1w&EF7Io%9&0HgmVUF>D>+JH20Ka_#9~8vI^JO2Z|Iz;99dwE}TW zE5x1y4d90fKsn)|;4}DwKLRL5%lZo!GQ&b8i!vzybRRtiACnov^rnOR31kO^-ZC-# z1)Sgs^++VSME=%ARwr120$?tqJg>cfOV!YPM{|uWWWv!OcFOf8 zc#8@(1qL663vQHo$pF*HA%gQ6+LE5F_kl?&+ID^hq>>$oU~P{6^p6p69mL^FklicZ z%JC9SoaqH|uL1>h@P>bNbM1Mwf(Vt3iB91%ZW6iIjpK812AXTUPhZMc0-olr--RLoai2em9EGTbopu;!(dvu z%n`|i?`l~4JdR^<;BF9z?J%6&Gy_a#Qek3S-zvNAy5q~Jd@kUC&fb&_p8WXMe|~ql zQ^|aK)0o~hypBfj@a4{A`B&T0M;GUdzN_tbw~`@l-s;8R@txBsd~K0S?YLiM+~ax{i+*s|KQ-)l zw|Gyl*%xWige^LLrj+G8)ktz$4z^Ae51J8b*gChQ#C9tam?5<3QkkI9^>yYcK;hQE zzU!>FopT9)>aS)c?0wyzx z^WfCdV3;%`IPRW-2d&jn9ri?1D?O|IhhNo5_eYKA+!7M%(B2Rrc)@*aZYCT0nbW}O zc-+Hb@@zZoz!GYo;`k;?7#aX|xcj zzdg_PJzDwofLEyvj`kOXTy`cgysyq~q~=k*FV!6&g^H<7SabvnDbV(XF+hF(>7<976ywP$!k|^(%cO(^7k!I_vK( z0#EF}Msh!Cs>p zD)?Ex3VmmAj_AcS#1#T!Szbw!1t|DCG7uP*GeUvSlMJO@;ePMDIkYuZnXNvLdLv+r zo=|tczpz zy;3@1`|V}viPGAu>x-l8uAEEfY;ME=$z-d60fh$%I5>jE>C_OtE4}%1z_AX zv-SWsZsh}ilfBuxWi-LhD$5BUGDUrT1XK&WejTHrp;h&uupCk&CL7W- zQ7B1ueKpLDPA+_p<;D9Cm&b>^->Kb z4t88_%eRI`z9d}VWCuRuNQ78SeRw6wX3#P{yW0~*_QK?$dI`Qxy$y>r1q~7^+?U6z z(rmAA73xIXwWTueBV)=PN$JI%vO|g6u8$RfU;Vi`yjoh>7B3RIZh`oKF!2r(1*IcT zHimjP@H=a4p+>=^&?s#K+IpBi)*!-@FM-fklK@}E065PeaN*_;2q;; zF;~vck<}c^IFY{c%Xc`AUK=-GJ{f7#S2X71(TdUbN-^`{9^ck@86M_=DDB+qjbo|Q z@b^cm=h^v1bOtqgBsC^ogVucOzE?53oIx~ndibs^1v5O4lo={aMY+CDO za9l%3Q?o`pSXneW3!l*_iz?;))JMm3w6k0;m(3+is%90+%{1KMUdCB6RnEvKP6)3Y zA;_NZS7Uds<6-~0zY?6dRG!^UGp)h(>*+>Qrl#7ha9mS`m)lU5U^ymf3h%KToj?#P zsnB*o>Dw`ZT11>3F6;^nj*d?Ovu*cb%eN0A?w9YT4_|}oU8~JYV0*HhOk%tWMa0NM z?R0xWGQU8zP&tLi68DMSJae*RSX4ka8wCw{5Kt5f8b5MTAH5+BT!qFqkev+|2$v0} z+n(+nE_c4D4QEE;>g*$g$nrl-atZTxA!OGH1VKg1oAY_jy)Ls+$JL&R+DMjdi}2(g z;BX?P#JWQY;#P7wH)viWKxA!RYR#6S?=LpT+$Xw@D<*#XWJIAkYI za;|Tt11VfI`!#HHxMzV5w6kHtE`duXjr-qQS%b3e+kPR{Sxv;lCmt??zzAxK2)$|s zv!+l6y=B{YHk%-ZZK=tIP$O)g8QSg88Hm5X|CdCTgcEI)r$+Xf**=%@S@8@>I$$LkD^bl>zBu0D>?Ck~ z@m@YC>WVM$V=^?Fu4L%Q+lp%o!{;($l`bC{B-Hfxwzx!YGK%N|i5(MPAFuVTJFb9= z5TlAn*#hqUi}%l*G@`V!#%nEA0kkE-VU9~XJYKB2j!xhhHp6U`6mG#3Eq1rpu=wHS zCwLZO{YEG0aOqDmJVfWAQHkA(6pmqoX+PKdiOj+R8MZ@%>o3i$n?d3Q^}V|-(r>QE zAMA46%ax|6@{OI|((dAY^%=orI7|OA{^vw~O#K#4vm=D|+6yndW<2lbzi0|)Y-HGlD%l2Y46|Vh+4Md>A4|0RbiC5^3UzvkWA9 z3IIGj;(?=!ROvo021VJ|8{3uctB-~HQd4_#4ePrZAe(}5@Rs-e>5^P-er;wuStf;q zgt+!;NS*C@ENU#$ep6;sxSAIcU7@c4#a7>7R<|NluhA)}nS3Z~`g2oqf2A7UdtngW zI&mL;4k>uGJi;Xj+YQ}WH_N3dSDyk~&8)i_=RCAg`&Sg%V@&75p#aTSoIEz8Vv2}8 zEyX|Ace6AybtKr@ai=1WxVb#WO%hA>l(XCz0%^hpgE=G@GF5*&Mj>oB0Aeh~Bk+s3 z_SJ5a;!9KLJJxW6H_PHKg%r^JsIm7%A(kJ& zrBCaTw|MB3(Aj?I#$FoDo5-!>(^6jp=|Z#2b(`$)iH9Bb!Bl|26W)b*pbmTG=G!kM zxWJC@qX07_P$=)>+GvRs_bW6cp~E_P3nCXP>ef;2k3g-hYB-co-dO-_V@=`Zg$dSE zUtRSresnqt&*)BImwiI7nD}U5t}DJgFbq#8*=ojUZqyd2!GE|ppt;AJ*;OTSt|I)Z zy;{HC^8te)#T{9dRFyopoRvmx@%r_`P9ud{96nT)aJJU+r+oep=y6*#nP|bXjS9j5 zgLlXQTax~*palIW2q)ZASmK-2^hUWhTE$fa;0u{tvrIrG(BXtdcrftTC>$TccLrkw zT+oRmc3YyZjE60)U;>`Z`vl;Ge~lQ8C_rSh=c{1J%N!~Y0*3eHyS?vK?+FpW@(I-} z6Mzu#5l${fco5}~iZlFD$`hV?I6d{f)teVE+DFE#g zhMG5$(u)nkQ|#>rAdLfXGB4OzN`rY@N-FdKmY%}HMXTu3L_~u=S+w!58h7v_3e-bM zRoQ9A?Pp%G-y5BK_d`L_7;|-gpdchM-~o71q`X#vuP;B=cM{l4njknY#rN44~bC5Cu^y-0Pwm!@%UurF`H}|Bs`Q4~l#v3vAnr(D8 zUmHv(61}~kQ(b-uZ`>0Z*NzTynwG~Z{{>mf=?y6uB^ePtMNspC;1?Q(-ME0J3a$Bu zRu_3h1HS8je-At8-260C021lN2JdJB6`{>+Eh-?uIw{8PPCUy23H!xz2o(&TH&y%J zyygf%6ik4>dK#9pY6Fc!M?}MifOZVEY_J!VOz8HsuC%`a?NJX(qEG0Vb9Q$q5qT3acq>!P< zs+*yAI6RdL@B!CIP`A9zyonFeBs~3s2qS6S(S0`N@e@4 z#L3oZmCKJ8Kl*^Th+|M7{B-~tMXR%Q)?N$Bn4r9rgi3|QvTAZ%U;iA&&X&{xWkhYx zvveTLVEUZ%kOCT-5ten>>f|83ZMBf`s+jD&eT^HN;`&Wyj@>Ce1j_F|Z#(>CU!Okx zqUSF}J*(xo5cidiB)(7qYJCt*p6w)`R9{TU9keh%U%Q78YZMn7Argwmr>1{Bq9z^h z21KTY>#!D5@Ri@9UkkiwK$T&;Ot%elj&HwXJt8@d&fN|Frj2;}NyYsJ3o2xC5T~m^ z&)8o#Cgq!{R?eP@+xaDmc^Qp^_pnd@a1IJ48P-ApjY{bd(cuUH(68t zE)rmXl<)XS^>!tgTn7aN+d4ykR8wsEL5@c1@zVjH#zc?Wttuk7E zhQ80M6;8DxJi5Pp6Y`P5LAR}bb=s|wGYZ=UG-}a7dC6(0au7#puAEi1QWq+?8XVm_ z&*EHONp#5J_rRQU308}8Dae%F*ka5ZH_|aPqhPS|#UGd+4ZO7xs9ruf`u5THK%`6f zr^fKR_8-+3d}bTOYDGHV8GVDccr4dUEQRb^J{}INv#cs5a404A1Gd!vDhgbL+a~hR za_4gz&D^rXFL6(B-K&ey;Qu>k_Qk8?nkW1s6XaF>?f}+aR~@(qDkL_9i`+h)5s=dv z;_nf-V(tkCCY|c^&9F<1$Zf_cqp?t=xpjRKM&@!I#c!02lIFo#=^ZdMlY?w7%()*M zfFfr7jNnTqA0OeMP9)^MOs?0vV`32x;`xOI$J61`i%*6A_%;%NX7Ih>hHQP8_*JEC zUbabltCuI#ai>R1(48gtoXp?Kwe;1Oi%%~)Jbqqc%<@`t&S7LeLYUZ52bD(H7ekKr zG3aq!L-FOz3tJUW_}x#D2wgUDrgVIC=AX*s*5l?QQP6RmUv^09J#`~E_0$~mO2Y3g z43=wD6%q&Cpl`C9V5hW_epG33x!v{kARtFT0iOjV4j-&{?1@Fq$-inmw8q$5i%YQ3 zD@QFEuZYA+kkyh-c%na-BRPCF=?18qpT6AEaXEcOQL9iV5s`klyiu1v{qxE$U+&%K zS-YN#)Rd=I$4pS=bGU`0RTZW3y$Zdiw^j@#d2)TUDRnlCCHSS`*2$HxCZgYcdB>Do zw{X7Py$V_e6QLk#(^DY%p3n@QCA<-KeQq zm#YO6?Q^bo$hr@|QCNa$!B9~#9i-Td!CI(_N;&3B&_i>t6A?$zWU%sM4Sb$|qFU;e~LxP`#*eIMPTkmnUTR654Y_I&u8?^0%7s9t=X z<6W?Wbms6cR^?@AQJ(a=wY};0uzWAK$?qfYBF{$f6*pUtRj^u?sFef@PY)t@0UIxMQA}M$~1n0e;(m>2wa{oZcDs6vJpCF z4fHjN$z(_73z7S_8WIE9EnRV<%5ZI-3TEB*8d{u zbo`y7!nd7_qUCYs$m{tVEv}4wYbn*IHawwYu~22TqNA)OfEItC1nw6!H>uJJ-Gs>#ex`&D@E%Xjv`F~X7@ws4GV7h3A`6)TIG^x zQzx>fL$z(bkiSvMM0M`Xi&P=8Zf1(f&av1xa>83fwB7`1U8piLC#4v)E39mfjoUXc zmgm*bg%Qh3TQ5x(JLn-u(27O0} zynWybG4skzqCc9w{D}s;L6Eqx*JzupQ0?C2BqKzY&D=`6;k{nHDJXY{z3%pfx38|L zyYoh%(fqHhLByZ*>4t3K$tp^P7*LE9C|Ea|D4_5|qb#f&Wv-vXD{QcJ`$6tMZ;+>G zmZs?c{uJ`|;*Jlo)jKS6GbzXX7wTCGs$(T3Z~lNb1xE1(bT%uRdHP)#tox=>4cINL zR40u8Vz>Y8R8R6g2fD){EQr&$VlH->51HftDZX?+@%nPr=^@ST+Qnn!p-jP$V&j57 z<2F93X?nh=^}EG*-<=pEwCJKaiJ0`mN_#u@ffuAC)3EESP&@=xphPb*O@@9^{j`g(aEb+^5mpxbFY(g`16G}1Bk!j3{q^%KF$q<$^U@kw~^MLcykq%_!z;-Q5@|e!m7zlKfMk zcasRLANG~&53N5?$&dTbaKgWjueFW2LQnwDez9HhJ9>8>dA2E5W`~f!lbLdT#tij! z%=Rg0TIoG?$8lWg5w9no!4{S|!7c#2@~;%E%c*o^92w1iTUjgQQ z;>Q+03stgcYs}5^6*I>;?j(9ow?5H%fyQT_Dr3xxFXnc8k}eGvMSBy1&(1^K6AXCs z$1N!$e=8j(%5)GatO=_z_DAW0p_taNCcql3FWqvF2hDJpGn_SArL(;+eD^3% z2B(;tOmDm|lPOuSQ&1P}dXuqYK{NoN3OCozKRfZf!W@Ie-9R#+q!FNpueNw>jbQD% zcU~~zAh+iJr6mXeVKH*vi~2lu6|r*QOVQQmcY|`hNodp^S_)CK8n8>;`SODtSfYDf%rAsFM}iAyun4u>uNayfUu!xI$)6 zrQ9DhmMPXfh%TX-a0EMmwR=id0iuA7Fu=Us;&qC94u3$a{&hSQPbfMU4U~2;ZpUYu z4q;a%bx?=EdVeTh*v!)DQmA;^ zI77Nm$=c|Uzua+VwUT$Yl8>0HGni8l0t)R64?tG~QBnc*33f&?N@iPUz zgB`MXr69f7L2^tkG_V=9t{5uO`8GZE@kaMMO0?v!F(o3rk1S8rNZPS&or#XSes137 zs?JpuX_mr` zd20)9K%a+fPuI1p@xkT?iy1GMgXstB3YuM|IhrX?jp5HIr=!T=_+tC$@VvgN%k(qa zV$;c>TYKTu;dv`&3E-?8WhqBGqTZCj}H98!I ziWUr{1p*Sk>{yrlk^27At=Tr@YO|pNPj$%nD20GfBs|bmh+mH`*hdD8rm#7XKH9oL zIy~7*OWgpZ&;v2;Un`5rFMx(DE9z!i*)J-hJw(Xa$X|6o4%MjW@$BH853kgR9R`jW+tteho+*%(>k$Cosyi^#{vhk%GEJ#BaQsr4|O z=fx~8yt49ehQ(}-0Biyyh^*iG#`71gWqkfcg$>QG*#^WG$6wpo8ts^OZZFQ)weelm z)J69G3puC*b`Hc3DL7Wq+hdy*A5svaScgKRJHulrWW)U`EA8`VL#AE>&i_Z(oupG+ zn*GpH(02He{Q=)A?VZ|K47&^)G}?motnMHFKcd2>(b8cyq*+HllJc=K8b6q@760iL zRizTjBiWm$eEQ}|!vkv-3cx<*QZU~i4;owW3GjT?7cQ+O;>+8{t|hhKT=S;$najZx zNt<^FxHD|HRI%}eTmtX=0)5bs#74!X?*PCq&t6E6eligG)F3+@Xq=${CKaZF!~=Ms zMTqLoGbR`iKr-zXvaemyrr@!NU4SEojI(U(qbrzIe(2F?dgiFt=tiQe@p}}cy3J?p z;?=mV&i!{^H9w74alf(_(v{A1ThKB7Ss!Jg)xdS>azk>o(%lOhv<^qZ5profsr%f3 z;54nj&y#Z>Xn~-Z86fz>_fUg!gV@7?m-8ZMe&TV7@6$4xaKTGW3eQM1ID%m-t^Ud; zmmfw8{4dA5~{BSpja@9x@s&N#tf=tB6;4^fr4DWO7;W*Xm6rqAxp zemq#ME`MHV7{sf(c~})tbZ*RU?JHq}%k9yX4T!?B47P3yo$H!z%d3NefEbpCXW=FZ z5J?B+!x_9n^g;#=riHY zV5BuNAd9BlC+e~fk$x4x_q-IP=^%v`b@qN1K-oGfBb6Ti1t{-ofV|FV=Ft91F8j&- z=**|#Kb#B2LRkTQJ$WUfe2)!2n!lt)_lmMezm*b(WC6#z?jw+^(q6Kydf(N0JVCY~ z;#+0C%_6-tJq_cJ|MNNYQZKTIjr1|&y#$65=5%x@qYnV$vTeZ0n5#v~^la#*Z?`Pc zsd|HwHRon4J=K=Erbq%ygwh0F*@Y@a*xE8h%Mr=Gk5FLC!zH7704cO7K28z#X!E)` zDD*Yn+7f^91=Z)>LjzJgStaSW7pN|S0>44<8fHLR!jpDr0UY#>UohTQTZTYD?;G@E zF!${pDF`=-EuC4UiFGiZ;72)J@*Ru_T*vA9i~hp}F!&GddGHeUWp&*`rz{p|n3**DC6?pm*)P6 zajgB9jvtAS;bGSPH9Rbshaso8<{3~ie3@B(=}bAgRSZm+Yk(V>0n{)*W7xjY-8B!? zW5nA^xp_RjU|%%zMap3C5zbNv=}-w^LBVn&)VrTA*ut-=C!drwom}`Ae$|`>e7{F9 zI^9NJT~^#Nw~ESa#S9V5Yg=p7??}Igy|-7(6|fYwe&z7}WwbKF*nMk=Igkq=XkMpG zGYkos3G`CGm*b-EmHu~{lEMUEJ2q5ex@_t0SSIywU?w?iVENki7=oWjMfLz zc5tq-p?dytHoB56hE(uxo|gXQj4y9&6sSDkWzd5^SZwb->%O%RE~zL;1b7X8B}=60 zzouEJ0mwe-PY5vWHS1+QThfqwuw6Qq#ASI!1KrsEH&497gg>QY5$m@5{i-IUK~`Ex z_&*9hm>_g;z)4lUJE*lr|9w(DMN)5YaNvZqCRT-pmPhy6wWu>aLXt5dBsHu0PC^^Oq*1Dg<2Vze6|r1j7v)cljB<;M^Ye1AzRXb-*V)77-t|QB{pn+`rirQ zmLj_J(CNX$wiJfbi|?vkD!4@7-~FWVXrgZl+zUPW(drBDaCNZr`slKegDb5ufuC!Y zR)1vx#)ISr!!$gS3Y{Pj%Fr!^VkzZu<%n*hZV7zRJP>9l_5b}|ob6ZV(kNTZC0cZv?PK1FVa)m zdzh`RT4-3abV{0S;QJb`KUzr~)LO9uLG|gP)=2$f7>2$Mn4l(-y?MJAC*Z;HKJNMd zg!((gR%R8?uvLQqmQucv$!j1|I+P}0d{y3i!Lzl;s<`lt%43X3I;~6F<^J*ho=}U& zPb5Gv4bJUp!^d zZHpT6yDetw5N~@^U{>=5xI^RmH22BZzsdxyHzM~MC|AFcTzMUtHXX#$XPp&#AQsy4 zKsGx=u$U9eA7$O0liN1eNtYw>#yyN`Q_Q+13jNuDm7{dz{}m0i>K-TZNCBO6fq&y= zL7EA7qpIQ;;B~4vXchf0%Wk<)_u9E{KbM~Mk*jaxaSYlwibSBNC|B6^?9H+km zfD894KO1P(`s7xyv1PX29b%Fx4~dCpTYkMPEXw;cokN@nYuwA2i`|XmEm0c^tJY}$ z&Iok}g{)h=G%|Ued(L!Yj(+^*v8_`_Ur>E23#)OwJ^v|}!NcXW*%x&tJV)bPKTypX zQN#_f(0R;{7qI=(y*2lY7wjng2gWl6P_HlF_K zb<6eWec39DNu%2Baa^JygRbgd{UcSH$cbkx?2CpCUKI8A^ZxxroOH7LyUdr~T;sNS zviqCPXXDyzDQ(zue73UluN@&!8`N5hMe?l1r#KMul27G-d2eQ5e(L{DnmVC^?sf~} zGjG@P#a?tY->bOwP)Flix!fD)Xe{dk>CNNVA-c@EC~PY?%0+*zDr0e*qWK~#;|u{0 znc1s)tydXB-jwGD3(3&_rUmQwVNs<|a#cb^Q0#g^H~OJQso$;srSV?1Yqfsc+h5qA z38F$NEZ+Ea{n49Q<_r-ZQe&i;ckqc@lGA(Zb9nOcjA4&3CkQ|y#tZq&eFM6ubU(pg z2h|~V*yuy@Z-3Fs+jVtH(_yYoHDtCU9dlGn2uP0iv_qf#oEnh)M2kpsRyy;+>Hekb zC5d;)sd&H!2AIokx6ux7G4MEAJHm6ZUyvMX(rz?n0EZeR>4}wV+Nx0mX%|F_B=sZa zGM39NG{T``f!`Bs#4G+*P#-P`mFddjuOl0eXVTK%CpruAqm#XFaMyT+we z*9OQz0dr)3l=jgkOwv^ebO)6Z&wf|4Fq*`-4wD>My!nc97(olJe(9EyOv7EODPb}A zMmlBeRT5P6tz6yRX6_?~Yx=nORQ7N7+3Twf_S=@%B=QLis9dUrroBm9^9>2Sy$=dF z{jv{;Lrt~cS*r)9oH}n$vox+fRBGa>)*mUQuC*Vd3^}9d0UTW?`DD|AX!j#~F3)b8 z(pKd`!Bdsu4xU=EJ_#LT3<}^=Ghj}1%m1hRUx3>wnCUt= z!~Za!EuZ-fmlV8Kvex@8{h_LW-b3R-v&k0JTJ%+yqjYU~) zuF=>6go`q(@TuA!;0vv2wnGCKTHfG+4n!wt*E#MT^IBu}4pOq7SEC-Bl#W1$PFbfQ zja#y#vb%K&?Jq@4`hSF}%Bu94>g1YlYnfeW39_Zs#Xc?23*{SyHhRBM0CY)-_GRx* z*Z{wztKtvRbQ29<2dNUR$V6ZraUnt1r*p%fOBp`=!)b*;m6xvCf&R*+SlsT zRl``3C3^ZfsZN&`-*r2wl8H~u;A)Q4WH@s{N5RN2i)_k{8gPi^;sm7u^v~{C#GxE6 zSQ0h{?mP0c1nEbvC20zJA^_{eNZ@=R`9*nUDHd|7@Ln3A<)E|rCQ_viP&Rjo&$afa z-OVbWKGxGSs*ilFKTv-p+P-K8>8xF>SF#ZdSJKIz{XNgMxQ*OoLtzvzzj(vs zcv~<%yxLFB@4ii55S_{x!@21767NMmnrFfPBqb8^jm|k0hofoN`PQ?0l?Di@!8LjC zQ3Smdmeu)ULvK^#p6cT+%#V1+`u_L%j~|4E+Ct%Y+?0>$`+~hl!l_*)u#_Hc=}V>| z%o zabLGh=DRsfF}8QOWNcSTR8ip*tifC7yTvtIqxFSP;TN33OkW1{UasuG8}=u&ak~a` zer9rXdDWs_AWH;~0&{EWJYv-nU8J8IEv-w+1ae zokA)OKR?@g{m8^XWL*n`zW=gNuu1JZoW0|?vLRw3pGw4n_nO*m;A5O~QaPQ{()!Nk z$25XTqQhc^+zPi63?LzsZtiKQ%-z$qnDG!kusKvQD4c+PfN%Z1G9J^-nri zTahqTE6MbT^5l#``TKN`qP*~t_Ymnb;mOkAM8Wa^uHTvf2iN?F7^VrhE42H7It(~= z|9Lx1Ex0&=9-}E(93rbZzZ(gv(sw9D7bB6Oy$d^7ADoY9yfjbAxb0M5mgWKR%1+YwOl|k|1`BQAVwB~{%l-1>O|EUt={u7&ioT7(U zRfhj!GDHBwNqW>E62yRD( zex*JobXB{goK3R&Z_^$d_5V9zxpYoVP3;!}fFSBQ-+g>n7s82Zv=;9 zJh*YE5?fjjTVsvI%Di+*OsCEg@Ib0z9O{``T$2X9O36oALavlx&>|?isZM&orO~nL zF4Ttg$qU8*p3xq+bza51TsU^M7`aAXJH6y5lh1h0<#B?4aks&FrJPQ~ZpBv?^hRl0 z`}`LU*+V=11_IGoYtBvT;OR6K5^@3Xp`1%RKcPX`$nS>pc{{a9rhT>UV zeO`>C_BqXu9Hj`ER-AUje@Ap!-bM6Kv*=+XsXA-mb%Hs{VO_ z39d0D{eLKX%djfDc5PG;kPrmv?k+((r9rwIq#LA5Lb|(@lI{-aZlpuHrKLOfn9uXB zZ?CnFZ~fRmIgU9e_dUkA##QHao~x$U;viPGU-N6%>1HnfX}8kFYldcVJ~xx&cDF@4 zgvCI;82SVP-Wkh-pUTZKbh30-K1QZdLV?A9&;WAhl5Vyi#3iw;&L7>s_kl<$?=h;C zH`{qf&EfryxGKK1z0};({~G}Zn=#3kF1CUQyh&n^G)X0zT6NJ89>Zl{0laqbhS^Rg zDe9M;tu@1_KUkoULr45?H3yV#K9fPJku;iNI(dcY^;_~Sm#JIfaUzf5XWq|qZ0Cf+P}B)VT) z#H}nbQL_dAzPSBEN&OBy)!|eXNVB8;4ey2@EB%^p3G z>Q=0i!?fZZK3)4yu$&vBbTMpr_Sl__GT7cea%$f<+KKdP>+qY=tLXLq?<1ugE#W@& z@(pkQ6Luf2lPBpXmgrzXfGYZe{{mI8Y}W<6Z*WSIO8fEznFhb{88#;U$u8V{7;D~V z9xwjOEodN)stH}1HSpHR6Odl~+jMk8z4;Kvfq+g~yP?PDi6DMOO#4jyg`-OG3?PeW zzWpEubsag#Yfa_-vJIx)4-hqb3*OR8BWeQHh%toNT8emBe; zDV94lqGroYD6M8G)OR^Br!N=yApL|Z*Kdp(5)(v}>pFgbJtx(clW!EN(I697>_6)7 zUx!oi`Ko3!w(h&@djz(uiQFtW48Ebb(DC`mD`DuF|BL0#r;A#}xFn5j#y&JdyY~9! zg={td?+$8}u}ok2}QAI-yzjn zdPe1ej7G?peG7j7P#G5LeJGFNe4={3-PYNm*a3ENN5K-D7~vmDix}YZ7(k`^fqBZX z$#VRFq$dLHpZ0R4sIP&Fs09$Bu&66VBaI^v^xW3(^MOkP|^|5|Sw zoh_Kj{WaO>{+9}Xr&Nimn975%iMv3OM+AYsM4+4wRTMMy?h%!ForFT|Ff_&_;`KM zMzg;woAw9-;fB}Xf_j$z(@FkxljTkrfwL7%Y{{vTv6)AYol^l+fbcny1pd%1TI+|% zetiRtM(sjC@iAp!`@YLcQjywPQKCgF&&33TRuLwdSueIf?Uf+faCLtqBN#6$Ta(Ch z-JK`0X{xVsvF3jKC=-cAtqKd<+82b0zvVkz-RW?`@7xo+jXkC-3g)r$knPEmHD>F+ zbUM&()E-aBjsDtOtjgpQlRiYI$X8oMyg`ySWsa5j9$vEOYg;%~$>CzR>MuZer@`r3 z-(2{@oCODcs@FY`;>SM+=2!A9h~xlyj(VtuonpU(R80zOauxP1c#6S6RD87e~U zMdNa%6zV^5-+tef_;puzbGcZg+r&IJX-Gi2P4A0Wn&5Z15Q+MP!WYh^2POLFGQV$%z)#E(4m>Y5P$)s z7d^~|J+JZX5oabFrCC~5;Z;hskwyC6Bteg+{T~09PxKaRBdf>997DoaQn7sEt1RQxC1~4ii zJi|$hB9C{M-H`?!*Cr$!xDp+qTC25XeE5Bp$k&7V6z&cf=r_FYvoaV*&Fs z`3q+K7SX$iy}TC~cODmOu@5%dQOu45t)4v)Y!dBVh^K(7hnmXES;KeYiaS%*g-_6B z3wkOC!Pnf(AO$+#9zDz`!IPL9nqXCyNOR)`MXCcE)iBk>S5yH4mFqX_=x09y&s2wu z4og`o>(a}z<$kIySJ`ZGaEmk?jOqSSvbP5lC!knrQB9Fngsx_I`*Lj7!3;t`$B!^a z5AZj)ZxPU`L;>3Hozt!Ym^f*tq)Osa3FY};i8Z79d5BFaE$*tQ1p~<$Nx(Vb4ti z)?LT?qei1Gthe>}>)SJr`K{xuoLBGMw?#wRCY>v z7sDBKilr}Q$+d^E8h-$zRkKCG0(8qnsp4&5?oMD{kv~J_pJ~-zN#Fn>4lVRGl1C@~8K>p5_a z!W)h9DOWagcD6F1Zpx4V5K|3Jiu^+sLiMSlklYZT!#d@G9L@#QHu`@E`3r~e0@}a- z5Oco$!+DD+xxRH8FQLSRCLIT;H@;HRV`Z(b=_x&+9*4TTArj~)TZB5Rd&wI$;a0WOM*Oqap6I!ze!PcC<7q}J;`F6Eu6 zo)XBx&OlF~(^Zl2WzWT-8QBy6FvHcQ7R;DEIzUUV&^X)QipdyEVf-r&O&i4)#bC$* z$8QeX3{2ggy$cLkC|hiuHC2>wayj3ZE!Az4V_L+j31m<|$y=@OGqd9bp$Qne89_V- z<4>Ov0hJ;lO9|QSbCoORxGJIS%{>C%(pw6R&wkAq!D~mhy>$&DCu?|YFs`&}I{q-s zSXvYxVuRKg<39L4|H{9P4ZzM=_RL(Gcl5wpSxirVbU8X|J02a)m5X#eCfoSJ#mg%} z>dHXrESDw_#gU$hbc{vznYGE}t>{OQThAc1{If)X58R8ww=UF{BqR=cTzF* zIFT!YrKmeM>TUfH>mM;+DMI&CV~c&#clPpcrs*e2VJH_^twzncnlodKYK48zsB`v8Yj(quU6_^VZi-~FI^!d>m@HUi#u@jl zEMGaBkzQvvVKbHg77b8CS;A$k4X%RlJ5hQ;CDM4h{7nMvUK!V}xuYRpb0+dcTy@-Vm#Zs;u zU|v4i$lSpi`I*%R=7562!^5*I;o6^Lb^ll-!bFB$ve5X`7y7@5=he$-*wB%G)%ZqY%qhlh{U#E+6U z>suJw)aRF}ziAYBem2B8o9~?BNTn0+NozK2Q90*FfAe2RPej(y1ygH)Q(=xQHM>u+ zy5!PF#z?8(FyIe*M{d*?jVH;tCBYwmk_u1|m%? zT4+apmE)rqIfC4ltpY9fvvE)Ex@6`AGWJWRF4s2Z5FKu(^&ceq5oOe9TnWW3fP}+* zyrnJVSc5`(f@Tmu_fQ?+poP1!kWsO&(0DnHI-YdzVM9fY-8}TZ$DmCnZgkx0Z2KE*MB(h6Yd1_) zWg=Yz(Ob4bAQ@xYP(Uk@ZI7Eu_igaoNiP_<+-UkR=3NO7~(Fw^V5 z?60S=Q*FuJ)%;uWWlODESYdgo|3H zDbtE(0$Wc|j6eCD71s2u5uoZxwZ9|J_Nu`6WxyIfHr~?2R1qxcw&`YZszwo9b_;il zR&M-e*6-#I=GB>M+Lx{UWZk+$I@XKyd%C>Hr*NgFU`*8E)mg3U@D`Qy7a^llc*Mpa zdY|+U;Tx@M89dQdBN4^@xlxMm>?y&De1?In2;3W@2ta`RMPhqEY(FA{3mOuzt$BBk z*PQ>vp*!HH=&Y>r7i~IgTfWB^+aGt%pG{8!5 ze!J{622gub+cKRUDOV=D!>{1yoE!YCh;tC~URGcSrvnaYqNbX0iX8u^b^n%#vo1h$d`t2}-`aQFPU z4TS<@+j($3wjRgza${(hv1@lm`I_vA#%@LYN`QdD<60Y~*zMugTK{|&0QD5#` zd0_+&8{D@gRs(=p;reWxc_X9$(Gjsvr%gue!Tum$(vgUp9BWUh!aV?=QTOF=i~u=J z$orWbdaZ&t>DWbMBbU>+sCQ6}VGZE|uP*QcsF1&tOKngDeTqKy^dxnDjXbU=nf~JW zg_Gsk`#w2*e=3{rICx2et9Dv-2zvhgUf7=|Vo1nRh5bZgI=G?YsH80mn>vrbO|!tq zGGP9DdNkn%g%1(I>1qDXq;~ZR9<}itjQ_9rJh;GyK`FyQ?KB{)CoSosu}Vy)ND-QB zN9D0XNBAO>eR`9N(&=)$PkDhm{nCfuA91){kK}7e}Oc zo^3KSJ>Jl1-1p9G-|KBGGh&Eqvpd44TJjAZe%X_~$gUF8A5u12+%Ey=m~t0)Rqcp) z++p~=H-*lWBI+l6hlFoi_V1vh0)dZ=V$bBm1Lyn(3i?WTe;hRzhezHovtyjF6QS)=OH{xpLhO4_xt=IO>W|8y)AOwsf)0NMA`iZ*i=`9HJA`@mPe_|< z&>StbMkZJ2lcjpPz_VX9E%PoOAMBIw?4%}>3ix^c;!?oK!J;eBG)}pIfUY{Sv?Fp8B zUU$m-DHdvwUf?aEYgU^~AVB7$l74B1AwgmHm{EGueFeNBIX*j3F?`P6_s+~wgj_+H zp3cyZvaGb*a(ica6K3YeH1kw6I2+lT;h)cTY5Cwid}WTlDU?M~FF4x-bJ>$i4!oy^ z{$ON2Q$&e~LHok!g5UFu0U(^9?*#LtNA$S_#pQ8fe>-iX2OSpW%#q`ZRsLUm>N{}! z0>BPgBSpL*|A7mWG9Bt(436|Oi^66MvxH2#VtYha{pDis2P908%f%;;>1S)DTc+%9 zMgr89(dLjW4ySP%iQq18vBWnrB7Qdt0p8EH?IF@40dm3Ogko?AcYGrGuOY+=D59a$ z-?$C1zU)Zo*yJrcQ)O-T}WWzM$oZlR1YxXtdbI zk~j+l`(W7&PMJ(SMV`>CeN~lK>-MbW1Vl?wQs-1*^d7EHE&fS$S_oZKJVhdBkRh=a zBT2n$4t~*gkrucdo8R>s2hCXV;xgiork5<CKGtEIRa>z{kf$CnsU^59yHYJ zzjN5Z-nHo`pjajrMc%s}Z|)vTti|<-soPDI_NMIcAM+}ifiBGMcus5i%3nd-9PdCl8s)0j zNu~M`O@H>+kII_o0OsIqYb>Y4_30OEt=+g@jKupSnP^CK%^c4YCTJPvMDmX*(z2Rk zSx!Ns=8KC;hX`L&H1HlZ%409hU}}TGxm-wfq;e+P)*&zz;LowAn)_@V%-)8*+3I_c5o9|f zDb_1#>Moziuk+w~jy(i@*(2NF()DtJm8(j3x?E}T_MdDV3Zd8Jl|KxLoxY82O9%ws z@&3!zpMPS5d(p(YdKhrKOOJ2XL+k|uEHq6Z!!TqAQboYoN=<#+i)pAt4ZX23|1rc$ zQFMjt5gUV>_2yfRX=jL3w{z~GPJ3gaPR=BFo;eF6g*q42gu%ynJ*D7sb7$+NJzgd1 zrdKEkSCaASJ)85`mFl|^EG}1R3xvH?pc(Pi;dUsTi)%63I{crnE zum3R-A5pTwg+wnIgtL)o<4oWI&0UY&w<3JEAt%I983nSOEt;3xWQ8oSRHJ*b%4YBw ztJ^Efo|17C2s)EZI26tU%E6LxEf(hpxcwDGB6P_rJnk4F-=J9~qKn=yx?&a278jRX zpY=*7T+&eckyOlW7m%mRE%SvM56189OJxX(QjF+v3>zy+O#1UXAD+)A-@mF|Iy*#H zon3efGZ(}l6uk&>drI){igGZj8ImDsA_^3{B;jhTwnCc$LQf=dtlzirOVk^jJB0q9 z>0$*4k(jE&ks?piUXKdw`c^&R=^dq2G6Vt$2T2to0FyM)C~uY?F;c>5bV$N{fj5l~ z`fHqF(^&^H>8Z|Gzw2{xcqvYXcbLr;OY;5`BUx=v;x>8HV&0_wi$(bnCRsJ_b(j zWa6G|Y#u7)4rhJ!m0C#t^kI;G`beoWZCm`OqS2VPrwfV_5cwU*SXKLaLHS*5zC5-SUe7j)M`4! zIDBl73B$v>nvg?mdO)-i57-GREY#Jsf&V99B_kw1;EEithx8jhpTK>bVaUbRv2p%k z=G`2Tt;!!$_d#4tRb?}V_+|@QZs`g6j{`faCxbe(ZtssH|5Grunf@B_1D|aWHX;^1 zjlg29C3*{7baxA;qhitB&lDa{*Te3P@|WxEUJQ6YqfWlW3B(7W8L(P+DMTD{epI&u zT@t^PEDr5i>xoNgQDl=1I<4x$qZ$!U6a%+aG_tE`6&iFb-Q|y&q+S@06wyS=7kLSL z2|VeDu)6tGsmgK>RhWOOe~V@~tvUf!*)0ysxtKg2t(?}Sh5^Pgyp+$GvtqtB15G!M z$B@f`7>deyjujk(5AWd%h4xWX?$tATfd5=qWNkLBtgHLF0^bm+A@?j&=ORd3{#`5h zVFVkS{9)J(V?bl&kVT5c$&dII(pcJIV~zg>=g$_OYa-5yFmBqQF6 zRjE0j!)ieNi6?5VDs4PhQoUxOe_E0eKTg#GGhi?>eCbTZ*SEQn1=y;!wXg;uh>B2w zt#$`>fE@U{&<94P;1|mWJJGIoE98zuoDB_m)35txiOdIbup5YP;1S|^a>Rq}XloOv zrAI`b`pyMzs4v*7o3K87&;6E2ki<|;9?wH`dpDV zT}uj@c?v%uiz?rlRb!41SyXZApcz9LwkY5d@Hk%(36TGwgqfyrhW{-%3cD`eLCEKG z@%A)r(*nV=B!d0W3rDFj&sSp5dO&qamz{80VC(tm1cOO{>+2e)52<%rKyl<$TGZ9e z-R5q-<1S}A`%?9ySz6J!{BaDGg*e8b3JUD|1eRBD{x;L~r2dHWlCX{TvDz7t9X*zX zAukqV!ag~wgDw8V2Ci2eWK4BPM&l{*!A41U?>8E?c(idGh&QnAa&L(6O@jE@;hKu2x!afuf_kmkUmQKTQEJx9 zEB%9G#jRr`2V?MDRN(Vm+bw5>kVE_i{86b%^YsPsS>gv(ZPxjqlo{*vEs*3LaR}DE>_4F&JNG!z!!Y^|9}1d zKOdhK1CO)6@i(Nk3i3E1E`LLeHq2(w%o6*1;O7Lvug^UboWUD^Sd&tKw}o5kjrSZw zy<(Dq3uV}lfbWsUd9zoYD}EHG89TZx#$I`ImuRVARZK3hWo6lfiOT?4kuV%Aq(nj} zZ#9$P1Xzeyd=1uw{ACD3R~s5v-R9dPy<4PjXAPZ{UWh%8QD&<@)Z@YCM|K&dx4Z=G)trHBB!1B^`DcKvaM>M%*s`wYqT)V^ONkyDfXj{sbMlIeg#+YqpTrto`m0JcH<4K@9V#d!7hInNGT{_Iy*Z*J3E^w_4^^L=kJrA z?5e$svgHY`z6>KvQf9d1;m9sv)^8)>qQDtNNQ-5#U?tb0QSNn}c^h#~xAT&LeMV+VQ;yFj| z+53+3qZ(a=M;)Hw(KM{ul-)ci{U7tgEzTZ^#`^VQOl?*OW~?o@A1jtE%ZFVG`F)#K z#Lyc-+!1LrEzqm~S_4iKc`8d4i5S|NnX00jCKoW6lu%#A+|lpsR)9@El}5Lrp#lDw zph%@y1)p@%gJE^$+`PZ&ia)tLfbVe8$&MnD`_ zvB|;FvD*(8N#Uoh?($B{+Ba_eB^gy1EEJt+EkgRC$sLwwa*b94O1LXRKl1)J>uVD) z;ZeN($O6N%d@6(nzEf%^cyrjS`tskqu8(PR#_}e=qDv}^P#an3piM%T%j6Gr4+jKc z1v0USfoIs_IUcf=&i$rtYQ;gOky?nF@*;jLOIGlP{VslEzbHRDgfR+G^Gjw{b@!8t z+(p16BlWh3)z}|yp)GlSexu21B2;GSJl?3w31rqJYO~tDs56RHFnN7uJi${dyYB7~ z5INZ$%sJ+Berj+=f4?m&R;uyb-GCOqwof*eSj0aZoqd0SRsBmsMi|uo{@EIS`5I;B z#g6%)U*yh!qc#Kx(~w(C=9!ERl#z%9Nkgl&fQg`TzB*bC2f~IB zX*xx?1TK5SUzvi$fCPTBcVQiNEPp30@%{}Iw^TYizPdqA1F2tt`{@~X7#3Ynj2Y?8 z!Nkkc`N5ZFVqXa8{KYT(^KA}#S5=}zuq;JHk#y^L-1~^9w&gl6JFI|40Gh+jKqOEG z%d%(=%6^(<=M~c?3-q`+etUYo{c<3cI`gL4!++@J;8efCg9kNJ&_}3y);XZ`P}7mb z>`JG>?UpZqTA^xEqy5W!kVOaD@cvv&HfW8VV2mICHq>H)gm%V9fmj4mcJMX8>#lus zTPmJ+rfOV6P33It9|v0$@#5H>?&Xp1auxD7iTwn6brc}(_yaovNJ+Dq<3`j7g*5m< z0S7fF3U1mCEVOrFK;q00hD!E(?o?*Ep2<)G!p>OcZ!a|p;DpYpQh0 z6`lqJM+h~Rr#wP!U!Dk}X4AFdnNxIAuSnY^pwhU!2yA#bv4w&|J4oMBGBCV`4CS6G z_pdd7qAXT9i6GIo&w!LbE;!19?t_cn6J=#E`d`5^X(-N#{q9b8U~NheN%D z+vVB(Fett`ifQu#ic&INVA;FVcg5d1N4w2~wk1EJ?~wL<;)fi$)89q3dz8PuU7FI0 zQ#JMFD~;i16S+2CY^94hHB@6$()%(gJXq4|Onv!@i$HczN~KU0jyAraCvbRYU9*TM zY4p6iO{6``YE>?U+oHUoS~btseW}fpK8*T}Gv4Q?*Wc^x3dwya z=xCg3IJt+_LArUV7cYTyobD*WOzupFZ9JH)cY)?~nZ>k?@_7H>8(H@01yQ*LxhbOi zq8R}>o^oCMj%xEefl`F?q$XeKNIQbRB>K9n+5WQ#TD9h1k?}cjYF`Uv;su1z16^jp z5^?C>8AvFl3KK!*>~c5<)$}tBkZ-mQ1qKHjZU6eVJDGF9>5SKZmf`^?g&t^|vYbRYG{^!d zreH}FX8|9Xcn2C_OrDN~yF1;KAQ1SX;kMJ_b&sW4Z7fuzlo!LOJ^f3dWsTdQb4a~ZRm0p7Al^)AQ;6stljZ?Mc2NJB~n_eH5WzoXu5_Fg^<^`o^iow<_@#L z>lW&%!3*8_yP8Du)pDhAFWJQb%Zv640vcm!ipk>w=vcmaPNk%3qq>7xM9^C6%AOy7 z9mf;CbCYc;CO>VXrwpTeb@e@JJ5qbllH&hZ@5;Ff%%V`UAwFyLG`Hs!0};Q6@i^VU za9V{vZa^qK&R58;`Sz$ncFh@b=hMKSjR`(S{P^+XIN@-)nKNHLGpfOUGwj%{Q`w8C zQRF>pC0r3XEN8y88b9y#ghi|EIo@l0Uddd+f)Blum-Mu2RVs=@)nY((H6L1yQIi>( zb%#Dt+ihEc^f(<->~9ZkAcYQA%20M;EQ(0yL-e*vGT1P;tF@h-B`G2wq!11Zp+$uE za{AeBz4uFs8N?{>@FJJT{|gIXXlsZLf0AK_KK1GM$4ba$Eov(!^(tg73u6a(`EGCv zJ8Np@c|-tm04k0wzQq_>nV0mPzz8QG8`ucgYpTu0ovXXiJ6tP-&P244|> zJxccoZO?o0OXYdtTWZq{H$}7F`S)xT-Gd=WcZTL%c=eWS=59}92Tf4#OO6y_966fu zOv!Y|1~=OL&n+6TQD4c1E8)P&`w6!nHAB4^^K{DwmmCv=DwD>A=I!kb5sL{VA#(!y zF%(_UPJb6LLA!qYyvT^3RSiFqg~NjVPlFMNL&q7Q#bbG5YxIGx0rG`Dpdb1CaARMp zStI^4Q~;3se;Gx}F#S7N4#z}YRO2#b`T81$*XU_a9ecRuI9e?-xeYuvO{fZH4 zV+H4Pi%=1rSRkKZ=1ieVTNuOnXl@k4yJ}Q=k`B)d%nz=WO!Gk(@`a>2G#DSi2HWpV z=FvchOiQJ5Fj91x-JGm%c#516{3>cep#J7IRqLju;Y$F|pji!*&g+VGRah|~wVzU4fg!RZb%Nf| z)IbKrKs_$MT=Bu^w|fUlTdFHJhbRfMF<`hJt@I85W{w@<=bdcIk3_=`=c)d|63>I> z+b>%d29n4!#C8#<(SO{g=`WlJd`>wwpwwrNAy3%kCuXKjC!Z-07#@z4DNndzQFTND zyam(KQZpa?5<+88_?w5*lNrJBt?lynu>vi(6bvGA!4kxmzcw(% zeBslRk27=3vE;?>4c1x6Kd-Mq<$ji<T{ouPY9n0V5elETn|IVO`W@~h#6JN2!NGx?cF>&os-TLJf)XT5*Rzrq}7}=r%E`xn;`aGrw7G-8jG=R;E&MN2-B649XlTH=@s{IWsy^~Tq zhO2TY*v)*Al)OIx%K`u9O*C0hxiP!>_=`W2zd<6Z8}(B#L4@215iH#I+h7HOCuw)sQiGUY3;m739gj1 z@uAexyqeIu5P$5Rq{`9KST3=@Q{2?2yYZP1^|{R_$h6mEAiqU17v=H_NI7mK>w9+s z5uqF>xvzUPiS9pbmz`Lo-8$oKyxHhOtapJRU6tE|Q|1%Ub`(a59}9Gt1Rj9o(rTGen9EQXAraKdIe~W=CNarkjC<}V z;MP|t*WreNf$8Crz_5adHf(tIY1%8h+@Q<_fe#Y8_)fjKkjUd3;SP#I{`IbRj{9u` z)JuM|q5Q~++^FeIa5unoiU>CB(WwEFtlqk$fk14&`E7|BzpctFAeR+}!;`>I$NEjg zLayku4B;XxGrQrw^?Wrf1WU6Gv_7m(N+Wg2N%}oZe!r+Z!-8N7h>AqY%zds;Pdg{` zq@Cx>X{IoH{c~Z6VG%LL5zNvUIaEiGKH*QcAj~no*mQLj%Q5siRP&Kg{&H~Jh@aq$$1#)m;xSE#z8KTw z=4f{YX&`^m7$P0C%tdL1jWygBC5G>%&SKC=F!@kUyyf8A%X)|n#mylLWU-v~CSfNA z38jR)}{!M=W;qiL)M@d~mSDGHzK7GPtT+0xynS?h; ztMC_Nc(@#y&&a`ld`QtEo1n`@S(YeQE*O1tS*}$_cXhnV%ps$9!tl+|wgii17e0eW zzwL|AXUTVI=v?HQlg3?c|G^Wizjs8iR4sYwF!V(#PE`{`t)u07}Y|DiA*=(0^082L} zonj^T9j8eQxBD4|T7_QOaG3M`m5uRmQY5GBVl{5_w0NZK6q;mud*)MQTq0tQURh8I zuk+WG1CW<~>SAM-AryS6O6Ml5PpHj?2v*5nO@rD9Rl%s0Rs2o4Q9ERTB=%)eIgo)| z5eL^PD8c^s02_3Yz;~=ZmJ07H@SJyae1KF?f>8)3O&ZC}$vjUbapBxLYZ!B?By6TD zNhK(qv&X=Nxr(~}VeqN}Z?0|BvQ)i38rsSn0oqz*JC;ZA@gkndnH#pF0EQ^QUwLqK?vO(LxMibjP;~iBG98dXlIF{BN%hd0`}?h`Ty-UZD~^z=$DYiahK2 z$FNI-F$K@J=U8KC&+^~MFcJSSMP0AmPJ@$jgrit(y6NL7V`rPr1ra zjDXig#_hn((-S{Up>LikoTv>lBTqYO#vmo2&R(FVyS(lARFl{e0ZX-{N+w)6W?+lkvg=WWHRhYLl;h#J%K74UG>(ESVM)kFDELihcW2cZwfd@FazNI1o zu6-ZT+H^rLw71kBA9T|JoJp}vh;m~4I-V1i(^e--8h{hGnA2YYwKcJ|#>EX!c7STJ z%pUO?4E4o3NKCt{W>^&Y=aidW9nRnUN!A~)GLpEtyTeP<_~&iEN)gsWy%OudP}*_4 zF++39@FwqZ2>bIRM8j31IEH7-OF8c&CfX-ex&pcM$;WhZ36xyPSZF9HsHU-yY3|Q! zz9Dtpsrk;4J!l^4-9e>7`~RtCNMu2a!F-JeIyYV3<9fb1>fF!6S15OKH(>ow zj+-TvE}p_~(zb63>YV#Aj=`*?Kl(f<$!&x~e=D?OovXe1ijeA2M!mqqn1Cqc~+VjKl9FU7Q6g zoP5T#uf3E>&qnP3<^(fr%{C;TC3nsYAhL)bjD^;$MK449M73v zAMn3c-j)|n?-Lx|u!(F`3Y->$`bA4Xf=>>X48*y`_>}QH#Ui7GLjjPpC{tXG1L<>tyc(Zf zR*fDH`k}O1YK{&X=uLh|X3~uT)x}mi)*>VA_XUVWMtd>toizZHS`mb-=|4TF%-0;E zDY6wApTS%FpK<_?lG&b)pqpCx6JgPN_d~W?>&XRcrC=Bo!uO~LKTCVz+8J+FpAEA# z?}N;LSLf`m;(m(IC2L_FikK zvFt@=-Pj5Rfq=jugNafxE-(j91jGM<>RP5ytsnhwQ@u!90~&LFj^{qXyeo|MZyS;1 zwsgfG13!)51z!_&0*KVOm4f`%XtH(^i_K~!;k_$S2k5iM43kq2sWAnc+N6qy%ID%0 z0nyO+@sVn0lb}RwyxSne1_QhhNUxn!XF3uO+KrCuMF!%hL^v(Ok|7llAeIGVp)-j` z;S7+HSy5wYyn)+6rPJJ67)K@(=t0uVJgU2T~Fw% z!zS5JKG@*VY#R*FI>b-MQ9}Z*s}tl=utmZy3l;i;5jZTu5=WA`qnqtS-8F9DPEJ7W z{fOP2E_e*T5j3Q-npMND638_R*b>r_Vfabo>i1cRIjvpy#oDcx4UcQ z2?qHmKKK?qn!8YJK_Wc?AKE7YFnIlz@<{-97;v?(YnI|kJ^?Vi6AV184O3!CZH&*A zi!X_d)^7`PW3!Q1%y~zX{y*o*V(n}hf3&}#(W{jnOQx%at9N?-nm*ouDbr|x28czY zy&y^&-K^SPBfQID8f)!#-A@XIZlppy6LTdqML|UZD97(hLf>v)WFZ-nsx$|9-M~NVkL7IMh=;kfnhaxEFEyZnTBlkqF;+Pv z%(O-YykZlcY%Yq)UF-n9wm1(TA3=f}DWJ^#?BGysq8d3F8NzxNa0g_c6thJUau#s3 zr5N#9)rhr=unCiY+N~Ob%v6|vXvpJug&*h)_Wx)i|afKK5e}=x6rS;uMncY$-d&~B=^anff=Zlm5y!y?5*>v@ng=8HEtfSrh_)vmb>iz0?ht zBBu)XiOyt~d$7H|I7RoUKcL)_PvIv*{>~W_IL+cLQxKglzO$_hqbItCiYty_@A31P z+)2CAQh?4)4RzB{sXr-`Vpf&|vX4IJ{83v)@nl~0Y}I=He>0T zz1v-IYp`UnX?voNu7be`NzVT%q?+^o_)WrQF;0NtD)G@mZocATMk%o2SHIK!dHm@^jMz}=qORO1r2$Lra&WDek;~Y@?cvFE1t9dYQ__WF zF$x2@-oRjK012}3f*0SDpw&wZ;qC(+`={LoKDUz>-`<;HCH}6^j|PImpTN`x+)mU> zO8oyG-^vL4jutrO@la6oX{0Z-R&EM>FbD}=l3mH2?S zOC-FnOza-?ehhoH^iQv|())$J)fSe4ZeyV;BiVxI1)^+ zPSvUy9^SfpvX8S2z6|b(e>0^BOJpj#U5ooXOb$o{JTBmI!2lu{J{3|=EaW09%9-Y? zMBbA~k|NAwi2CzA9OOycWc`1^3+U`9gU07ZFVcjnOa@QfPI#_7!1)GT!#^eqhT<$Yx~+8S%@HwT>eVNLyYM+B(d8YqSDegioE57-oXS}@KF*G! z^D=;{F|INgv$h9>0I2!)&*U}mzR?B-TwV_mBzYle+1ZFi=*zng4-Wy&KS2!+R0j)& zBgBrhfKqE;00O$`Nyl(9vm*(QUX8KPhcemolcx9ZUe{>1L%VMlD2G2?z{eWy@*#Lb zt7EHg4P|J@WNjyzjfn)@=So}t#u3{s5qV55)`0;zUYBde#iBc z`x`~J*;|86lU%{)ur>JguS7kr?EbH`cSvpLUhUyUMx!Z`oRi?b_d>8|H5>6miw{C- z1^KkXtS?007^t(S@VeS!ys&sL8&9=yH*|ZxqgZRMxWYE#{=HEa17>=O?M)efB(}UB z1dWyJA`J*N4+XFMtG3x}Mlm4V&Ggu1Vu97GST;tBsoKmBQ)Yag7tq%)iAhbGj0-1= zYa6p=EUEkREtGPJg$UihHiBmwlt#@l^-PRW8t zU!haWVLTTk3u4&yf%(2>*JG^T6(He{1T2A|H-att1d*_w9s|w5gwt^(iZEAaAWDrv zqwoUgw~GEQ*L6O1lYBayuMxBM1(g!w~sUAcsi~dS_n*5)I^PnS@bNr+I?-o{rICHosU}UzPk=6e1h22UzePMEOj< z-Xv%3_4msXJAVvim4aCq-o-sx?{ScqS_TAHu#etjezIRuL3YVw2Lak(Tb1?$~sSN=PeR0xI1d(%pyxk|G#% zNFyPQk|HI7fSkE~-uKIMuJg-3UhF&8z1EmxjxnZ!vO#hF6&1nIFoWGCfke`rJB#LA#Vyw}b6BMb(mUvnvz z3C0ojJ>mu)C^nu!F+%h}f&0my(rH2clo7!RA}ddjE7>idv|s| zSC^vWaulr>(`-P0vos)$rI}6plflsMcN6GmfZ#B%~JaiT<_njf0KZpu+1BNT5Y|4aKDFZlN4X_#xgp1AuO zfqW<{o10-lgS8vT(9`(L8GuxK(%}i4h#ZtBUZY8?#}luK?H{J+d;n7PL_U)2{vwjr z%WI{h@M@ylGv;v$uS~D`u={nt3;kCn`NzjnPipYdAnniTzUE4Q*|va|%o89y*G5b5 zryCyO-<=QR&r2FEhw-%m{q)EX-fL%Tze#%X_GNk!7(3j_k@wfIhJg-66)GVRIWJ*E z#L{D=Q`qY~+U1ojIlrwhO*)v~NuWPaG-wm^+`cX(WVP(nVfkz&T#ohYS2mEFnl_iS zb`fv_Ku~P;J-%Mv{wx7krkx~8?p>TZ1#3tJ4~7;2YT1`9UR3+A!Wh>+?|$TE>}w}s z)Rdl2mGF_WVZ_x;WwP&35zaWhYF4L}yE2+(rf+<42kc)HoEKK8t@Aa9N13<8&Ihnvr{dr(fva zOkOE_yt^b%%xP*U)YKJAOO`FFo*~!)1IC{yZ46PLDmNUU#JYU#z53{$iUmi!Xi{%_7@ZbUY;bVkUbtJ}YT`{6fXe!Jsk`_X3xmxkMG z%TQbk6Xbb1D=s#J^w)og$#_jF1@qn?ILs=#c6h7fJgwe|%}y|k&h=lWi?!(?y%(8H zuJ``D#xkt8C09viCtse7rXC$vHQ*^dxkn8X)wOoMIrFvT7rHb=l;E+-hE{;0;_9!B zwcxoW^=xsYRfU0T*&?xg4dGE2W(Z~i31KU@Wy=%ld+mrx<^DFmKUG5m!PCjY{$Qm7 z%!{p$HG1e6}@k}BGtp+X|0(Pe#DjJ@`WRjlJ|iY7gYe&jnVO2 zID4nRzuGshI)&O#hq2d(s2CdFU_rS-@P<6MHDd&^Ec21`tDXLyY_Sv1W~1n_yo)*> z)O*uSPtIGN@`~9F-P$DYG>1v4YA<4GUfmx`QLCqS9B42u(6boTwZHWx_B$OgWAX+W zJu^}RG6of_rFWsdj@i8we9|CU}}PV?t`WXP{5~F+4b@SUCWYP-=IZ=t_&vwoXw+&kCBs638F7L zQjbSs0RgFz9e8Nf%|z`sDSj&$0beUc)|i%;(yh4*82XIvN(~@9i`2FK9g)NvO(tG% zp^COIp}8LXivoh`3TBVd=X2R`Xlq&`b}F>JDhLSD6^M$-kIUSu*=j$H%u;7gLdrU z=OjTlm<@uYlvfMBG(443Jy|Q*Vml3dN7Xh^Y4u`06KbsJ&2wx`mR3^k3+(q+4M`gZ z-)9U=9gP~D=wY;XF@rleHFRUEl(i1oWwS*=NY#0Iyf6QSI--q%q*+q-b8SmYi`+d+ zlfAL{$C4{4B}p$TI6f0_5^E|p8_U4rR^8WeO zPs6Xj?Ar@Oy-|Rk5_R)SwO=W?Buwtt>D%WiBQHEk??iU|RFjs1<^-`{W0eJZfmprR zG5``=Hk^>LU2d496dK`SHy+Q;%`GYN-RaVZqkFcqqz(hCm5M|kei242TSCbOEk_{F z5ZS`IVUw(eujoqWf!oIMcKaSxP%nBK; zbl!mef_V>R-AmO(J!Lk}`Fj_0O&|qu{Z4i~*u*q))Vf8mo`$q^Y}4 zS<_`y>z6e?$?r&uHGz7FxnlT|g(i%Hh=f+s_uW8-kaV^CT1QMqM#fz21@A3wZLH?Y z7Ix_3!}kc~^=+jub#TG^B>e66!D!OWzYFurxqQfL$)lCqbqXw zpJD29|Mf|yR2du;B(|8cvg|^c6y<U>?0?P#@jwUbnnuu(9xHZE6Er^gHe0tPJTD z6%~!?@+)}WsW9pFQ)wd^xKACsv#kd`+ztL&rvq&OV=yoIQq0YL_m{J|;Cc1DVU)7&8`X)*|t* zSeo_6!~ck8-{xO3fQ{Gf!mTe%h&YS4$VXX#;{%rhQO+J7F*4iT!=dT{)+H)E60|OT zwK|4vIwrK7#1&;Qwwwuf7j~*8AJSz9LI+bLd~^efR0TfHW4C2*+M6C4(fwt6EIgPg zwip<{a;_}R{~8>sG{h$|K5HxoEyYZ|W3YWi-iN7||IhCn1;rG)-%5)A?Q35uHZE>7 zk{K8`o1xriUez+Y-y0_{V`QGGWO)yHk_!9%<_E-Wk%S?)avkzlK}13L^Vs1&v4iRQ z6=1omy;*=jUmX>r|NQFLZw1{$r%Jg?D*d4&y!i8mu@v4J^ckv>Q_Z{)WAvKTy-fE? zx8~buAsJ;=eUt!o`c@j#Ym^@wVZeO}z1j1PEirk-T)?tRE!LA$K!InJ>**3Kl&Z=%{R$Fvs1f&T(pJw3lDn@qxJYMJ!%)N|Z=a_C66@21! z3lplh5cs$w`?b`*2~T{X;G>Wfg8nN?p&n*(7f3zBKw6~deGsOU32PHV|0LQ*?Mi^! z_f={C^mg?Lvcmpip>le-#nyAa`%LwDH$>k*qFLA;W$#X4E7=OdE3<%}m*?5fxABFB z7k~*R<*Z72A4n!nWSw5|Botr_WiY^)CV(-w``k=1ZGIF=)ZcmI;UnQHe3I-qAYt96 zZ7mrka&2h5%kSjys>>=AK|UmuAU-bPH~71cS~!d>N-tdOjd?n~s?!|dzWeMpO>1)? zJ}ra!v9lK5GfZb9r0kvpOF~&?r9lH$K#&Fclm|#H+3C* zzN$}N<8%Du=wS8p8U2le8o!6`PouubR^^ii+vbdqYnJCPv0P} zcyW$^Q(9V@1x|uLXTR=B=7{zd&KwC}pDK*57~Gg*!SI!=8xF5#r~}Conv@6BMM*`P z2q_**I10#j8O8BZ&H{-93+kgc8yjnI>F;sbFM(!X3tEr2%vucNbX~|2RLkas9H)ao zZheCzySo@5^tDIP#H`vkW;u%^Tn0nG?A@ZtjmG8<>U(WFDiu_k_hD5>7JK(sClVlV zbn?faH=cE4?;Iz6f3J;}rmkh5qfcE?B^4|G^%2I#H*W%Y{fFL1^m4VGVt^?{kNI&^ou?BS32_0PeU6SMf0s5Ok+&Gt{taa>&-oo~+=@mns zV4bONs@jKcA$cDrQ;OwiCV$jtPl!gkvs{=RuP~?voXPJvO$;@bEE>aws`>28%i*6X z{z}Wr%=8)^i}=YRtTD$=f@$zE-+OdyXJvm#Pg=@WyX%W|xBABO3PrW=UW22tQS_#4 z`juf8T`^Z5QTtTel-@|NE|qvPi2kZ^!S@8ei}pRi?~03y6Wq8F2A{3P+E5Mp$eWGNcN!JgvYSe~ITK<^K6^s;iMk_pN@p5hV`JpQL%y9r8cNp8=67v(lUUqAu= zW(NtoiitXpPbrnC8#e#tJvvUKnnJDLG$JmTz~o?_<%q|qCb3-Hd!3gGK54H(zSyxe ziO2Oj#6Kph8GQZxa5b2h9SD$B_bvd47es)wYrhQb>?($Bw)Vk5yFdjj478VIDNd1d zNWgB4AhZG0^*;j;1?!@GE&1DnxKmPV@Dyklft z5fhQ69yOe*R03QH&XikBUaBVl90}=LnX=cNcXl^wqr@2=b(FUPrLPxOuYd z`h%Jfh#k8Lv12QBvT~|kUX3=Y*_?7qC~=7%*#NVxO4BCo(|fJX%a>J>*%$C;mY6|HS_Y5BWxMfm#%&(e;2KkuJ`P*ULUpxK1RMXJ5f|h$MGWc+(G@gvuolida~TD)+`I4dhWLE^huCc82W zR05eKs^hMt>f{npP{o=Mr!dt>3^lRy#Cf%#_% zm)43Z_g)X2w6f4xb0@#z_1=|u|J9r@crioYGW_b#C}tR}vWOiR*)kx!D*IT#dt52_ za1Vq@5oCgH9m;!SMzs%cp?E5)Q2d+)rU3fZk|Rdq<&YM??YXy`XT?Qdd1Pmz1}CnA_&aB-d4YCNj)GaSS6GM7ay zP`n`!83p-Y+3hZgqAoLtAKt$a0wk$}IkzM29}|}--{sNE{<7b{w}qw#PI6>Ts4nJc zIy&obsWKNpV44%lC?<6=3;X3+Z)O8qoaZbg>#c9xP<(D#eXaM*9mf@(3aQs2J}W$F zv9+_v_D-5-K(%-|Bk)5ph;$~{%ZC1Fm`M@ibNT%~xU5VqdWAq)hUZbKk`XsU4i*cY ztDAOcwgfV;a1Xlj(rZKG<zBN}FH zX>mKSS(*?t*uh-Z?&mxO4|h7dYc&fE?dzc3pnc8O@#n`?9pD@Y3j`v@rkdR=w(%t4 z0e(;kt?v#FAO_V8%5-1}OEbJ|6|w}7H9XHRbRJBje}cRS3L!%ayL)pnq9y zV6edkoY{&UTLVz5Ll{fj=~3sDHv8AZzw3$~)>Tp5c#J8$pLLY{uhkx`jDVZkPvzKv z@Rf&hxnusIK6 z6unH*z4rmap4(KNG1TlMzURALv`E-D3h{~(m@qBg#+A7rEY`CW}=4+hSX+RyIIr9WpUODfS#g`WVX%CmWFBzE7} z6K48x)3vw8s?X_o&kBWcwFw7SV?2M*wE-B?@HU^H$s}q^S>wF8-yWEZ<^3|#%N6%U zLL9U|4($waNWtcgR8T^2QP%!G2G47RQp{{;viZMooL-}r zZ(bQ-uK8Szr&_5te~9*&7}quq&%6Ef_vXt|gO}M=?7iaS zE#9ti*0Bdj#oub>azR(UU!mx%%I44t3?5m>Qra>CE!nh2mi-i6@YOPPlQ#cNbCRF? z0Nvx5{|s}&W?&s3E0*pU$?L&xdG6I+r zuS`aeum{~+pUwlO?h`|Ur!TxUswXiocw~Ssz2x4tF$#`>@o^G3oj;7HZ@(YSi^GW* z2FW+kulaQXCh8j48gDUFciO<#_^f&PS5HRq@?bSXEVr<*Atr8pp%RbVS~I`*H8VTphIW4&AlY;=%f)21tH_mGEvrzHC{;iI{p&|8bROgoGJ>2PqegB1Z|NAx ztwmnwXVgrFycYSP5)0~f6^5;&J%DkALiF%8UJ*ummBzghoE`ao_80OF+>E$V(Chm8`o>8- z_PBWct$M-1H)6RoGV##YBzjjm1Pj@|q_0N-XjH3{#ZOx515|>Qv@IqdLA>+xC&_Y= zFcMHS$&*B~>)(4O{$%nx4)@VD*Q=A4R=$WWJ3>9Rjp&fErfya!)?Jhs6xS{Rv-M=H zb*#;;VC>v`ODp7p_dj4Fd^2zm;w#4;!t7S>r{nts3$vQ^T?>6B_0UGIJ{IlJ5RExu zx5hl}3Vn{nZyPHsH}OQ>Vza|&aT3(v{OIB*6;OmrWtyH|j}%h>-d{(mpz{6+E0Ipf zb@Fy6)*CJ>GM_6d&qNpva{J`~W{!1yua=kixp;^+o+RXBR$?rx580J z;L4?{vw<`T8-NcI{e+v{*f#he$-l+mUiuRRbZ*%zZUQ{KX$;ENnRUM$dL+#2>6;K% z-a$};OyR-qqD-xdSUHTKgW?eR%Fc}CQF^-2#mnHeSH=X?o-;Pp@QC*E^&4~n^g7Kx zY@qK#l&wAn8Q9_Xnav=+PlC38Jc@{C;_j%Y&a(8a>byx~%g_TZE+dP=k-amcxPJ9+ z+3~ChA^(^qlbV!H@a7rvt(!=0#sp&~+dy=8RWh|BmO(t212n$YMTTHxEg0PFu& zkf`fi)iq)QGreI1IF?~ifm?v!eLMq^XhI=)4d`GD_fifV2%#k<%!#5Y6l@LFl;)Hu z%!`saY#0^AK@{>EuWg1gAkmrN9hAM1Cp~gIE4|68T!y~F!%0xP7}nFv-y3g1&G8F7 z!nh9M0g3-CtweDDz)bS{pe%R}tUob5+LQt3YP&UvC^5Z%a{u3-`WTCL2^i80!Q$Z= z$YR>aZQSrM8o(+8ytL*YAo83#uBYsn4B2{6zNG>k|L=CqhDPW>nSa*v>fj5L2Bh;+ zdq5-W-B$4gCiMuY{XfpkP)bTl%Iv^#Ah$)`X~Y+sWPS-i-gRU3YhaBlF1efX9ylN0 zqn(R@H#^@4Ua7YK%o%^5&W9D;MWP;CVF1ACl@o3@U&pMm>5<_N1$I0=!$t2hELb*f zS7vl-0`i|%=`hzq4$ZnV`e{y1&c?^TI0+e5KztnT(KFtr2q6pgIRC9qbW@^{^bG^C zc;Iy>=$!Z<9{=1R0dZ+D2fz308mNYf9Z)s7O4vKaJ>SsQ;QFwC-s4y~_`#kFJu-_L z^;zjg=vEqP`3?%tO9+Qqi$sVfK)I$_CMP?6v1 zOW-A(+}<(bGm!_pXp%*KG6B2QN%rlDY=o2UKVMB=3MjY1%LR{=fQl*#cH8iF@Pi&m zO~1Bv{{Zd_zko4HvY9v^L;Esj$s9Y|`egD7NNj-0quOysaBH&q5-4>?DwLrd2SypI zKV3l85=rOz8J;m)fq<&J7i=JIN38X|7raMbgI1Fj)+vz4zr_2Ls4Z}#oCQf)tH_AZ z*q!n}u{(rkbWgoRe$)YK9!2mKN{RPVQ%1h-YeV$$#tFU939bufiq#i{znz4uLP~rU zjj^cyap5mW#41ypyjGo9e++JGs|pkbdYwNjueVHB+lUj=$PjcywfXyjXnjfSb@%sV z(>tj^_CfE*_L@4)3kP@!tmw!pvPh1eY~ZgyMVurBd(qz#11=Nj z5C)jYhjecOK3+P=>pdTu$b1SEUPc%p{H-#~@>}SzMcBZ8i8Hz1s-u7S5)X9F<$<*m zK8yQh6Aw)P?7|1AG~4R6D^n>iB?`?D(nSanf%MQwb&^K?ypaQc^ddBlf z;eC6u|KB?bg*!Udy{cQs2(T!0B=vKd&T42k%E?m2ofGrVk0(?&jZ8dlp4d!a+(YPrj6MBLtk)@OML8jyI@9(P2Q2(TJJswUOEIH)Wsjnyj zLs-WMx^lRIW!0Sjgc+K4%$nKl&{sha5+YmQ_?-v>mhCKz`TnV4)&_R&S+(Rlti2h< zcht$)`><^;|8uAy8y-&8Fs`++>6Oy~udRj2_bJ`GkeHHXCeD1w&X_w#Q$XUZ=K9=! zI6_Nx%Mz4~G@=U$ry@iDFpY$gi5q=W7?qg};g>jzz|9s?@5r@!kKP0>{%aGMe0332 z-CeMCw-TYX>WgnjD~GyXRs%|^$x;UP67!UV$vGkNK? z<*wLfA$mAKWx);a{eJX+u|?b$qMoXw&t7k^oxyW35cb%*0dvrvLW65vyu0WJtE)YH zgxo#{HhB2=5;dQmv2;KhP=B(7Ex@nG=^Qg-o=eICO(s;4?WD;q^aN!@NWlybpkp_Z$V;^S6k8U?B}L|X{#@} zPO;j1z^FfQW?TKlHa@?6DMGn?u`dv$e|rnv+WE+P(fjN3Kza{X(6yW&#gxe2$IM)2 z3Nmr=EL2ZduFi%C=oLR%&-gaq~XFNC{NR-99A6vumn9u2b;B+6&mF4KUqLu|i2vD286bH8mHU|e3pGAFL z*iIOXGd_LWkBu2@`Z2jpBUgUc%tkfzKFpEmUFyzw$xUkp=xft0ZZ})J~m71XR#~1Fn&<5zDg#W3xcHh|Rcju&&d^ zOGguJ2hXLPVk~__2}B8oUM!?k)WN6Zy~sOzmF0^kBBy-lj1X}7qr-^Ce+!Iht%ag6 zZ{vm7iyGW?(zsXJ>@a`Z+nvmS_IH7G$8H#NL28l~Xq2h3{%NTcfSs{jFb@;$C{bA( z;3bEMlxhSlB40>-yi@mZmI6ANifO!7aw!$hahA6x2jeLUEGV@vS$+SR*Li$!^kUX) z?>855x2~DqQ=zIaLC&(B_p9bjb*r?L$6GBn9Da@nJm*ItD)^FwXs+y$My zhWCq$i%M})=OzZ7Q@VXCzlTALV4eZzh`^_k$e73UFK0BB;#r~zWy7?JU&M>Loz=@7 zCV2fiq;AGvi|KqpL9Z_A&g1=>)o?U1*RX(GdXy59$t3}yIPMSk5)U0{^u-% zTA>2Mz*+(Hb0z7~CPoicE5B>N#$y@C+va-~MNLL_Uyg$|&^`JE@O{-v&Jucvg}~Hu zC;MVP=(i0#?YV@G@|;3{;oyD5&dQgM+~}tA(`I4Nht7V7K~${+Q-zTi;$ z2YI?(WusdeyS0q{|8%kZhIR<+KJ)T$pyJML7s7u>15;02fr+}Ow;vl^ywmZsrH$|l zJ)6v!Y{1b+nhy>qZrTfSy83|s;ooi?`JXIQ>(nn4VD_FlRme8_~*p`M=%Wd zZFn>IE&C(A_m8?kuq>MhV!l1^4>AKVVxI=5AL6<$@coB^Doa`B_J^}JdT~dJ4=YHSC=ia*fBlK zS8`P|rwdFv)eKHo5^kyo?CJFt=nTemuO}0^Y@#R|+ced>xaz)@F@f@V92S@z@*Tdq zz_6teA%>4QKe;2E(f2UTK@N>R{a6UCp>_*-PYi^H80Xzq2jmWHyw=XQbJu3Z2ftA5Rms=IGF!zk?@Yy2SXO1}w-djqo{XNN+= z((f>PAR!DmH>mf;X8rpnoz)=m`5MVar*BA_vlFE_&dpy{L`cCGxa~tZAs-B5&P|_; zl^RgD9h_lp>c1d6!y)5}nh|=myq;K6s{e+9X&^g{kNML=!*C+jMS0h_FYo;W$dWMr59cba5Du_d+LNw}VJ76d}v`#xPK4D^**n}0OCEsL+c$;e6p zs69gJ_HZ>FYsGTJ59wEHn`qc4mh;NGM7{9fokRclC=Dua9T1K{d6Wk}*yj>6{6ZI# zIW$u_d5^#5$%ab#kz02)Kdh{#IHb4F-9B>NsJ=ZOFt$E#b)o*zbWE{hM(+IJSscHS zw(scC|L$Vd!7EvGQ|!i1JOt_WLm`9aNpZ-pAFF0L?u0 zxAhQxEZ8}CgO;PL0Pg>B-nbIA&&+U{4s+s9-`@vUL5)C`1J)^gGc&U#>(<3GM~~IS zF3A;N!Os>Qu-JbZUKvCOszCDj-_@=Vb>kV2MQ1KR_sLejH{o%xsUZQ*F4aPpAuPf$ zs@CF;(1aoGNt^)6^Hk*LWR9Km1{J<&47g-&T$hu(CjV}zEh>#SVz6l0V7WL}b`~^C zewWU1+T*fid0-X{zwM};@1ObQ^H&}ncaK&_ZZECrc7KgK{~0T@aU-@sLqK(0W-S^O zhqp^~lRN4$ErMPtkFZGf-tgUP1&r=Vm9mapuW`z1AC&i0yEQS)S^Y zXwXBP)9H}z%!UR_EN!}mjT?1JWtXf@sMQWJ0KkQN*gJfh z#O42RI3EkN++iy{iILrz&<|b&8NnK~5+U%a??mUyn+KJ3RIU0if?F&GE4u#ZUegY( zB=nIW-uz=43wr!zg>Kxyykbl>(k2JDF?-&_mgsB?rky#^rq$d?@fyuY0lD9?Pj_GyZ+H;Y#Yx>!_-dy zdbs)(AvPv=g#*fGZRduev{w!Ynq1b53j|BM?l~AW$fu0i{+8v7zu-Na-4@UOh%+#a zg=_EyWdr8okCZtMV{OGA*ni%D&m5F7=;TC**XjUdLU*Q7hL9Ds<>|B!@=KmhjLR3 z2)gRD@Wo$?HhEu+}bV!XZuR&@FL9?H0EUhPn`o17-~{Z6MUWt2n1l_0Cy`=iTx z?+cd#=8>`0>JY|7iNg}(I8p8KQZOV(lpsx5yn9_p7aSmYg^ zd3kv|J39-Z#s8nRM6EjTN!TQUF4mgXoEGjy;IPEaE=0Siee5-zd-mLT#g4(K zSoQ>V9V3^B38PvIfwfX%M#IA#zN-QopSjb1f%)#nq+cA%Gfn}~%Yr^Nv^XT}HzgC0 z?n-1M;pqwPrH-BzaiV0LrUc<}ZR0Em>UfE{cD9=|I#sY5OH zEBvWS+bv@r)n6qbcU1(LkRDe~+@JyqtkR?YQM%XQEFFq8Ow2INn4w66= zUYDOnwbIzZ=x;=!X7R1G1;Wo?Gt7&qMDc&zhEq#qZ-skhHGWiaz`c9G3e22I)35Gd zYWDcfhIGrX2X#zXF9yVsK}zYTfYY}~%%i+AY^H5uRm@v{Hk~e_+n=yYW8OtJXjY%} z&Nm*eVT=@aM`1cM2#J4rElanwEGF*bd_`pBo8nh-+<2sx-8l0y+*e#V`o45MF-dK} z=8Qkp627G^4eybJO_w0~yp+%#LN?{-87y2BRkgb~r5kES^cmc3W{D@{6<<`{ZHobl zt-XlLheN33A$(iR}ofAi26k0C2!w2k-9(6>pZODbY73tEO zV*%rnIW)KZspQbv6qmuKO*YKhxq=5#4qwgTqM#%Bl*6Qc0aP3}#5^jcf(aRPp6hi) zkYKU12(&IvrHoD%2yV$9+6c^iHi`}{{{AvSz)ctP!FS4Ml@E{Ozdq(typuohlKr5^ z>I=@Br?l)*`g#Wb-+0Oj%4Ms$2M8AI_4sVEjJdySg4FP;=$W~)$!i+~^mQxj_FpbNPQW(%3*FANXRIINYJhDxF+oYO3bS#r+8c&lPFLtH&whbKlS6S&dG~mAD6Wzw;qfrTMDi zMLqWWr$R24zb6P0PVpEHo_YkON!q5zB+=O)C<@-zR;Y`gs&+QX#Dz{sMwNSA_|HVpf;Sha_;Ybbw2XXJ=E?d~1R^vrH?k>NU! zTR|utHo#l5c+pPfco3Qd`5y0u-#@qm==V6UteQwT(97ROG%)$+T?jG%ZSje({GTCk zr76U4MQR(qj?_(Wr9hu0r#AtF+dYs&BU3e$l){AlNhm1T%3o{GKo5>(gPvKF_JkTBDnQ_7Wdr|G#0FZjh zY|s^SJi7tsr83!*Fwh5v-F6dB_)=d*`*pKpDg zSftD*Pg$U6g3Y&D_P;I^WoikOzT}le(cbEsLRcjCpy)JzL{SIHb}EIqx!<714$GDc zCxlT!;XGcIQc|Co|FHu-AG@#JMb*YbQvk`Am$g2uK&sIqY?q+&U~u`?9}X~imv$w^ zIH0Q*zZ%~L9ZH`r+fQVJ4nv~2XkQE$-;c?e@^R(Q(?2_lmwHn;A{Q1GZZPUpWr5my zk9MWHJ^PJf?9)t5-iJ|7MX)E?%lHmT^1Q&7sGvb{vBB+ONYQQogBt~xf783BsNE3v zd3AKU6<&+|y!ryv;bd)OY{uBPaW}#<%|vjQ^A$D@*2l;m+Rk!2Y@vt@AA7Dv_Z#sR z0JvG&mzKEvQ6QSV{PWqxiqxw207kX)-JJRDKVMI)vHWmwm9O2pN9U$uhaJgK-6I3o z`Mc=$7Xw2fkBLTuHnOtNFyHMe{BPoH5r+L5+u_%fiYyTGNh&*D#d7hEo4U%Ub|G;N zwI8OuQ=yrCb3QMi{C^ww&3g=;1Yd`!Q_CjJ;F0wIS*M<~g*DgqBo-{B40LqwP~Rii z?vSu*V*+AQ1m7oaKyr6&KcClY;L0Rd)=29={oYC*BNw?x{Kt5gOyM|IZh0aacL7n& z>$776_>uvF1!Ab5i4DDUy?Ifm)y}&G?@2U~qDI&}S*6hhDEpUIOA9sT+>SA;Yt|PO zh&~RjLJG*ojqgt)2GLc5r3#1g$jv3s-?x+P*}lB-!#5JIr8`h4U6Zjyh0=WQ7&PEC zm7e+iUG98rzC-~k$wsfk{2OBS$G3?0Im*LzZ2o=@YwEBfEVw~i7Ze=a(fBG9pBCvj z{hBJhG<-cvl?6cU|9m8U1Vy(`+kF23vLP~LT02*?uO_$9UbP=>f55c^8l;<=*;EJw zLIwppVjJNNO7C)EoNLa)=uQ7qpUFZ#(BZU4D*<=*+H zvw%{ggU>Q6-C2-4T~|4aQ^c0^;Z$$G^i;@r!Ylt8upmc_ zB8xjDWm3kjUcb(NGvj0e4!YqFE1y^sY12R1dcH7#LwEwa*7pUBoay4VG#R&zj_acr z_CGVM#O%-wmyH~NkUF49W*QzTp8q+MPl^dzTzDT?4K`!YQeA?rmJj>eTlvZ_Ibw=| z7fuM?1e2n_5pI?RtbKizdPQ7~amhR3iFo{`<%TJ5G1<1GeDnz%c8{SJV=6aR9(X>M z$GbYllIGIpf7U+O((r0abZK}?cI(nK-MyEEczHadPe{M1Wd~M|`pds9{)w)&xIHF? zKmJf%;S^_^d|o80;H*nu=7t8TiH z6bPx2YVL?{tP6ME$qDtI$qvi+32_E`UVSu<`||J0>`7)%Eu}(1Lif<6B$|kMJ1F~S zw>6mK{JPiGs`?q<7S33z)eOjp+<$_Cqk+r@ zgq$f{cR9x@w2N>+>%!`92L3Jpa+atJ?JFQ+4uPy8@{woXlJ}Lb7_)K7D`IY?jWxNS zk=i9!MY|;TnIA?>*=W@k34e0Cb@PUDB6G)&#IoZ?CqY)bu25tS3vz;5F$F ziPsI!Rlh%g`rw1XYJVC7;U`>jUO5kS-;asGVPf5M?HCwjvVsuL3@+J+TvEz$po)AX zv!+xFuep8Ugh&#lR4<$4v6{M5?owQ|ltx0-mJnBg8ChEJTRMZSZkK9ruNpib0Hl zR;+@uVsSnb+Tt$q`WHcS-5O^%6EZ8xWbz|-i|;NWjgf?0E3LrHDp$^{-7(YaFGWlQ zsv}g=b;^|P-n|Q%ez=>TUn&B2ZXYJcaUQ~)2ZOreaN-2z8((75p$5@-w=A3 z*%OsSB_RW|Zn&(lAHO81(0<5g!$73BxBnQ#i;k!?&Vkw4Gj_&L*%H?P+WFg*KcHD9X~$Pzt$Ulo4ssz?w=9E0 zFk|Q$odWgY1oG#=TS!m9$pP@`ExW!KLzhd7V&Qao4fh$Cqlqcwm!-RgTkl^%d{E2l ze>C5sXX-2yF0NW3r{vmRFB&}Jj;1iJ_EU+hlnpLdNkN|0&he;q)VWoF@G@7tXtfdG zrSiu5DVdc+jA#mh4-mP)p@c#{>`i88b`gUkoEDw>Q1q#cTEa(nWiCF^x;|fg$d=$9 ztqXv)IS7>m^!$|8|(#Y-tXk`S(+%22^G%NY^IxAm+=_ljzaDC8lw&4A1nRZ za;1uSyW@ICsRY_YLfA})_nS@H9v<1EG!t97vfJJb`uQW~DTRFW8@9jnI$^Nbbtp%C zTOTW|rF&A^^tiF$PD6bFLH@G-69*$!$&xeqv1J2jaddI{J$c`5M1-nXJnts*snWR_ zL)=7(Ks;L~dJq;g(D~wXoY~ru#g+DGG6TIz8u2&J_C4_zyKpSx=pI^H#!;vv%4Fk3 zP3j%uvtmEp(viH-y4WMrL2U`I-kfY015!6%OC?*CJGbSZ(t<-aE+&dVdbHWy-nsH7 zlqq4`M|vZc%2?hOUs!aHr=AS9KVTFjVYZRI$vim1hTsnsft|8y`S&>Gb3uO!pnVnU zrpMKs@?1|E9?iwL*Ii?BuYBys-c*lP3xLG4y_AL8u@aRRaAX!PzFGC$y1^Y#lEbXb zVOb7MAs)T!{;pM}s3Z{Z&JnSJJtuY+mIhNAH_BlNap>m8_1#?`sBLkWKW>Tuj@IVH<(^TuefOe474 zVS_lSds?5~c7B9+|K&E)YUzg-2uDUrc2IsF4C6(CNx2Z*ti7EH5+Vzj&12Z)!U*CR zIzlm(=to}VGd=5=HTWV~J?=)_iK5N#w%ig8ecJL_sLT6xz-n;1*T;qAWwzrB9Sc|Q zxXK8_o%N9Y&`qdXw%k#Uq88N145$6QiMkZzCYkN>U1@RR@6IyuHk0FR{ZVkd`}ftb z+KTUqF=2?8tcTcZ${R}4h$hz`Q|c|==c0!=`(cLkal-zDjU~OsSJN!|+q#)lia%W= z$4&uI-v6dQw=O{Am(?>t*p1HaN&t;ks6~B7inpYn!vUJI!P;~KYmEy ze_!TaQ=g5|n`B|vHUXP!rb`Mbjzud`NrhUyWkeb1(`9zkq^C~ayR`n%d<>`#KrBaZ z91IAbGa-gO-g=f~f2%$QYWEi=-I70RFM~Nisd|EkK9>b1`)$vbY8Uky@ZQvBhJhs@ zL3{_o7<42lP#JATke$oV5fXBR27-9!CvEdUGB9nwl-0m6kP#Ot~w6EYLU%R>7=gX#%62z9POTU#I~QEpmGz{4+LI2G z1PBkx1#^(vhM^q^&7fqvWbRcA08v*fHMn$R)%|s~nd@^iOq`L$-Up?Gh}kbH6Vf7X z+aYruXK(Q1_%!hvi=CqpoUWhW7iu_247L1m`K%A2`3PcSG(^=Ej-T>Ch z_&WWyu*>MNW*o1jq+(B!i(px9dOS#jp09;plYOL0@{jqpyKe=N1@am!?J8vl+HrKll(jGk!w z_;tGDdS%#H-d;HbAQGhGgE<&dn+dvJg5*0Ghzkpdz+Dnk(N?s&c^qPGqEN5sI2pP% zY*+u156Is~43Ec9u_qv=ssBojzBv3Q`%Os7QJin}8_tuxqBSI2)zgzmNjv(Bi-t;O zGPy`86G@a%*d5fkoA{hPqKj2-Vfo~<*Vz*1TubA%264xDq%3(b1IK1@GQLqUNo>6* zyw+;ZYsPn<$QOQs0312w%A-1{fpBUQ#>s0T52uk15%$c2vK7V)`d|kw! zpb$^U4?(BB`o8YF(@%#W-86Q4$)-$AyQ&MC3vuX?b8!#G-L)c=s7Gu-)v-2RR}nT8 zJ`ExG0gw%j)GLQMu_Ek|(uH^q{blr4rgJi+_(@M2yn_58Sd`(3o`SzH~;R?WAxE$>e$qE&HiVi*^+)Znwpwk-rn!Ky1L}3(9Ul{Z9CD+5b7FNQWwBlYUeG&A5-u^!i z^o@VrG(3dAH;vvQpl6e+SFBy60*JVEqwD3vtshr^iPy!`?f$9}b-Ln+2=m&<{8fed zq4rcwojKuu-Er6$RE}?i&E&rW|KsHBd|Vs=eDr`no*3f)5cbwlQLgXaxPeM2-O`;R zNJ}$xcZ$+V*U%vd(lJPvGIV#B(k(Hh2oh4#-Tdy^dw=)uob#S{z5lpeEMw+*?zpb& zQ3vTbu?00eYjXjJVx4+#_&6bEA! zUh*ABmX z?RyV9KI;#|1v&EzD%s}jkh2OqMf0jK78)oJiAEm$9tdC70cv0=uz6r~a)Zg&6M*i) z=2@I%0eXpd4@)#KEYy1nhRSA^j_Y3!&7NYQTNVVyg^s_i#Uw0#?7wDDv8=dGaiwb< zQF#~Lb7N5w9Z}x!Ox_{WFuI!WRkM4=be=k`L>|aF0Kq@uy^l!v?9BSC=?_fqx5wKm z!Sas2Og~<1C8crry+^lF(5l}-f;gpCiZ@?$Mcy2)rhv7tiBs^xaS1Y*CZN%L7nai0 z)P!ozV+T0k!N6T&6H6&oDw<&Y@!c9eqZH=;fNgYe8g2C(x@HwqoEH7mx#=F~yQTj8 za&4!)*3__>C=e0)N=-mTazb-rJL;*Pi_5)x&kZ)U* z6o8#gcpy}k1NN0EbAXT$PQf^kFI}DIdGlv!Pdu1b91eazCp~E(jc zfIDv!poU-Gk}r_ez!>zTc?06H-LE!9qKq)`)dc%+eP0RwFswP+qjrW?t?~;PN=K1} zf{+djf=~1c7C`PRQx1qjY^^}nfHOBYr|GxR3lp3?iYf!%{ynu@-)5O)z`^)i*YW#0 zjw)pHF)24H;Qz;U?X8ajsxt2eHVaz~A?UwJy< zrhaGt;5I4Wo5a#IzG$=kV8PFoJkNexyd=OjhEGyWYhrXDSW`>?k+D;~>OHfxGp9&KO=#kjW`Rd4_FLZgIeyh3q;WNjjZ#;U^bP#BMbcvT=s$$fUPye>OJT*jDLOghszfMs>3vL z=)ay0Y%a2Z>;OzS-m)5CgJ~VRu0QnoBd|IK#Up!&X|Y+s;rZc_Zj&oJ=uE@^dxiME zGxWBuBF+o73XS-#04Tawu$Oj+{&P7xq3SOCLbpdB%=x&Imo3OJeEgkDeotXCG#jvF)A!W!+x{nb1m@6PB^%gVd=KG5{EY zC8zQm`ANxGcM8bvw!UWW3G+&kG2&;nbDEk2^`mO7jUI?n=t0H#JzJnxm3wBSY-HnT z0jidecR!Gh0sH0~vq(9z>$G6F8K{H)A+4zCz4@E8f@e+x5WEFsDY_@$)9b_raEgfwU#aC3(MW~80Sb0tOYud# zewH6xy9H}SFM%!^x^MRG0{Q>@>b~4m1eYZ9{v1#qBCD`Z5EsZ+ePd%7#Kn5DSbVuV zG8y5hX+P@t%F4dkMKe9G(ipj@jhCk@ti0PQO_ygvHn)%&3UNR70mYSun(Mb)w(vU$ zBv39x#^3S85k(e#?HPE;ZpVWJK{YwncS=)ZTUfs-lS-dIc6x1&)js0g_B; zvjE^n$JaE?@*Bu+x@-zOWY(aQ{6Sdr+;8HY(7Fo{PxlDa@oFJeDygyUBtuhO}r1aPLOQNDEYO7+cUD;TX6_qsQ5(G zW!c(wXnM3~2X8}=)=&gK-WYq+cq@iS_OzNTWNU`Ycr3|i>Ck?odH0u`Vp=1(M>2$jk()O*4eU#u`0!|4LK4q+cQg8etS*;#7TfGj4bhr$a!su zk?>p8Yqgxa=(Yd`E<1b1&5aMdD8Y7Vd#&s90KAjpRx0hU49J^}VeprOJa{gPiiMY8 zKnD*t#3b?@oz7FCY2YMe^8U=f;3+0ypQqsjFwbxSk{iB}OV+vu=tpFG$AozSX{x2`m;7 z)_=1Z=)1I5^T#bq?g#bK+U&%@lGd5^mZ*#TRjc*5jpno0hF4dw4&VXL8(q93(H=U62P7rqAU|OnMsCBZ$zrM6j3INbg)2>kE)Mf;X{zk5bTyYM+5&I z+)pnRsMOa_j_tj4@1Dg~ge$u>t(_s!*1R8?*5(m5%5Vd1k;C$&uo;IgmbpA(_qXl4 zCN2048in#Qb1kRX(e~TjJu07L*!^CKmyQrKbIHX1&KP~dH*5<5d(uQ|6EboB3W6ep zfxtUN*?kp&D|NWI+@rG2(@W`0coig|5e&Buei$}s-WLz}YV_;-y4J`95}r}Fhz0_f zU^N0n4}JmoFpUJGmGxNmUlEf4v;^i(#JzTjL zi0x!SPD3L?)uy7ce_i3Dpw=vFw12qWbi35&_Z`L&%~-oD66QH{8BVMBR9lG`rj==- zgHz#5?ap)H;nAIf;GFbWPPlK9p{->7#^dYT4xu_b(XY?jlJj+IiNgp_)T>L$GLp@= zNi-gvl9}(XoHJJSu#(7nq~>pu@=1#+fn!GvPiZYK2h`H;Oe9?YTP=+uJfFs&_oAS| z>V*1lW-}y_ph|u%dBMBY6QfYNGLXX`$YsfvIo7w4^M&bcNx6+&N@CF7!=vZbEr@FtyF(#hv;iwo=X zZq2>o8Y4`wihY`%s4ah!d5`_Y4jGt?yI1Gx1G5Y8j#ZTb=h4BoBm)wPK4T+cDzuN3 z`y;0JRsVlK)Bpn&*@D0w@|9A!(ST9XbiE@j&<+Ox@OE&mV0lbmEY=fOJHHLwAv8V9#gU*MnXG+({T+~t55e@C4n%=W z7L9KXIu#Qv-gS+BKl_xU=}{oj7j$qF=HH{m2n_(sbDAD#=6rgAT*OHjbj^gowT|%J zy?mTZjd}-TkR~Oy@&PDrO3w^!(`u-oJ3p}`bA@1|&&nvt2?L+h)a}y5R5|#JoN`c- zb#At~!4Fm1!p!>9XD46W;kUZ-K8yU!0k>chja2D!4JdB#UgD5UcBV1lS}lAA_uny` zk3d0N>ZlTXHTSqBdM`LQhR%_OIy%a36z7?MdT=i_P3YSUg`TOftk;N#(MeHqZ5$nQ z3&EkL#`2R=1g(R%K)zL4&ZdnclP5du(TCIvWRJFS8gH#^av?OZ)ULCm{z zPDk7Ug$L%PKOySgwLFF<;CftwfHEX$6u=L`z)~a?l}eovczM9+dUQJyGv{3?Zfw_x;R&r@_hn*HxEwfNUmHMk*06b&>s~`ueNyN`omv#KMAngwxf3c+nMZrz66Chh?h$l;`)| z#7!uEjK9zavcmJk2mEy@>s1EFv&GG~X8a@~7_Upyf4;A_D|kZpz3zWKmx2Gx`dud= z+Z<6V24v{a&jmIfeX*R3R9Tg=AP!mcS}DX-Y^W&Y$-U-pkx%sJbeNIFkW|=Z*F#W6 zc0FX@c}fn-J5Wsl1HaDoQP@NR(9QlYtL6f}_|IZ3Wt1X4gJdBCt{7+dM9F%aCMXN)hY>+wehe)m+4m&7SWH5k&8?5<9 z$TiR8iSpZ_0LT&Z`{mCmKxfb9v=JB>7_zJb7`7j;V*cr_Y@-mc!>24tzKdf}M;n&U zIXsMNOGtvE50 zI9DHMp0L*qu1?ucLYXEZt{gtHbFPFOvsbHwphFVK-zPRe%3Tf01AP8`2|>UT&VGHn zU;L_x5ZGjP0Tfi+{93!zF!bRQ(xjDNFfY^}d}hTh6rdB?@UJ`^`j#E_7j5?NS{^Ho z+zayrq#XXmfLrO+M3{mGtrx)W{s?MC;4VQ8cLVTw6!1ctAum{-HY&T>1os=IMnXOo zSQG0Xs%XCh#s$(d6^W?_1i4rwi_8JdKk!EFJZ%(;fPV?~n@luCP;_+k=B0*0^lO1i6$4YSeRc0cGc=wcZ`k`% zYYCYlh?y2Ix9Xh1NW4)3;N=eH@wfyJ-JO@_lykZ_XMz!8Fmnjfqu>OM9ocme<_jh& z{`GBvK={KAao8;IsR{xX@%t5ETtVdt#J$-UH^6@2N24;*P1dWNw;zKyc9m);k2Z5p0&P-@mJ9;!~*y`vM=(dScI;D-XwvWIo0%yT5KcFoqBj z1RvPA4N$6EU%&G&bnY`l-Vx{jg^fl2(P53>O-LfmI+hEdLMv9zvyPfY*~Qv9w@WZu*8i(wCUD;PwKCy ztSvFFa~`1D1T0^~|GVoln4{^wQGM zD2BNaD`-Ep1Sxrc)WMwXzfMb3Fl5s~J75YX7832iaHt#@dpTZ(#g2^{V6!mFMfS^h zy1PFw=$N+v?!WfUY}z3Y_))-`)-NnfFSRT0dfdx&);%f*^GUs3yv|oD{QsocCh;1J(b3LYy*-rNmeh zslky7xT?r$8z7=Q*3TmxJY~(}ZDR?yq*RyQ6aq;cV2Y2I&U}}Pyak>Zfwb%bBS6#Y z2U4+lz^>}}dSfEJ1eWEF)#(^o7Dic)eTZ{*wIN(e{yEZ6D>0P=@=dl?o8W!tD-+}; zUGXvE5ky=JK%S<&Hq-9`X)!F=5-|x<(~v%ALeG;C#*`ZO!IZxc1K>Dv4YWq^TE$e!jSOHv@&!W2CWc%M@0X3Ip8abyCy z(jIQDw>MrBa_N?JZoi<@IQ46#St%UlREOC0r{5Xq99cs(DJBN-zBt73luq=VW-Kjk zGrFG&N@OO?4H=@~GRVDmp;(}e^s;i6O5zzN1^JUi07z#o)PGusOdNQ9?hJKwzdGX_ z%a$`u@M9v$4ngc6A9sNffK6wQ(Fk@y6XwG2)9|8#cAM~S7z}?wj9)VD9cqQ7aQ!q_Q$RzD9CbU7?{97V3;fhJDZy+2j1qj?_mg-=*r5qOu?? zsl20~F~QS~dv}`U&4OiiB2vo28J|o2IuSi}JfDUl)^#zHoupUy$0|3ZHb;aj0aZ0$ zuZm%P*URpL<~c}KCIf_jAaJ_f%>I;<0-(SDZMGdtyf*pgqvghuNAotaWoW9cums$+ z3d=vhyc_K5S3jo1E&i%KxWl5}jLsvCzD+*MN1~wnw$@4Coqm&9X{EGv!D6fxLH zXDrRYh9#SpNo(y)0sORgVrnShOFk5g7!#702MG6bdKw{x1h&R+3XqpSKP>mG1syZF`59^0EW;aTi|zRgaP!dB@!A)?eF))@Mi5v4%@vj`?gq zmRhZ!tP}B$PmVkGxL#XjtNA_SX$Ezfit?+e?oU3EUvqw8d+%nTcNMNq;Zq0$4ipit z)EPjD07!_A^p*jA+|?=qsJ(F)Kyju6CD>>zF=`R4wfkAvXk&nrAA)!{-zo3OzlLaME7w>}l>1z|3MZ7j#q2Lz zUR=z~FN;+Mj~_|n2Lr*%1Ew@4=qBQ7Bu*;qWiRy?pcPbpbThw?9765eSYfa(UBrx zzMgY?CPjFsbsS((Yz-Iz~;PzwY{WZW!3x}8LZD%w4( zmyNLk4bjWUWZ>l=)1<5`m+?SSS}gF_R3}@O99Z{m&DJQC{}CDa4;R4Zj9%Rm#8>45 zcV28o*b2B+z->NhZW=$a?pOk7hXmu@`~_Jtf*|3%2h7Z&4|ITWnp_L0G}WfT&%&cE zzGv9xw*z^m9zY8KhDvjRj19m(8sl$V&?z<4^RBa+U~y~?Cna!&o7TmtM6NbCECwvD zmq0eLGY`jAL_-@w-@;Abf&Dv4K$HR~JrY1>Tx@j+@DkEEHd!U)+Al2N7y!{L;7G!H zIg^?IjwHVDnT*!Ij-;bU=q-y;|5lZp1SeGR;Q4v3;o7y!lUx{u2(Tlv^#}srVNz@& z39)~A$~RZW4F9+@9)}SLUOuA=CZJ%0%e6OwJ(bE5gk4#N41yZBj>=@c)faeYjBqjt zS}0s25oj=~MY3O8JdfLfRa~bi%i=SsO;xyA7&iQS=n61ZP1iytXSV^xK9YbEjxz$( zTNSZvUw4qpe@D@R2>tN7|Jk6~xO^pYHjQ*(LVCERo+pw1g9fBH)@}@ZeHI7U65!ji z>wv%K?FAIMNb^(RD*Q=E%>^Z#g$VcV>q;{@JTMVBM~)~#CJF(JySoHr{{R&d+#zH- z)7ParA!uyMq|?+5~ifpzPHnO&x#XCUy-o^pBn5Zl|6 za)-&adw21G)$ciOrZ&7egE11q=dpqC*)MC_9KGPnG@D`I+M8$sY_b%+>8g;xnB$6 zgXEHL>^iKS3TfI!1&S-1K)z^BEflJRggTUxJB;HNFXKCs#;h=mkq47rizg2-CZxrs z;d?tGrr8n}5)#6hmf={N68>_q)AI})NtSpPC9%1_*rdjOS`xb&x97gn3xnU&1W1LF zEP3c&=CT@CRD3u;n%wLaKZ}q9w`4eg!69fWRmW)LHzh59t{XRkRF-`z9_8oRwXrBCKh~xkP7x^e7JDCynxbp=+-5y) zn}{NADbl2KX(VHq%rS=?N7{qAby671J&>El18*iL0Bjb8s+qX4zc!1@(IxMxyyKXney6*~0)EBXn6Oy92PD#N>`PdI^7TWzrf>%El^&75Z=h z98Wbw9ZBC3%i*K>OGU57%-R38Bq$b%=ns8p*>CaU%yt;CZnfN(4aauJaYa+vNf)cF z9N+ut=ExRH&!=lElb9Oa=7tHroFnGTKP~dDuQwv=mxp@(T8XLd%-o-ym_Xdb@6_RM z{4eKgvQEPQgbZ-J+ReH(XM6ypOqAubfmHae2h+fQOcJa$DfkM2($G!102jYivm(Z6 zUzLZIh=_`wM)37x?vq z^$!@8xk$Z5Q4#YqtIL?&{z``?B#PR^i8OIKWo_b0RAL$U=ZmqJdx0ASr)0d<$E%_t znusGKRLqk#71&!(oaXUvC>F=%qJjf^_I|;##;CzBvgXt3T=yRV4q(d_`{y_wv(lL4 z$GL;!8A8F`X#aGrJyd_J84MU`3P1KI*I7>~lhx#q2|*5kX&DCaJi}Ii<4MLHC0(eY z9f2$B4NyD)h{kn0g$ik~ET1w??1?4@24PHk&cMy0>-2?}X)P9K#d~pJfDfQTX$5>@1CL)As6Hsc${%F$tu= z8q&@0ctwtKw?!@;*?08sr@A}PKHi;&@S}#+Hbn%6;!%CbdwPKX{Ui5I0aQbC_j00* z!)|#noodd*^ND@^oA;}(%4DL+s@?!xQVn{x zW$C_8%wtAgm#W6%*)(YJgsRFp7a)T>faRZmU0of2g843J zK0X1N9+-5!`J2^FmW@$%JQT8vR!(~dT!xLYQnJGr1gf9P}w>Zgm=}bL!_r7J8 z;8>7sjTKOB_PqNFF?Hx$v%C2)?XmEk@6``rybX_VyHwxIw6E@Fh=k1C14C;a+ry!?;oE|4?Q$ z91th}P-fm6Wh?I`RMq#QW+!u;Q?aYRSNZczcIy)Vb0YSu1PykBdzd z93N0?#ZQVSkv)rZ^XI*Ciag2nzLLkJzcE6)=Oj`9kq$y9fCSjf>xm0wu6KUmL`vPq z-PTpUW7; ziw1D~S#Im#!Q{$cbsKmNXY15Rt(g|<&HK-dC!7$ck+VlwlyAD{3iur_>SJ^&A@8c~ zCJ6f5uH>IA@L-5h;IrSe`XLEi|G|g}xagy2VpsJ4%QvJSuHF9G zq~k3!_>|MsiWOwF+rD~&ZzCeDLKgbO&qjJNKuR-#f_bOoGMc}6pRTa_S?J@LIqqah+`^L= z9D?Tdw?qNZJfbfq^A;g%v0~%LNZkss2oNW&<8}yrnw+L8=XJHtIm`1ldgN(luMQKX zw*4(0Qp90K-sr9xlCR5O>AenwuAR55!#NO2_6zSbeE*ij8_4}#VyK`XI2B_p4V&(C)lbT!BLc=&u;GIkx?U+^nQFKpI!nV3Q1|quZ3;eCRa>;)ov?GDse|@wp{H)dKyb#roJn9#oOy@2uQL*&9g-WDoF-m6s zC_s0nW#fJ5==x#6IlA%qrtOk**^1xP=-c8}VuS#;(a+@9)HnRS$mq{6Xa z;;&vpR+SrEe$$ZFm2g}%<=#;2kjy#mEa`E&juxBmF|QVdc1&Xw(5~rNos}}&d0C)Z zvkiv1nNF|q8IJz}%vCiEP;K#p%*pUy2nVKb3T)GI3s8=)km#3p`UsgUst$Dv5 z$yF8S&Igzua#}3O78^96Nud*rD`E+LB-q~(F05uLm?6^_I)rN^pr*S!;)R>)=+^=D zsVVR(Llvq{zj%Hu^WHn;;JxlFA-g_Z;i+t7 zFS!+H{rug+!t!S_19CKyB1B!eh?Qn1Ws+mfD`VD^f@7RepS1^KBZ~JVk^h zQl(@KQq`&-5}>N3njUPaZRq$9^xMpB(*)em@veO(moJnSmvD6;7cqADQV>Dw(MqD; z$BFL!uN{)Iz+{H9(~%lIqdePZpp5C_9ndHzgsfGt(7cJ?)XBaDF$oa4=V5@J$~4X$ z|5zcOCH_o)JDA~6gb01C1=1iG(>sZx;(dcpf=>$$d9TQ$ z?1VqKW^GzqVbpx~7-fEGJ|z}6^bt#UkF#cZh9e;k>6C!w(IMc62E-^BAA03 zT8LP(^c>ZwcHq2fF~M`)`584VT0_*6anFK{SG?|B(?=Q;%e9X_0WKABhz9Sgl#udF z2B}^7QHdiG7{7qZbpBFCGV!Y<+w>*qVd^`~0d!=ltW&po>7Iztc1rLib2omO8r z1hwrH$V^wyes5l%d}-mV!_781fXjp*d7ZP%G2Qm(PKJmF3ymF%X36^#;n~oNo)1x` z-v)#Dy_VQ;=@j3aw04a+k3`d?SRSnRO*ff-=J(VpHsi!UW)I7_@7LK0`?R;;A@i7$ z==@|f32{DYh(#&XIahG#aG`6a91LuvzJRoGU~KeGwFgp{Wq7fp4+ zID}^x7;~2s^$#^u<*_K$mGsCT6d(;#Tml2~DsfN{dztG|eINvIDu!Qg$S07~M#JL5 zbT*Co%yw({tzj8+!c7G#Z8s+06-BuVg>O1HI_<;DOw-g}T3z+0e|Ce<6;=U>T$WM+ zbX@r`v$C5@tjDCI3txhd1jr}g(kL-5Y>KmO#DAn4{FoZo%YHvq8+>^|gq~0zO>v3J zQ~0{?Jf5C-Pxpt67-xZ`v9TuW%|GrZxG1nnvva$jd?yw}9*r<$qx2U> zRnDg_Blt%-m3X#%owt>Hy>amiz3PbU!Ly%x`GXy9c{cI ztuACY8zu3?VewgotQfSJ&`X_hw>aw8ZE&bMbr0@PNLX6ExLTvyl1oA>kPKA*pbhO5 zIPa=T%ra@{?~h~|*N7fULN1PkY@&kkCP{lTVd99X#WL79D`O!dKZ4@E=bJwlCB)Oh zm2d9Cc=;skP?s2wPYWs>Zyzd>{*L}~Exz=E#;94r-8AmD7wBt&cAIku`n#Q=x@19l zUF_gRz2#SsZVjV*hdZb*g3;w2jxH=qNWKl1_W3D1SFt?pI^n%$5C9}n4=g5VlaA`O zkHtS)RuJB^I*W0XFfiqk(f_2KJsfE;rB=bm=~b zo)mRHPME3xdy@e>n)DWfuuS)TP*p;h;Y)ehbe%;GSQ(@gd^R)L#BYCTf3HN088)Zv z-X=`}SKxUpkKZ@1+bvA#ADvLJtg1I{R79%+!R8^WLB4NJt?iax1}H7n0)g`}wA}C3 z*PlG7Q|+G~55>*RjZDN_mrgDhxd^6Zk)_N4@+3KVV;e_{RB6MW53F8O_a*{|YPiXfm-s!$XiUW03L-2@MGmTl+k_C^q8T!N3MATg(g3QQLoi*5HFN z8Pn`fM$ONwWw8)$>$~EwVB7w(x>kKu#qhl&HXei1Y3}=y(n2<;28zFAr(X}{kC5{# zzw|JM<5Uum7A%EEoXTfMKfLmE4I!7tk49Qs$cmbtBdDzc8NhnWgZcElnM*`a#YyrnEgE3yr)a9SvwA~H5gQA#P! z1``Mr+ija_Ijh8fpYR(|DQP)XgKD8W zzIf7No<&mhl6AZD9|bh-t7ciTONz?-%F1EqC>gk9V+QGK^d6_={85AFWA#t5(X*(g z2|2erl=@!ol&sG-(dI^^K8v!^D%HT8vJ3pm9~;c@M8pwa=?BmA7*X|XWkhELZUKP6 z_miGOZRj#z>|(`nl-QZbj>-MAp@n8svWMrlz4kSljPil=A;Tb8Ue4bi-ckfd21A_qGRI$-aH5sy(-j5BUZTw|;vF;aC0pA4tXGXf`kzzp^ zfOK>Reh){!l)e907H0(zjUKiHW=jVugD!tGM}Y{2r5?thL!;uaKi3tC18U%2l}Lpo znIH`XfHavJ+L0UI5G7*Dcv0_`Jj8S-Ihxf4#5Fj(0uuXS$X6JwFBJ9zY+cSEwO^* z48syF`cEEX{H%uF_a&VsUzS#u7+@%QfFt=^G*n0q`_>OFg?7HJBa`-#8fbqU=~x8S zWeUkS%TKYnt*7rv^%H*m)!KpWeTAx=AsmYF?e?qv6)H}Pv-bi?_WAr1y+cd<#yKIs zsTk*AU%#6r0rZ4HQzmPG9K>df-J?_QnRdFYC6lgYyB$b`&lN806!cgqZd)3;uvRh@ zRN2gP!Jb-v4Y~XAN^o=|Upjj@>MQyBR6`rTs}zoS_ckzWqv5hV?>1j@PjtCr;a|xQ zAsnv))>>GzK)?1cL5kLyL+-!GG-Uf^ey?m)v1D{*;D8w)sFkvN4Jz-#xEV3icw+dD z^e!n%q%cJVO{55N=UaVPqs&Ib$vkn!b3M$WYJh>tt#Ql#R5y=Fn>a_}@GJfgHBV`Z zYzo9TW776De7oj?s#Ke0(W>ny)zIe2yM>f@V%wXk{UOq%zRitTXzCdcz$>v z7bya(C-XS~ci9QJ-nM|l7UwJenBdzHDxiHEMS-9@D}mIhGaKLo_jaNX2+>@3Ef$Qx zxEa&#%_E}>@b>PG6J=TDP(Dt8$)@jTU5&yWSw|Y2!(20m;=%`}{N^kMV#%|Dqaza# z9+x}pPDFhSsF}P&vsun%Cve1BK~ihC`#B~Lr{Kx- zE*?l^p@ka8{8PDnb#AP9LdJt&Qu6z|h*H*7ltHmZ7n1VD=-yt1Sy70vuhjV+W{x)q zB3%fX8Xcz8FV!sCgf^wmRJgoP>+ZIE{NmL^5%N^$I!lKeSp)&pjK^EUngqn65WwCH zi*4pd=D=ms|1VZpXf5kavo+RWoVOxNUNdBujCAtvh?X(Hx+?>0?NNz0B7|_B8#Nq| z-pRds$!GUd3^X*bVRoPDY)t6RnW+J?Yijm4sI@{XO>cvYZV$r(I1p3Jt$*>o<<^H>4t*W1Gg$EEF^0a{M+av z3K)qP&%)m^w2B?J{B3oC&EX5ak7kRyjYiU3jaGp=y;^(?pes!`dsOx2-~rS9sMkRs zKZdcCAV8iXZJ1|oUzHNRE-QG6Sd?Qa2( z!WsnBfu|O2uG^%5zf*c4Ij*jrr}3RrHkpQGwl+VA`GwBXSo~A|R=?zt%bjbPGw7HE znTV(Ml~IL_n#(Q%)u^G0Wb*i`i&ZhH66MeR)5V2g6;-O&n1~WR%xlkDE)Q1Um1&RB z-RZt?lVbb*D`Mg1)OC&;>5@m(l8xL+JE9|Pnb zo8#-CsV*aLr#MmUyLHu8IptUU%0DyfVc5Z1T)wWLFFn0b!WNjAeOVZYXNfjH)Cmn7 zJoh;a=_NyQ4zpD`_g~Pn(%lP|&(MIf7syR;c1IlpbU~E0`*03hD*STurtF$E760>RUo5f-&Vu5!kDb*Eh zYL(nmS&yPsN1I|D3jQ|j=v@xf{9V{+|3v8G6esGnCE!gqi0chl%W^tK!iih;@P`bh68`vlNU2KTK*jz+ht7L3|Ov|4m?|<+DBj zY9CM|s^4j34gjU2_=YJU(o(yeov(fFvIK%y2*~B`+U@SCW#(F+dhJIL^f`Z2;XU}1 z;c4IWc>pc}FP|V2a3XHk&oVIVi6XB6?5_RzSw0~Jx%BD`t^Jejmzh6`_V{`e=xOcJ z7D8UEmBI~=hnHT|aThGYn8}sXS|b4HNb4}P8^D%%m@=_>+*#_=gCg*W(}&dN2Nco^ z_V9yTW{!eA6I2cNy2vb-qp^yRh zm{?-mTFBPQP&$?+N7^@pURyFY{g@gnjmpI-SxEPBDz!)D3Tlfp#_z&J{9KYTd;UhZ zEoR>Y+~*p%7&2wV2Wv4~jYC!;A}+V4Kxy`|JL97=H!(zakVx%5wdX{i}H=c5#1&YBgXC^QP^S6AJ^!=97MtQC zM5w6l01g&j=thooCpwGrsRnuCi)o!rL9t9z zQW^F#r_iPIrx(`#*`%HK)e+jv!4$y?dL)d-nNDwqCyBwW<6tF99nvx zwg<_H?*^exKP{k*Cwh`pp{ogx)&Er+xi0YKDZ|-$mPeNMl4+s-sP!8xwG2Zm{2L+1 zBm0l2PcPEsVH$EyR!w~5&jV!@;#t?{UdBu3F=+MW8&vr-#oNv0(`dH=&vKa1(JPU% ztEAIQiF(TjMVC5}J^Le)F}(3>%PhOdi;s&Ki9p|o8jUhkf<$ZQ%kf4+i>GfenU!o5!rd7bURQb273uS z!==8_JP7K4acyZlydt36T{p=$85<}ohV^O zlAf8%(d5{CMuz(yntBNr@;2ShNI?o$TiIL*o1z#kRb3s4$P}SMZjp*t5-%g7E z+EiKH0h2aTrb0%^XrDk(!2t{TE76nT`|@G%@OB+1>B^P*6)Gyd%J*O4D`}tJo85gt zCm)lBLoYups}6CJjtvM{i7<)Z){yCi<%)=hH0U_>w{wAg3P>;ELp&o$h(0Z1K{8N*W*G{a(M|!?)Vu9$ZTN81@wi&C!`?vNphcff z^Ia}W`47vF5`)9EW>{=1-u)3tM5W7PpM%W zk_x*HJI#{QWc}FVjsUexoA}bJ(3A>iv+p!3gBhIdaAn!gI;B4gH{a>z;jSt^nI~55 zpE=YSvHl8)V^+Jf`+I0dyXhWR?moXLM$2oQXxQBH<MO8mAAZbJ z{@l6UR0m_0yc5f)Kw;-&q_y;WNV+-V9p`F8KP#;i`6DMibOKRv?=@Ve^Omi5&L;>x zI{7qIX7wDgYzN-e6FRfS3Fz7hd76-``UFl4&z=2+sNK+auXZx?voI!gA{sXhz-%Ds zYM+W@rHr+5N*SaKO!Sm)k&LrUnwZ~{UWwsYpe-hQ+TzW(B;1awCKEkg1|p3zEowtV zsAk20YvRQBN^b2=JiXy+K%LMm(br=4x>yXRlnCl`Y(D{)ZQS}&Nxt*$yf9CPbU%0| z$gXzHhWb3c`Lea-;SK6Y>nmYe~%1jI?9IQyvNF9$jAi*?5u&sL6DlZRQx6U<5~ znVd?)Cb5LrA>?|#(9uoA+Kb!sDaMf?>(mdLO(o6tdd8316i&otx+2A~)v)Y_=FEIkMWMM3gM$YnfXFpbR~$87x{^8r!1=#Y z%s(-n4G1ysOuxuAFYYQtiRh(nl}TcK8Q~uNTmmr~Ubu{pj2A6Teg^!%Ai1(UrH z5@JCshJ^1={jId*gH6T@dSoL$S7H|tMrwO)2QqG6Iq%+`SJv8$7lyW9%vIe&YH+H+ zi=Po(j1Q?E7+=ZGUuG7VU$ee9A11hT3*0Fu{0`DS+202Vqm=S18aq@4U^N?4;%;gAs9kR9s%_ys8Nt5+D&T2jv7Q(bum@RM@BQ2UAJ)RH~QCBd#(CQ0n<1KK2-& zB49J()Ti@~A%ann2OW4&GOyc^e7pmU_o|*>e|;-_Rz*|?Nk@i^0Ntn6eQJ;WxI6qO z-_gqa^+orNu%VFb0?n)}Tw_QCnqKLZ&waO@en;oxDJ3q`WWdaK_pWV`~W`f0=8r9eeE~#EuUYqVKu%RuaQdld;h1&9w$f zQQ_i3j@VYvr2y}>H&NC;F#l9*C}`irl8o_T){hEomQKx zuv8q-4c9^=hDk2?wxSY{5@10EqqRKre%k999UZ1^pPA|IX_I*p&_dFIaZmvXH?@k- zSfTI(3_{IRjBpF8fn-u1tH7D64=O1MN^#~EO+tCV=9r{VK3R?Vl>cNFO$CA0#NA(E zCJ#RD33@5eBjKh(T+nkK88+NIuM|r#>hl%VC+uTQMx(ozq}|1?h77v0Vu>$?IpfbD zm~8HwKe7_oH2m`@g91LBGcW~g)b?x=gQr}XzDf7DMSqA9u;=MrJ1w>8fSdmrE$pDG zNp>1w&`RIl6+L?SnD`<+g@^WX9RJz4W+;!$DLa(H`I(7AKO4b>%@+)rDJqb5)|+%` z3h+h}tWrCfEnUudVZBCrg?htKK^n|ZS-Rt^^FO#XM+*IGK0T&q(=$n2wTb$KtIAC) zfgCCrQsgRq`y#LLu+?OubR%$R>)X8a?G{=id!j$6Ae`fXV@8p9CZs=FwZUVZ#4BU6 zqTS~%Dy=30Qquj3MT7FDi zW;f#x6lh1`sv{i9psX5~O|DNw3J~$aZr4lD6Z$t0Om@L+5x3q!s`8Qur4>K!r%1i>9y7?62a}_`I?5~ zK5h4qc2COb<|NsN%UX?l5vZIr0TJYrPxS7ME$vY!A5rd=P^U%1=+U#g!)xV?MC$OTA6rq>;u)q<;t_;ub-26>}xIt zgUYd0A_JbXf=irD{1v3tyOmju$x ze!w1RQ_djukXmi-w?Sxzu-8%|^Ci=!JBDt@rOe6G;IG;jVKz=-?ck7UIk`~ssQFcf zgAZ+HC{Di{COBK+{fS;@_fzCkzK@C}iMR(x1U=PW!L~C*;ugA%m6nP{y5aH2&2~9S z`E5wSEhfvsf;H;^W8rqr`Ch07y_o+&gNto64m=9=r_(eH^|?NgM|apmDk%u5#}sD6 zD+S=*0Q!=ms27PdpQ0&*)fFlsE6LpNVsQhr1HtQ>1>G@)d4HTZ$d;bKS*@~o@HwI& zD4*8V%p$Lx*{GQx1e6jNKZ+pEw4mr4|iG<0?EawiEKlvt+PpoNJ0I zL81+vVv_2i__%>u52rKzN`~NALQnMPUjttrccA-Qt*gas9`ekh;M9vaH`mlXz`YQd(KCu47D;0WVhbupWq`)p$+Vn+ZA2OAo>Z$_C?g@Szc*!9}g~UQ_^u zI*VU+>3TV1Ag-(dJ2?C;tp z^e}t_8W>%t3F)o)U1=p-{fN2p zv;MR&)M7BnAMh{UfKZ3GBVYWh`e?}d6v>kPX{9XY{my?UT6L_(NMS;7p3qro+M04y z`DOn+kSb3j%++#KK!m8u+OCDlhfLcFxmgd-dl!M z*>&y05{jsRlz?&>F{|+H(4viewjR=!Rmk{uHD*o&YBq>dFi@)TNkz@)Is_*Y}T0H|n(+_6*H&yC+0nozT zPFzIT{P3qQ%&Yn(9aq!5y_16Eo|GUw38*{8(94{Aq+h9&EXDF&8SpxXE!lUmeAcm8 zWs?;n?HoT3 zT>i}pMDN>3Hr$O`1zKP^35|)m(DSd^b1EkFO>M)}{h*&n0;r8o>qM?^y*b^t1hs#E zMA#8+NyANTNDdYazH?zpQ(!yZhyBN@iL|#{11KrC(v@xE;^wl)Enu>%7aLX6sOfGU zukdzFGl+oi4AybD6F1kf4~B@Yj7DN?-5WSj8d2lrj3u>ZF@h5WNvzEdyWP z(pOb?NOx&-O2?*9K0aIv%+|{An>ccHq?M+$0~c+PByvTd{*je{*B956P^S*4p(J^2`8=S9s37W9qfbO^h>#=CPUgucLB5?c!HQJvC%E8*7 z?|czhq!WiN9qfF`o^vf6x6H(qrtI5mo!)dh@^k>?X{(cA(6d3qTmo|B2LPtlX3hna z!ry^EUbtVjMQ#j)-Efo>iYecZx*<(z_Mr+BY=(h$VA3Uz!@OJtZ#f055;TUqXv7BJ z!#1v>Ar%o;DQwx(beoiM)n=>_+wp-{xN1UG%eU#nx%LC=?eiaw&)BE|06hM3(vP~t zzf4qy3fl9=~cdJeL$EC>GaKp!` zN50)-==~ML=C|2b^L47x#ftR83ENd@tiw213@R~K`3ghjBI$G7AsX#_^*m?Eg_k}C z$^liVPJJAJcaPcsOFK2>;&@H*S5BIb*) z0SamldXYPOM2PIUB{Z|Wdb*dNvw58dZ1$;+Ax7=cp_U3{Kf#JTU*U$h#`&l~Fq!e;3*eIq+cL)cXuKi5EDBSxGx^&6QHUg<2B5b;4 zq_q}d)71_+0i#j~U(V2M{}xt~$I~F_EHnSEj|6Ht+@}2?rqX9m*W{)pu_BE?QN)$V&0cbDT_y~nJOKQPYINPwZ!b8<@olq zZ7|p2Vg-7*a$7eU3Rmzyq*Om5VbQz68>6D_%J^QT>|xJNyPY@9t65*U?9Xz1{(SPMRY0zW6*6s77>AMuu{10`-?p@9)7a`!dD+|mUQQ|u2;FK;Z>-*N%?EM+VaEImD zYc8nQ#ZdN_*pe`=@|~oMR&M@&zp$r8dsTPyvGtVGKKU-Am1e<*KZc=N-WFn>r$n5h zw)utR+!pt&wDW>j%9>i5UtQck{v%5&Vp0#KI&RIG%hnT9!_2Cw|>&mJ7BBCssDTpwV$sZ($-tr$BkwS*ADSkQL2+ex}o2pvxC$O!n zgduvF4<=qNEhfiRxp7T#ZgVXvr~1=2_#>>6bC0dMn`i3Xc8OA-SQZtM^BuB3%hSgx zYfAj^%G3W(OSouj1oj6_gJh`Jodb3OwWm>%fMjY)qBm?CDtxlXg2t>l{*0{%%D0id zuACr(pV1i$ELGYd{z6N83mr5WN-p&#Yx1I$>45l2Ly3IhdVDh0#U9G+w~S}PsX#2; zp8arw31q;aaoY%BNJTDhK_k~v3Uv3Yha%l{jKxK>P+0-5I_78D5W$}z87<=mwA;tu z_@6^+sK^vThNHtqAnAE!d@AU)w@Eoyoh*jPCZQgM!V#uCD~=h@#8L%aR4z(5Ob_4d zF7swH%ivPw`6l6svITz1Qhm7*bzr3xp0++#MGjBU(x&T8hfO1%Ef@*9Z{G8o^tk;q zs&TGfCB`;1>2Wvfs>fgsWW@FudpaaOH^v(hBx#a+2~!RsgT4cwK;(x7EK507wg1l^ zZf`x9xLI|2GIN5_Td()nk@nUBh3~hyTdcH8X#~`B`+-9`rCuco!*eWWZ?|NJtdyez zcvtHpv!8R^Us6eJS)6`CyCNi?JgQ8#$H{arafS#E<=;w=aSx4u@^As8OQ^^(K#6)7 zWIBX?>c}0sqHeFeJm+^n2#OH2F}43`@gF`j2hIB^Z7elKxr|WI2C3<~)|oQq+AlUu z_!9R-^w#Avnlz<%NN$O(gCf#?X+O_lq4U+n=|QJ^0kz9$cwf7>gg3qJse0SsE8C*` z)oO#~=+Geo(@Y+belc63`0dqtQYYQDm?k^j_t>Uc4q0f>$9kRw z@xTXop1pzdQEFX>24A9HRK@F)TPEF~WO{1HF85N@xMMCn63Q*XLMdidcMy#CMfC11 zRY<5(8J`u}T2WK!4*n2<3vaQ>1Via!0xV`(Zt@bVmpN8mmGqp{0g3!HyU|9{$5uCT zNAqs^Z#y0jZUs;40md_bjIz_C!@Bh9iD)E6Z$2 z_NttDhK;$RDDuIEI7po-5&3rHlLT;7#~bLxZJG^S&j8I4R0oDYt?WL#76&|Z1h3v) z!r9(U-#oqZ6yK;RS<8^F{8A4cZ!4ueqe+X_IItvuKm@#VLl7|N*wk`G^jFxxpp_`a z2p77%mYpP}06Tlus4w$F?=(azxdGM?1j|)9#f(5ssyxWUUhEFZajNl#gnIK|FXIKL@Kc zBFk#0TkU9slzNAy2}&nNUcgGMq&xUmcrshYldYdt?2m-%vqa`G@R;|7#CMr}(WyLY zA%=sYjKPci`hwDwCCptVQuWx6iUv)U!lxwNVE?vGkgb}9f4MqYVq?B2`e74^kIAx1 zDlyiy_6B4VEa)OsE=NV1KcZU*BZPnEqSJ%0Sx zf3??GX#4H^*zZlow^VAH9Y}fMd=5-_2D?=U+YcC25`M``Kaq+Fjihv6MV=HFjw|V! z9r+5sT01FQ5_CV!@61d8`RN}gw88xdrSZX)h)+t|v7pVxx`Xa~ySh`}*=GA8!EHql zP&ctqW{iv zuf4FhLmva_#b{{9z$~rSdyar)L6=WTX+cvElV?IY;TN~_PhQ_8$ddCRBHN$*HAxOz zs-pMbwx3jB6D;LmgLm?+&ga`@sy>sA-5M`e-RPokkL{AzR(EsW@-NRBG+OEWq0_|b z@`%Vk+i~MmlZYc(`$EMxnL^hQObX_;Oou9X?j>wkrJ1H3A!{5KAHFuI`OrLay4h|9 zws>mFUXDTlN1eGd-#Tko2DGm)-(JfnJy@Tb3nwGRXQ`QIQdA-vqpdR-r>#ABl zDlR3DcKUU-+zNZvo~@``;zkDb9-x|&RcdRG&RzM~X6bws?!RqNLt)ioE)tC2mzSn} zXgu+g8U}`igs?Vr&W;6he`yz2isPp;74@jeQrSsjLKJ7Edbl zbU0d(eB`xwkxIib4~J~zr{Xph>JOiDKB$^~+8yeIFCWTR#kvwRKXJo7eL6<|xKzx9 zKP3I+UtEC2@Xsw+cxmnvRMk$G3k2k2Z(L`x(N;;}|23o5E7r*|A3Gf7`FuOucb#== zDF5NVKi1tNB==*|dp`s;4&t zP=nif^Z8QABG7%+YFfewmtW)rr>F|kwdoP|`S}HIy~`!F1Cdu5>82ue#%8-jxc%zB z{O?A0@^~IhRKBu$LRl&7v47v2>I|dA_=il44@Uq2<9UW@n21y1^M?1Jlu2tXUioH& zY+zIDC`l^4!|vavr8SHGSfG0cQfCAX$Rb$y*ePCC%Je}E=Kqm z`L%dqphZ^j?L4W`kFbwE`Dg(I2Mh8xv_?`q^jLuE5rls8&I~9E z;Yhy$3QXAsBFE&lsNPa)k8KcGEyj&v3R<{@M%?mHn#qcv*x62%m5UW2j>=S+H?~{X zxHwR(db7FA*cwKCmpGAFApMVwXwid>#1QxJEjr^;VHajEsN{INf=}L6Qp<{U6TLx5 z6)a0!s3(#%UNS+bC{059`Q`B(-g6-2@EHr#VSj)oZEjFps4x`=qX34zLAbSemJ?P| z8f>Wzu_VE@$*KfPu;>~J8~L4(Gk5G~3%XU0XM-O-wRZ#KGC(yp%}c;>QHjrX{PTxQ zN?!T7CyxC{QkI?*-)6(6ly9l-Dt{I;CIYC~YgIjpH8Lc^-$w{)KAmSt!q&ZO_Bek< zMy$xIu9^_5*REyQn~O#U`OB|jsE}C1>-7FstLleyOU0Q<u34nyEY^}`pRu=$1k|Ps8h+2Rmgc$ zY5*a9;+3oY#l~bbF{vD%=?VXPu!&OlNgsKC1Scab=`}x4bcZ_v4NNsaK&)^MOA26A zETmWfz@Xe`7JMvari6dlft(NKJ-X(lXo)Wu)ejfdj=#Zwm;y0vs|PB-9F8_$pqjaX zLLQ9z=LlNqM;~@~uOuUz1Q~z|J$#oQuLd6bn6{pA1^{tJp&k?VAW$D2VMg?2rF!Lx zE@AXd6mNSi-gorDlxt?gJgzTQNbg=_cI@DVwNHjdU0oa*wg%vT*veYmK#$o8@f_y) zTKYMS2+l#wr2z#%6MYJ3(4qi?%)Y6LT@dBcmTKvS@z@PzhVJp*5#3fIvsQ%speTJ- zz086AE8#|}b|r2TQ>Z!a3iE2dZrLrMp|ltqmSDmbT3p0~wNE1slf}zDEM2M0o@=Hf z?+a{3mGCQw*a0OefKq5*Ce&kQj6HDQg40nBFSlGCo#6`>nrPAf7pq5!-L!oGznQ3- z10#BiepB$8DOKQ^h>PEtH_bj(mmdqDb=;0Te)-X=J5AgFg{~ zxms%BYsivFAOIi(P;H2X8#&HYAB7Qa#0_knup5TEoFqI$kZy`yB_y`{C}?ij0mXH- zrz#!R9b-;8b`bwK$(XzGj1-yK#(|IH$K|V}DDmHzl+6RW-b|uhBdk59i%JNr!v4wY zwh%Ilk^J~m0cw}buo}iBV_aQ2C8sD%IUi#xM{x{vialWxz_E)5C91YC$~+aTXgWht ztsM26K>b`AwDyCsI=3JSgx@fKX~$!`0e72*rRhHAq=?k5v>~$^$HE&>+d@Qvf@M0d~&G$BOsU_0EwzGWY_N8g+o?i8Zeo z@B=Pe4M)dVH9e=F4SGxshlBGQHq^m%kMjzP2H@C%z5`1Cqr$%zNJ9-2zmiM>Sl$=S9>$QXF&skGqb#FIl=136GiS^TnM z`^-hmF#sZt&hU~9-Q#DKQnu0>%k@d#$w*t@69j#-YBDB%HQju%rA{mAV9w!(tp}MK zXe0LETtiDrqq2zL`9z$Kr>im9WdODHINxdis;35~KP#mOlcAxZtsa57>gw%@T}+)G zGx5B?w046aCK(n3+62V{z?#-6y*c&zuFuH5d#!*!KHL+=2_(=|LV$!2Y824%N>(b; zoNgNo6b#+U(qqW$u`Mnd7e;fk5|yS*ZXWN*2cB^|VocB>zveU0g3Yrm1ecK(@RlBo z-@|R9I6mSDlbc4UD4XK0V8)oq`(W}h1HZT^XKNk-nUBvdGx zcsNdLJ^|`iT0uhGXKRP;d!5VP$YV|Jm6-y|` zJnX;`U!jqw#i;~Tpyi_&{?q|(`?Q-1IO>ngw6;O@gcxD+ni?E4pcIsOOKI#rNPp3pdqhxcdVmg2J=&yVei zfNUukVbd-!?UCl241}Egc(gK=$ZURBM6?mAn!si2CVI@a zV(+hy5rR4(%sP5u8a&gxRKE0HHaON=(?Qz{ttzbPs`~Mgr*ASe!Fo5aw_P(tH0pFs za9x>Jfso8CK<(8O=t_a2{E^{Pg-GS7g}ZXBfM^0U!h=i{m|IR8EJhqESw?mh#bHbl z21H~J0Tp{7?9BB!o1XdQk8o;3*HX6wE6iKB9QyY~xhktbaPmJH;@*HPb&hi%nEjN) zs9j>*uH^^(m_N8AXaPwWTs7ao0iqp^DzmKfF{J>P4AZk1htItKzPM4$^9mVV1weav zY&*_(uz~W6?@>CaK-p-u42Vb30)Iilr%YyL4f!E4q_=~pVt8o|jAjgSI$oZPItzpV z`QX>^0QMGuPk-wU30M9-&x2uYlrxP42v!Aub9X%N03fnArdoX}tgQUrXtlPzHsTTl zt^HaN`U>oLd?gz>ZBq`otDXjms2?|%DPJy+s4Ygkwy=x?HfoHr(R)b0#WOb+s&V5uxPg-k4Atpa6tVgx1;|Uq$K%|JW-oN7?bnjy-yH0nYyR_y}i98#eD@13xj4ftvS*)vZqEea8{kAY!Jvxo&ra8kv73s~{ zp{+@G96p$W?X2PqFOR_!9%`T5$K~-?qNan&p)+iIgLQEe*E2SwoMmLejoqcbBC{32 zp!EU`F#RekyCr^p%8YpnPu|Bf`UJ!{K~OifidbcUbOS6WJ9Kd#nP)puS;NqZm7*wB z-+U8o@xF%de+%GbH)|c2!p4tw0M$avNCgbwwg>7$72FPB;wSQF0l0ZIi6hEf&wMn6 zfqgKlkL(Uvz?P5)^n!1^gt(?4AH7qB%O7yubtC81vq@Kpzy`g!vYF{MPL}UL2mrj$ z@GcuGN?!>;9MjA9s9kDXJGEruB`fa;kSn6cFackd(PDmQ>Dm>934M6~9vPAhl5|=? zP6wS5C+u5!^EN?FRpo@;w2Jr;0z+>79OGlP)0GAB z98Pi$rGNqwUt;MnEz#wXr>n-^w)xUw>WgsS5&~~CH%@Mueebs=r8~7R>bO;sD zJ~jM@itZT9KnX^KZMerjx$#T8;*T8rS8P(?ZKLv1FLiuNA*=XJ;n+6kj%aL)`DrWa z9KayJpv+gGZIdg}`(RZQ?#n>!!UfwwCjrsF5T=U1;@2H z9%>HxelMcneu^ZkX=Lv-De4k&Qi_0-w)bYH-5n@6x9F+ACY0EE%t(JGEy7dufwU&k zpb-?`0ntc40qfrBPt*aE2a}-2AHF=^{<93`kwG(tX;*I#9q-KLjR~QGNg&h+j`4pi z+@oTV5XsU%p#J=PY|QkmElqW%DuEwd9A+$*a*F5_d==^P`V8XeZUi%X3aYh-#XYDe z*>@v<#y+C=NENal-69*W>J8s2Cg@e+E0ZACWgbrMYf?D$P&Y4*41B+ z7AtMIv6S`wl0%u$6v5x?jC#czJ#_uOz!{aeB)y{_M9l67@|9e}u5VO4?##gd5y6$B z4@R8)JckqPdFw`r;N4WT8y9KWX(AeWe6(n|2~79yyr4rPxE%}3PU)T<_G47c4d%`T zp{ktV)E42qTdzP3`86i zK5HfH*B1Lr)K%B<0*pof29h9d5c4ltYC8;Sf)mZ{w5$gHuiY{!>7tQ+WlDT8Vl%*2 z7oZwvACnPb7e=qU9ys=woBkTJASh-4ABltPi3a3)>vYR4@PRRl;@ zREI`sAc<_iBKOw?u<|McH=$a)o=p*;K~m{j=Yw@Md^sBh0w^Y0H-ZcWcaxuZ? zkJ@~wFu2`TS)OI9f8&eOe)J;r(fCnofSq5OdWAyf4tqL1*c`G4&gyHc`vHBaOQ+M`Aj0vy zy5BTZyU0Mg)Z{12YBj@;yU$VJUg74_62pBeG>stWbZ;A&0?KW7@Vs7sX7UY2mDNwH z5Yor!hM`me>I;hTOx3g6eYhe!2d!VT)ysk`&k@tCnW_QLDERSj8+F|9v zG*R39$h?{)_?CdsPgy)&&hG0`#1x8aqQ&w2-7U`5JVrgS%WcBS{OR{G8~WL)Rj+>q z0T!TIxe3ha9Dy<0cGvLDc!?#}gKoWmr&}aX=SD)G{)sew>F&HPoye6+qdoQf_Wy>c zqPIObQH2}*6z1xUJ^6WH&gKFil4Vq0=Dm%;AQZmMm0L$`5YU%X&}d$ zRZXZ46o~%tbdu!z{IWMn6CXeqgr5^}CF4El5TN?)yT55EM)Otlu$?(PZnu6?rBuO5GU5`?{cRXrwKim>*kg`IDjfeJI{QVV3)#Zm zXX4*mu}cn)I91T*K?RQ`sv^7Ad+Vo&Lb;_*Mr3I`x1$WE2~P7@PyaU?Th_UhfUqK) zE*EKl52rvM{=0x`eo@vK{}CTU#%T5XmRk8Un7O-eFfyLQmB&e~Jb6#Nx>SF2k8MD{@Y98#pDL9f3oK1=Ei-$LIXSk+g%`Mk z?)%rlz~fohinV>Wq(=LDCqv#Zi8;L1mfn?-M{vtOA?ndFD0}iSQZfNz{e`xm#=^Ew zoG8DO0Ynl^*uJSXUjaE+4%{8S6IFKXP=If0%%UP>WOs6jejOy%Fz`P8`|fU2UbSK! z3BZ2w)~Rt}baOr>Xt;GG_2B`|(?Dp6QTx-D@Q18ljXTfn)asS&HQUz!P9~&fr~lD} zHA=!azA|+D6(E}&dx+!lU8dVt4_NCpkR5J}``6?xvPejE772Mv1BwraVsUTu|-8 z8P=a8kIDjgo-#lTTO% z`9nDK%p%dv2!_`Rc`tYL#q54Jg%D~Ve$@<|N~jnBb;53444l8r z6!B7%i58ftJk5DG4nCYoB`I6#MnQ1WEgdjQvVMZ2L<4z}4;RSSufRo>D@PyI<;FAh zrHHod7&Aw(*nd~!g*Q^1GPd2LO2bUHTT=aYW;*3udklqT=}*757%9xztNrAiEH$h? ztd{S@?NK!|@h0)O1ws8e8(D6v;j}fX{9O$w(c$;V4*(VRiyT3*@qRdkwRZ(519u+^~|9g@lhZE)bRVL5_tpLSF$-^ zX*hyfOgg&003R~gl~Z@m&+fLc_s+dMJWqdvXeP@=AACr9eA%ssg9*VShxequEL33?G)+72-+L{u7xX(xeO^$w%;R_HI-I;04tN5Zvr*1CSeB- z-8(D&GS#wt~SPxf58vyo5W5DgXYxanj&6V+l zBFya4VSi(s1VaW#yyq6bT8$1(Z?Y)XE3%EN@8QqS8CY)DRc>k_J_l%ofs#<(v!xy_ zT9C`LMIur`;3y+pMCWF8GY#P#RfGRNxbLO>jVKpU>lH5f!VK1H$Mmvw4(mVM-lz@XV3-Y@YDWM^RXiP{yC4`&XCt;meXystW*mtAwV))?;Y=@mxWe#; zzH_nDruuocbpiB`LXf23n5zxel*)GHP{>@P!@GsYZ^z?CgP^ZsWgsz1mbBM&45+-) zlMlGAbTQq6Q1?>~fYn)lkU`m_4>&*?E@~?|J%xV zt&l8WtmA2PS~unAr;W0LF@zt5Ejzgh+qe^%)*~0`bOPScCejp*l{}Q6mOmYjhw==` znw<*hORhfIuTPc|s2TC>0j&R-{U_rI|oJPmlvu}__;zl15thTw;PF6$ao;isDkCeg)|gQtlIB`0`fobeJ7A~mT(r~l z6j_Fvg$889Ynqa3{e=0?%L(Yb0$Au~<#b@O>q5T4cx=%>s>$Z2T+(Ye#@MuSN#|Ld zAdABDfjV}(nqj1|X>A!E{EO@hsc94tCrmi|h{o0D3?2H9hyT_{9EgqlV-=^g5w%?^lsXJ=c#{v$QTQ(v#;%4aWPx377+_I0@sJqb&6-o zQWN}eo=op=FLKV+-wlo=EoY%>?Q#AbsF2|mAd|kXua3Kzq`0>xbZ>98SEJoYvF4~x z#3Q09DPfYwb<^Z|odYq3+dU^x23m*FyLWdCvdZFrla3K>&^{K4eHX!KP|-Q6HM#*Q z(LW(5wb7|VYk(eTi3$cK`7$atQX~o&8Pq;BEZ+*mpS~xXVfD?ZHz~()2cBA#eAJ;{ zz%j;SJR2Y%SLyOF=rc=a@U`Z4>F_zW(hwKGsF(ZfjKe_byc>&N7pJx1GI^+SIBK`IAy03@tB8_Vo zOmg{FhFoqA0Cd5ii4=JKndQi|00MqC9@C%8!gqe>-MREG?0Xa~cLo1z3QB!7ZJ5{v4vO^^S zYcEo76?@nsF(fwIp%vjn`PVU@d%@J;vV*SrJ|%l)C^Ajzu$MH}(6y)U1;(|R)J)(S zEQtY!c1ftnTj7fci$}Z)c?6n!{&eekC;_Jtj?ugd-s?BY8FrF@-YMGB9m8%NZv3l_ z0f;axKlm@$pnX(IPD^C^&v1Ms*yrv_u(`Fh#j{k9+bW{ zyuVEXYbROH*UO^X0u5#P2{>Ns0$9poxLk=c=w33-6HCKo=6!1A7BD{Rk$V8B#EAt> z1zS;ms6IgtcvY1rd;YZE#2zup$eIv1Q`03!O9`y2$Bs@dcH{sm`0+c2bMcF1S8>?@ z!%KN|J>epNkA{m~9%FzRt>Rag7XzRdL5%}2%t3OaDH~e3iM#{Y;w>OT>EGGIJ^r$b z6+uXw84~}VGSPYVJbq6uk(QI~8EZIGDxU<>0v6q)y!PbP*!jY$S37sc?%6-moYh~0 zcI{qRIf`L)J;{#*PQ50veyTHQJeg)1()?KM>ImQf;mS4!OVXIEsLp+^s(w97`tG!P;5G_*kdgyED>Mt0?51CvU_+=paxkL z+PBKRNczW9v_t25Vi-WEX!1*Gs~jKE+i~y$#^nB@HQ?dIzVP|vw{}%(qoIM1%UWLL zX3Ej#ghyba8Z(ZiH1I=O43_=?)3ZZQ^*kh_5_Xw^^8;P{HqhC%QBME96%ZT%%{0J+ zGpdnp6-9B@AW?=JT)_gvfgxLR=r-x}c-VjTtUF4xtjJVYCOw#xE0Ri9=@H5?RX5HI zl{snN?S)dzcFR`ezFhYGYDWYgcRInO6y5q#7`dQbqdu=CMlI|u9{5-tkc|T*M6h0M zS4$|QJ57ICCt3~MMZ(u~T22%O6~$x=5|g;(=2Y*?K+s+J z2b4+`e%&ug11Qq+SQ#^zN9(&CWBBp80+Bie#AC%-DvV%kY~FkG@&iIQIEPU;ud-lY z0cjt=M8_h1)^vlOSOojd_rij@;Yicz{@#`eO}Rej)QS>WQSLPa@u@ zUdjp0T{UiVN)v?yMP5~abooX3l`{ZX zQ!$}j8n6N0qL-%j0=S)~uQ9?XtYvE(P{6PI%f8J9aBvdXGrVYX0&+3R*^7Ti{?E>4 z^M93}S0H+qC5#_?&N@|O^{&Mj%2TSxq9Xk#m^OPX@}kfLDvW_jh3J~V$4Ri&p|Qs&lI$^u9pQAII%*Q~Re2vP9D>DX40HabHE zm7(#_hmcQL-fI4nS$z=aAM+nI5_vN(k?ZLX}dqbR2jHB3QP)7Vt z>`1}sAd8G#kfxgnNr5UPOD(WSHcj4r*cy#yThO6g%u>hf$$HC1t&aGn^+ zRbj8V$1%^5Nz!Nd#bDu7A;clzZyP-O{6Eb59}gi0{Y ztLi@=V$`Nh1pJ0}3A2NLgC{iaVljW?!^0?`?`w%ajCOVrWPi3)&l$=vk&mX>YIt?^ z$Z9L$uKA-cMv)4uwJIV)e+sI6GWKF1;j+#F(C{nIkn@#k4cXwTtpTdxRjnUQ;g`}Y z&L6@hNM12p_vQ;aY~N-IP0BCSFEfT`;{-{z^^*e{TjkH41&6`t&zl#uw3_2c6fpIq zU?SaX={`zUX;VvxW%am{#Oi|UKkr#k4sE{*954Dv1Yaj{R4kACkjeO6iWwDH(%n}F z(}SG&8!e^@=^M$y6o%jbAd?*{ev9D-Uo9~EjSAZN-=31DDU_#>!zW|>}1Fb-OxZ_xk^O$#2h{%!l-W!$tr`$ zkA6G+D$*0L=tfwDP44y^0j>%g-A;G<&+SJ09GBOy1WG5^go%l{0h~YuH)cH z;fE%fk6K@Ukv_d2b@@sF4O&=^A9W6bE{29`3a4coN6U?Ms{(4-UBN^}P*Rsc$H#{U zsCxst%Dy=Vzs2?M8nYBX;KhH58_)XiatafCdtP^o99#q@@4;z{XjTNj z)E$j;-GqN_?Yw^I*4%`s`~~qxtPwPtrRtpKGJ>E(I^_UGAVmF^Pa+69sXXrJ~aO-pvssCg7=grCYo`!2XGq>}x6l^9mAP%gAn_0QeydngBLPLI^)g!~)Ti zTje9(;@1BGgn$?HHd@Jx0na6N9Z`h&qbObC=&uRD{l~=efeS4Do<+fx&&YTRH<3iGk*#s`~F9#PzI}!^yroy6qbpX40GmTu(={3D5 z=S8$x%A0Obywv3xh$sZiM)(hFec-Nb$lZD#s=#D6|DWoAB2g8?x78 zwI7pmf^-{LV~(!E6@?-nYteZ`6!@9h{)v-?$(ETQXkTC#7p(@3RDf&WZ@62Xi z(eME)p~{inEZ)N28aar6B=`N}o(~w0Na)9rvg^~dJdLCfIRJ;m;UUOr! zveU3-9u6b`{Ino&kP6tD*nL6IIo13tM{YXVpdcxG4LTX5&SgJ`M%AC4A=zs96%Po; zgLWZS^u|#X31?m0;fSzBdFc^XYdQXCK5GY4N35`jfs zKAoAbps#jWBBBzl*?OeRV=H7N8yt+p=Aot0<(l-EN9|+PJmQry*hj0NPi>c~fKsFJ zO&^?>dd7}3IKcP*rjw#bhfe`_6{ssb=3G%1M6pvJ{F|Mco^4#@qMum-xLSm4P{2L~ zKR&~V7Ul~6juP$E>-%Vjnjgd`@*f64pWLm+?a9ba1JuX;6tD?fWP|7^YmD&Wzm571 z#-U0XvFmx=Xd<`OkOLob8*ISxwCg#C^l%emu!$)duWfBR>PEXTh?X<&+X)2RILnpz zVylHCO__M}x&NX%asZ7@*-aIM!AJKj-=U6(T=PRrLGVRNWHJX84>cEksKD7?mKZj8 z0t|v|jWBXjnnJfk81 zi!2lbJq)Tc1%%%rp09{B)Q*dz8xYKZPuku^G`n#N|D_@S?ed;nAQ)EWv)2DS2nrR? zklzAcaX*EY6SdX1>JrHd>poHJBKh z;{IKi-!KK)ZNsc>{D?z;xl;Br%U>Tq_WJwm@O*TslK{4bgKk@9#j^T~V+P6>iwMJ2 zvmA6s!9M14cmG=H2+j32^7BL+l>U=iE)Q!r*r`;qhFIt^J#+f}*T)BZ4q&)0@lWd~ zX~1#3^{!HvA2^=}ZzvNKVX|1+d@CZr`AqSq-~4N()_`OPw(()S!lNb*YhqZUQAc?0FbRd&7jutR(J|f<7}~2cgzPtDx1Wu7t_k%a5Kc`k0{|LXa$pA z;3&S_+o3ML`mJT66od71HbWYOv)JTyaJng=%9v5+ zWSt+5?+n5QQZ=mwQD9n|WwX(X^u;^B9FiK~=%6y|vkWsz=#vmTVp$$*5`6O(W01Ks zq;xFz%+-2K*!wTr7Mm}j90wl4pM`u{28b!}3WOPbDn4wOXxcG8?SHU1*P+& zyAx_fFWpV#qF!h`B!ct(kT=%DpZStQEI-bj(0roLTzo#m_xUG`1V%3_kbn9<{tm4K z0`nLoq_>B4ne24o7nzde!&f2|gkKeKs*hLpXlza$X>*sX1Iq(>$kpicXwdrpI{L1M z+bZT?KP+LqI{F#(8yCE|Mng)tgjd*Y{Tj=l;vwbeAgUAUgchHcakv zy3VSesbU$|=ceVf&}&K2SWTH4b9rZgKZ#KqCfk=iP}z=h93f!>r*S^Fp|G@8^{&DY z4geLf!OS;vA^5qKF1-YRQG~b)kI^%(7f1NSU%#C3;(5B=Bk*!O z^8HfPpkwK3$co^CA@U~l6|YeT*q)cK4Y zK0oQ}y9fs?^eZDPzXxD9mByeqAQ}zfzV=hx%gXb2gt ztw!g?@hs|7{B~Cx~Q8&I_n zzbG7?NwMfhNK^w|uD4;75I?5j1n8P#B-_o);|)AUKIH^n?3+RL<|d>`aL{^={Z{Lt z#Pzix_1M&FY#)o5<`IZ`{uu;!HD z4YC>bYk$wIVm}uUPF#vQ+%2XrG@@@#2-vn);!Kp-&OK=jrdIX5^vW?}(aX)|RfJZp zzfb=em3uT9k)g4tBKL_4M|ACMjD4%wvv&VmjA7+0AbnW9Ra@E37|4I2G0CI4b_8fx zwI7508c-!xa<}@?Sj-Soz7}gk7o(8dP?W%fSIZf#OgHHfBTN{K% zJ7)|>pKg^FoyyZFkKFUV5BEC%!|Y^7D6IedusKr19w5ic7y6Aej_Wydy|F&1GF7tv zTef%7X6{SupYbka_giuq^9IT!i`D4A2NSiE#OxZdZ&j%~)PK^Ltg@QuREvz6_WzyA z3}~tN9`g;FQ?3gQ70)y90I(r6$s_$gt+<&tc5XKSwO~#l)MM>p0al4)cZ0qB=JnlS z6S?fwFL$UZb{m16_!16V+Gz@h(P_&w26XF-kZe700V7$Pn)^i!jW0t^RA9_>hUm~3 zLrze~f;eDNUrhoaWqrJsoW1JOM@g3_XZoUgR@TYx|6%W~qpI${cTouyl$GCrC_-o{xt1)Wlm&5qqfD+k=QZ>1VmQ zmFW&Cwx{bCE<4&ufg6K@f7H^t8T#dvF)W5k%Q% zSWo%ojDFNh2jb92+lhuNE+*nKOEm!mk*J(lw@}7eE8D`t7j;3_fMQ%0*NbTHBmlD_ z;Mp8jkcE({I4)-2IdcTu62&}~e#vO5u8#5w+F{Hfj0W{Veu`TMjMk;iEtD4AQv7y& z!^RGBVrYv5T=pcQvZU`GC^7gwP%GK5wz=+cIWp9PG$W$O4r8|2@}CDog@h3^K_ZAd z!C7wD<)JO&cQ4>9?{<~T5ChMRR{4;f9gZ)Ux6(u_`P^dfc}^t(m%b()`B<5DXOq`9 zg~j>DKq98X`85zqTQ3$5z`E*ygBQu!`1L;U9zE{N+*4B58YZ0eLTpao7l746B$XKb zL4EW`8Rb5W4k%ZD3?{R16F%OKimuhsuC|hSe`xabylBu=VVSFU*u{DW?tvlm<)nZ^ z`{RVy@7BygW8%-XEIaHZ5nhB`ml>pSx#2p;C8F<(j75oL0Ly0a{Rz#g4F;@c-3*Pg zGB(KK&jJaFte`4E+1t2gL1K27Yyoz8_D`d6fQO6nu)B9c^wyt=IAbLvfmR0KtK7D_ zE;1VLi!MsFmIIv5N|{PQvunbo$w`EGEOnG**biEn1wazhLb>iYvoct5xZyxdqcDt| zm$&C*6wn#bYyohq-3g;dR5~E8GyYyn{e!mmRUT$n1~bB0s07_1h;c`%sWV zc(`87mU*lCLB@T%u5%!oJVJHyDjQkP?6bP?6!B$1jbgB1xcp>S!V9PLeH3+Spj4L+ zkng~xsyM$&ck7;PT`y`ipeiTt*o|QPAhANP(2TSy z2qXnxShg3dY@1wpeh-U23dY%Th1DrkPMxdJkyWLK>0`K@{d;d6kv6`o?;~lvwy$5k z!szf4fCHd8++fd67T(OSdA*o4ra0*k=*IIOv|&0qp0c+dT8pf$kO}!kVLIZFK`&4V zDsDr<=AL$X?Og7{k)lGhZv?}3<&>oR;&x6gh$xJJ#e_dDSXuel6?HoZF$ha<%HKk@ z$-eqLl|)3)xJRWciuS&%!)91qh)U+b#k*I6<@~UW<|DkTj8eCld^W?L+Qg^kZzu7^ zWqvS#6rW{M=FtT`lyv3=v~3mx1aGJUvqiK%fS5KzyUDezOb{9LSv z0q{sH_{w|a*He2`whhZ;rUJ&M+vKRxcbtDMY()RAez3Z1iGg%WOCs^09zZ=@oa|6p zPJa?~6KZl>VRrp^LBhwiV}Je%6!k%q+apk*vQ4XB-%jwx7JK=HE35f)^ifFE)s5JD zmb4}U%Hu#CYN&P<{r$hN0BEZOf-c<3Olm<&Qp!aN6_0zRC;?hqZ!P)`JcK6<- zB3)kWw7}X?VSt+FzV~2kc4fl`PI6+jhh)hkt^e2KeaTfoeu3z$niL^NBX5e;>LCSz-RQ|CJF`P+RMEq@7U&uCR8$ z9^iBDPhnEOjKu&cX2fukgwU2u0Qg+4M4iLH3ImSZe@%Bk5?H7 z6IMTY5i(XUB^J^k}g*(po6#{sOb!?{;|Q}lK}8=q1QXLSF+&uG9G@ac!Ac+y;~v6~D*dV6dFdC9CSBCI{}J7HB!hqc#usYaBP;9GH&A@V4E`5K{ z(TaBIPLAi%LQqs&_QKyAWnJBA2i44ZZ8~RltVC!Zzb^uI;koO$q|?R$?dm6n+`s&t z%kxYc_Z+7Kt{F<9^m~K5a(ORjD;bEA(JT%x@{Mmlc1NM(J?dhHN$;nm88f~h-t7)b zeo+2#Yt!b^J%zj^6u8Tl&@22|@2&tx)|v;9gZI2*`mkm^m~&9h_D8}XEU#&Fz}x>0 zDH))o=vbg}=l+0m$Tbk<_5Sg2j?1#91Mld})B00~AgsI8ZOl9?DKU!~8I=SJ=@2d$|o|P9MA%0Ce~GsJe34y zJT68Ufm2=Yv>r}8W<_cLQ$;-PahyJZ;hI9ftQ!Tlf~foG1sg7F%u~4+_FHEKg$GIr zWs%$_SdGV{W{K>sSm7l6ZvlWw|EQAVlkHh>aA|>N(=&AkS^jKS-$6-eS@n?DDju%U5MDg`+zU%PR{Du%ue?ESczUk*zq;4>#texePxmNi@^ zU-dw}c@F-0#oH(8{($;Htmhs(#!n8ArSvy}U~^*ptTRP0;M7G1ZHnd;fUhy6K|u zdP}1=ih>#e#;+M>##Jx}H7`(ZVBmEhny=Fb(3N0j3qj>ho8e#(Eu#;eS$Hv>d}Sb3+9aoC2&x$;!$rZ5?2h`ljbWRej~1ovv_&D?A-N-0 zIajUJt!8A<)Yg=<<`s^9r8#1@<5NO7MhIyY3sKHhmo}Ot>6mybx`vmZVu8Z^f#*yc zivEfd9~HO4Q^D)?i6)aVv~&vScbxnqb?G>HorTFTI<-cO?Iwe4jvXq{t64hNZ}a=b57n$rNO(z?kbep z^28BL%bIAT@V+?IC2yGSs7)mui00F*}gTjD}{7=r%UNT|>oaw{Kx z&|t!D8ef=`>HWArDf^ruiVr}ed^Lq{rfbmviGH{tR6{|;CD|Z^xcG&Dn7t3^B(~Iw+ zycMJ;gpY~BnBlb9v!o(B8>oJwXZ8VlyjRu69Gcut&x0sIz}NP!3jbA37bBf!W;qpW z*n+s)l)tLE}kPuAG%C->*~h?M_>D zhs%bAN!cv+6c<@iUSGK%_o!($*`iL^8D{0TDa0Y(*?kX8I_+C4AnWVI3+>v1vaJu~ zI(tGoVha@Bd-ALj6)Y6wQw?%F^4PTi!IT_ z)6cyQ%#%$o#18|kL!xD$n+7uEJdVEnaJHeBTi$>1BN&7%4+=l7IUqU@9@bOrg`k}Q z-9cd08xX&IdV2qmcoP4Jc%-vz!3)~b-{N@5>@Xd~TX|^;;8VIUWT>zmsBijO(|m{J z>D8VsY2G_J$HVqY%x@>7IluJEG@z1S`L;l7JXS0)TBcrx7DMxTi6H^{t`JwQR9>wU zZ(Ui2c?Msa@j`OSN6LE)R8V~QEB*s~UL`)yk zc^PWeeBKciFD!_SnP+e;ABZnRHoB!%lxaU(Jo*Z{_e8UWyvN?Q%@P_jy_c<-8Y# zbdtUzqwGTIfPX8w_7NCcCKRn9xp<-Mzh1PyhLGA6$@O-a_U;rLvW;l?s}PTh+{|B* zl5Jo)ym^`CqpjtZWq1*P%6=I06hf@*_3+|&({6Fn2G^lftH~cgJE%mg@3nieHe)$c zE7r=^Hk)c4lq0w8N|rFh_bL2u7PFKoW57Zr;KE}pI$Y`wX$nP#9TSFqTH3gV`Yw3}IQmny8?JD<(^ zTL7pG6aM;I!wGLV*}k^=qa=cAFS+7j+D!Oa#0C%c z=6p8EWHmtV4N{6S;O1HfExz~Q_$VE6&B=ljuO{ zX!Wy2-vzR#r7ZbsMk+L4@AH3;W+u%HuNVxxGvVo;9L$hF$wOxrS6aeJV*_UKKAeuz zSgAPaNjFwuRZNhmp8xW&fB&Aqh^bht~} z-mx;R&V)zDuPk`*CDNuFrP&-PqV$T3(#|?jsvYWM&#IQQG&;9+!?C0C`kk8MA`J}H z+bZobeiz6MICCw2@U31H)WHS6C;)+8j|qFmri9j=_c0+ z$fvUCoxY|b6L1NA`}v$DML8_Du|)j+lY#}NcMXo&;gA(->1(XYH~m<;S}O}yv!8_F{9c$7qqv;kXnsefC+41|+C;UH ztyf|vPi>XZak^B=)}jqZX5}F)M7_-<_8xHbQW@Z?7oBmFjM7rns4@GH`PHy)r0ii{ ze*Y3QcVoY3F0W5CBEt1&aumX=ldzk1Y|L^ zfbA`b8<{8)t5`YYy)3hR)E;v6+g2gqfFUuq9&PpUp zIMIbU2DhWhqg+cupSPcJBWaX8r#?fDmVSPP?RyVa+OdSko=Z=IY$u4LnHlJNX0CD~F_ju_ld zSJc#HAM^C2{BdlKL(OsoqXjaglD@o1 z?d{NmfQoq#OLX#`iD%Tiho%A|f#M~DA^p!`K z)dzp!@rzCJCf!Orr=(>GVa&iFvow~BF(BOnuXVZv*J)Y-y)SLaR}wf(r#?*qe?D}} zPEA_)>?K4SY)X;JC-H$i_W;fn?pcz1uQVUa``Xf(&mCucHnulvjVXdWzdFVmEKC2)AxaG&j7E$@bNokm8_4_~`#569 zDeY?_E3Yn^g2m%l>z2Yg6tlyZ!Yg2Cf%L~ZV;ucs` zArMNOCkKEU1}QM#ZogH?xgojP;_AzRU#3h?D60m*d;}(X zYmM>0Ccc0Bz#M6tR)@&eXwdXqB*Q9T1QZieP#+d<4G&o^9l5Ys9=MA`mmnn)VM57^ zczo${htM`#^n+wwmj2@*efo7sa3C%~DCl{6la4hKSTjyauYXuU>4zE@-MlZ0J8#lj{HYpbGv>uOn2rE;HOJW` z1yP(l7;*E{GPJ^s)iu$Qxqj;_9SJ8H6(V-r+4jOR; zCO9tEx2a(OvGQtoo@5vH8g{Q!1BUfRjS2j{Q^Rz8Fy#1nj(`r?Af1_)ku zkIq$TvEajC(i2|5&9&qp@&yeUj*t00NbZ=Gf|^t5o{kkbhfOM-PL6&-tHbK-b1}2b~mg($Ic+R#Q+)22udy9S1NB%l#lTNP4 zyZ`)4_O(Mf^GF+tF2g7{ zL$gr-^zeqHF7WM=9hK!Jck|m$D?rXZE3&Cw`)I8y=S2bUyUPw1-d`;MF?6~!e*V!2 zY&4iXGlm0l=voV(X90|6T&y+REX`wf)l33*K!+}pQq3<~!+GqF$DvOyToArDDWibl z2FJMUo@4!>wlm*fH}$H-g)qB2`uomhmdG^}V7I)&^5&fNJD~s`<0Op_dwQ}>SOcO5 zOrR!K!kXh@MpZUF@8cpP;%1^qx95bn^WH+Cp~@3hDdBkCnU<3~Bk$vaCc?`d{zJ@Q zZ-^@J|8|%?AA$sovh26MIJ3&RzyAEI_NvisgN&1TGALev__QR#zDEcEvi2fu4*6I% zKN(Mg7P$s5be5lY+e7gCi#j*fe&l_!UgL*V&QaFbCP!5O-tEI15f~^*6t+7Fb>@Y9 zFzGS+j;y@T2=CZK(Xb^K^u2rmqn;P{p?prmG<0}30C)9J30w4=0eD!|Zj(>6wX3u%K~)^HLp@IxP7 z;m`!fLyF#%$ThfXO)Ur9ic$?KEivFA*@YPle)ReU4kZV082x6#6;w5t<@#tKac03S z&0)}aRZtLKj)e>X%Z#w_jI8~4)pFgtQa^2(8W5_ay6bLpbQ!Rf^1 z3)jQ_m_q`@;8-f)o+a<5c!!6)0e$;0$~tPPcoYG*V+dd%{9+GTfgW`95)*JZa>o~| z`h_GgQ#yrD=WFLYRH(uVv9l_7&7p#J6x^nQ%k2HS~VWybx5CM285H?KS|(qyME#y zagCN0+`@9C(SsmHiGxWi{&|v{*7}u=k`k;5owHBcc#G_#4{Nky+sF2vG{aHSX z-4UZE(#$@tx{=yF8pH16-u0VwQJEJbk|QJi z(C;e&)UV?y9evsSbY_O6Pu*^`+06%qJ>n_Py1OEo5gd1B-uWS8>t1$!qf=`9IAwEM zt|3Ole&x1QiRBfHPw!JEa{v8)?_sOxtHIkNC#8ZC14XF^OITjpj^P^r0W(onI_-kv zQoxL4Z(I*+Kf9iKPkNGfEftYA0W%j`*%RSnhz5Fq>NkRnbfI(~kmy+hR=m!1$h1V_ zQwmI>=FJU_5%43s5Je=W@110qhaar0D~h7hkU5%|hF0x;&LbVJxN<|2 zPY(ip=`Wl4&y*n2Iks__I5-bc-;cKS%}nh{mst&#Kc{LB#wGT$)C|cjA*LTR`L>BW ze{TMyMg6Hh{r(EmovLOiF{QV^6(=M-Gs!;j!B_`qjM&|s7cn0%|2*Vq3~p4=hf1N- z>6=M=wHIFxGaxHWDdXrDNaL&lpcu}T_C_7nosG58&JHAjRRG?6j>!i!6B5)~Ug3{m zDZK=}PMv_ukR+hn@fRqIh=S?rasnF~>R_&B0D6Y&mC>((X_JswbY8kg2SP^uKfjMK zb|nd@5Xi0U^acRdPd%lTy_n9fXd#55T$vDQMyLFK17yWluOg`^#RSzA zSYUrQn|(CCTHp zOXUj(R{_Vmqnm5v4PGoVX}sZf_-V)rrJy|BGTE?}f#s9^M3eUsY5hl~?$QVNbXd8m zscw$>U1@4U{Yp|r)3_>foyAtX_XdEVC%j%Gpq~U-(gEq#@cYmikD9hL*t#Y@ds{f; z!<4)9LSct;rJ<<;*fyuN@|9UY@fThjig@uM)qC-lKh+?bFN8-_gqP(#m^s*9=)kx) zHJxNC5hwRxnbFS#+e_pvJ~lTN4NH3MV@)}+{R8)bYJ_Db*P2i%N^9yC>D-?^t3ug3 zcA0=&#fAh^Q2r)iEoKk8(;u4;$TZ1nUkYtalMb=40RmChh2-$hRWM&jtW$jO9=M4u z;T`$u*0-xjk6|})KiWO>wg#9O+RvuD;tR}eh5`L-mhf*nehluFMicgNd<#Ts$vnW| zmaksT0E$2#DD5;uN<)@M%v5aF_Mrwn7@_zJE%}hh-YL%c5WyKm%EgMd#~J609MVC& zSkekD5~Oj%&Rf4woq3>u=Acas{lj_@E)}i=^>e!rM`^h#Q>II9dv}5a7ES($4X)Cn z92J{i?W{EGiAU0%trIqbfJI9`+t?R$0&>Cek>NO;OrW{l-of_~YUZ^N?R!1|b<329 zc|6aX3ye%d*O{f&<|m;eE#A$#S{Tzi6V+!=V_NIQF3eiJZ~)y8Sq6jgcGXsC8CY!= zxRQA zTn{DTLjiUuz}aTugXBAvSC!5tT^_nwW62rTpEDU`7i42PGxev?!LZVnr!MP=?)#sO z^3|$q>UT&4&)tfz?P$W&`NMlClH~G41wNV#_U-A(*^0~DSQPSVZxSv=;7UEcIw>&4 zjs$a`3RaJniV{9}Y~1KxXV$6Kz2Vsw_!Iyx%!1=+Y8kyXYS`5+f8Us&99-SVfx5Xd z+ekn+(|GN$mds-1Ur6*@Hbv3cac&BIl4VMWo+-V))MtB^i_S>{Ql~B`ai$4MGh0V4 zuc>9@;Tgx6J2%f-3i=IB3Vl2RGK6Q}GRRdHTn>hBQ}gd3wkzJ=}}d9{_XzEz}i)xWTZ>h zMA3M|4vT{o2a^h2u6}rb=@rT-LCiPN~SvV@9;sW{29Jm1}f9W$Joo{VJVe zz1I&%A~uCtnimqQAhs@@w8sX~jLPnirxps|bUvDDOp1J%UvX838KuAZ-jLlB%-VD3 zGrzN#xvIISH*?*D7DzgEHN;rJXV6ZLa;1xe*LNvm5y$*B)Qb0U!mannD;mrR?lF7p z0WgK{zkM1~JMl~hFn~S?@Gceg(TYcs1DECCdN?keB3rRc9$N_QN+O7{;~})pDz2By z^W5dhIof7WJ;&>>16HE>7oz%3N);;%>s>~vQ%=ug{Ci_xvDY&IezLv@|$o0P=s;KRgShSfu+Mothy$PWkY0UE0Rbn8J|c`0(QYMeOPnYb44}dVI2V>s`0Jq{rqgej1BE0LMCrOpO>tRW|zQLYrS6|1BPh~umj5_O^#H0A%W%af0Tolo$ zm1fGEJ;UAY9$yQ06(#OtL0XDPuCat<#g{Qw7%%>O1oox*bdm9+#mMYn-{-dbubMm! z6`yDz$8^4|_DdkB{)e5xa}eit1u+_IJ+lOR#=v-)A-yuU?pK02uEj~HAJ=;k!IZP2 z95-k%&Sya79m|Ai2d4P^8ZU5|SS^VUK&y8D=CIbxiWV(PYZTRkHviSdeVO@v7`ourZVF%o z+7{RPW=FMlk9w?k{oO_z%FdD(%Ln6k@C-tW(G!eca2-NPcuRR4G^xDWoyBKATHd8g zJ@^h{ztpRol2*{fM*FlgRU4svJyYV79LNZONLXgL!(R*pL#nX(M<9ALA}DdghMTyO z++PEf!tgTk6SlRhl+6Kgu*YYCz0 z#urMUQwxvM58K~~^ODPU7mRavId3;24px-|H)VC^+($xH;C~^GI{(n1g;&;QN1n8U z)JzhC^|OjUnggO#9Fs9mD< zn!^M#KoArS^16js7!gFzIw~eQ--WEmZG@ScVxy0x@b*RV>dE>MV_K&~B=ox-@Ax3M zgo?D3H9xNs-O-sJI$IscVG=x_qdhHe>nvn3y57R z>){WEZGJR&-U16KK42$#3R^(Ws~>VIumwm={jV1QQYQP?!!G!Y6C4by@9>|xOhQP6 zY*4g#JqsNxCXdEs8b?3Xe4)_`d4e62`YOz#8x$;4B6hBzLIy8y%%|)p;x%oL%vGC} zES`J*8n#g^RuZi~Fj=uQ4LLEX@^tvPU@3Jl_Syp*w_CZxN8}<-V1=}znCLa%wrt5d z-_|`eSb~Rb^l zYVfIO6w_y+o9qB8m;Fgpvt6Z@{?d4Q>}QFrM{&5BBCf?Vr_oN zmD=AW;BkE4IDw;JByBT{>mwd-8TUl_`_KoN+4!&fK6INGwOUS{93e`ZgRu_OehjbZG(0H z8yNFh*doOqMy(Q#&mst>wfO$IA62WI5uV*O3Xg51))KTmd|Im6@YY<0;_$yG+p%8CC-^ne9?nSkuKZx#K2xR}@p;Bu~#es8^eQ`B(;HjwQu0 zX`@BHpJ`&x{MrT8gS|$o3X9KTt_~E~0IB!2O zSRo+bjKU`o&%t0#>43AJGLJF7#7~hP46vHSWstDFN%P$W;ckt5UcNq~CFgP$_@L9( z){3b?TI-8*EwDntU?oOx924!}=I82xSUz6;uiI)2-F6*oUBwd*ztT-SbpzR?s%+H= zpx`pM3O>#=Hy-bV9cUsawrW0I57({CET`88G7+x|olUB-<1eckdQLqU@SY?E%)a)t zFr7%)zGjsd6@=W$2ZHc< zXlZT!b$l3%j!&p%53qhU5sY2%t{Dl`d!4|QrnLBXMv!Iom_ad8*h-klK)>Z(Hewi~ zb>yu+k;xD$pXkFv8Z&EH;uGXk=U=lJZlvc`UWAr@NygTqHtCdsr#(skTDOp3y)){; zmyFs~^NTh(;A4^kC8$7XT>ai>QP4yheP<^bG?At-6yx0iyY4(j4w`ykd0@JAYgpek zA~#$EG}UfQR&S-?h}&W5={El-+@FUfPD0P8AxJW5r0;yYDJ_ODHi!)#QfCC+KH>XZ ztlm>NU>jyR0%4EIKMo*~DG0!`fRkyJkuw_k%row&hgzA-95^iTVpnh44^C#z^<+oz zQ`kc;nP02#X6WJzC>d6Ytvs^KeGu7!-y-Ii8)nVo!T16L?M(ddd?glPmm)!OWs~`vMVSX7c4$^^_3Tv z?7u}aOPSs{42kjcxoyHTr#A)HTQHu9M@;kz(|`-}l~|f*jPM<-a*<)4vmJ`dW_^;v zMT=1G3VwVsOHcp@fMasQ!dM2E+SHthqFx*0ISO;0%89HuML3?&k=J#0@5DOQ)XwG? zE>zD>*c-L8IbO&wmRHhGKK3!jzSX43n816N{;h*EdF2)pb97n_<#VTtMd9X}AF;m{ zEn?SSDvoqQcWO)x|3;|;3NNW3x^b9g`2x?q`I6kL`;zOwW8#3i5xJzvvv2 zVX9Cku-fOFtO|2m6na1TK|02*UIf6SD4L|g9sg?GBy^M2C4fp8%Tz56ur z+}ib5wwqt=&1~X{>~N;l3_ElfY2ZUJm4ovS;Z?cu$A!}B!Vagc@hFu-~>ibT6-r4jH!Y~~SyS>_LobShyv zFh;#!THSFR5B`1)N!wg<_|N?$#=wNu1Dcr{`=JP;IA77b!!LQmkW9sS8Ze zdMpH>*X?)gIl;^h3^qhP`C*syL;An)db(7A1ywRXYF}n*`Ly`NX!X+4$Xv4lxz=h% z*`aWxG#D-2zO^ywV6aKSNlB%hj2^(@RBj?t_~GLUg`|F}P-;jVwUDG=E-eBqcvaTy z%2j>Vnw#{<9Dc7_sO*$Sp=X~8T{>1Gb4o*or)(nK2dcdA2}dnxYR*AIrc`o7oWNT0 zq0xqa$@+;6TgM0cck*OMPB*VF1~Z>a1-xE+mfbe1<)-vn?3y17)EpioNaE;XyuJ)>)DM>Ii%2)Lk~oZkpAeQD?SHuF+?{J`SzI zpZ{^JHCXN9CyS z1agoIlk^KNZlUx?&YGEW${PErZ9I_QTtEM8Gw76N%TER79j?Bs89JmzpaD<4#f6HTpUk39App%GRNutI!{J@iiUVk|o={M?6`8vWl7Ac@oV+~D> zlTrrr#=F48-%i3z|0l^r-0R6t0V}l#aTNErsuV%MIsg3H{%&j{WrM<1#ZPqi;rb9N z-mafs8_P_wJ=zSR`T8|lMsi}|Ys32p)y3wg?Yg@DwM!c%itT+VBUMoXDvkm9*m7Pp zk}JKYs8RH0k73ZX&?cujcl6|&=};&!9woI@;8X!{F#6a+Eue9?_lI06AAk65tGG81 z-QU*`iVvXRrzXnPsybF}F>U&l3Sg_>m6*ZkcgDhL?vCT7c^b+1k`5!;bOcNu8B(Do zRpupzb!y3!)jF+w)7avmq8n|c?n3IZzJY5IR)bBQ;d~XUo^h_?Ydje;KEgLrc5GX` zA-o`J`vQaW**;TKUj8>YAEpPjJZQ&0R9yT*dd;n ztA4p)Ucg^$rTl{mb{^z#4X|2J2>TScyaR3ES?SjU4#p-ynQTim^kw#)yhc|-P6!Bp6kJ&S8d<;PCgEzY`xAXQzcA%GWc2mK ztd<`$rUStGt~sJPY|pAK-Lq98D8OFGBjwY9xjezVk8_cWz?~2K@`jO4aI*rZ2Hqbg zN@JzkZA&2uQ54+1-yNvn`)j192J$psr*ON%w>?H_W2`RZr(G9kmyDUTDl(J{;bMLf z9NS;RA>g`7r(Lr9zB4m?v3Kp{Y|MDHp1%IXBi>ytIFLKqYz4rO8RR=-U|K;KW0avX zHfuc5Oe2VTMcp$UEuP`wcFbeDm5Av8K{Nw!G8&AaSBsj=X)S>OwM|ZK7x;Y;IJ7!4 zhAnonu!yuJdDuSiH^~4ecz$z{URViI?>%e3>`!l0$WB!Z1831I=n5fA^0tNhHy=vc2ZYDRn*t*yvo9&eh`c)k z;=VeXNBqpF*U$F?O;>!5!t{Q=P|XwQ`OiawtCt5@)9Z|;`$pi`}(}aSo4)Gcq z*?3BxQRden0N7Oo(vf-1b4Sl--?~T#QQL+>n#B=co-r&9W=MF%CFEM2cz#c-$f;W# zWt}SBh+n2e3lgX1v;1bM^an<#Xg`>6)dGo84pY*210gM-?e6g;;?dqBd5kHFrNr*J zh*(|XGSz26-d*;EQPUImX38LF6aV-9SR}} zxIp08hG7gA!U6egUfFxTl(_wK(o2lMW4^o*=Dr)eM8=Y3)7F zv;)j&$ZAzaHmASj#8MevH`(F_oIFyNsyC22%x0kp8^)n>poF$R(7p{hNLdp7H~;Xq zM8J59vJg5g!Pp098S;8i8lsZ?uN1ulv`$T%1uruP(Bd6`@=dT8;{5A%e7D=?;^8c2K)+r=dSqw0Z zf~U4XXuAfxtttZcm}-!hULm&~l!BFLmw*>P4gAC2ap(vGG&MhZ!Nw6QWBg~DBfuF{ zyy+!Eyno+}5Y2Y)n6N<1IA7ha9NT7b{ZmV$9F)m_29uSggHL^E4N6BX-Bi{!TS>N;!FztR*dnif-E(~yi>{7y8Ow~ZaXMi|w zFtMBwb{I0kvAcjtQ-ZScsh&)Y`GjJB-R1X&dyxQD5_aEhA??oBdncL@KQL%kKqig9 zzz`Xa=KzKJ_xsVpR~|A@QzPD79-u=>y@*uL0rpIX>F*5Q3+OTds4Iac9J4bwQ!qg$ zuU%E>G02`GrPotu&4H0aZ~D_nw?0s;7*l26z9pccp(HPO1V%gNb6U=Ct7x-7z1?dk zY*=QEx@SXYRoETaRo(ss{&m%Bj+wc@Nvs;Ud?EDDs+nmdC58);l zf);|u^aYH2$0Q;uq_htO`Iw82Faa1EMGk(Ja($RyLnVl}rRRd@MRb$UM%Y@+8=dQV zm#%1mkV3O#Rh70X&!l$)vaXpR^nhK4g<~aT3t8fFv>gH7q)(p%fj}0&L*3aoT%b?^ zhC*Imoy>7NZKtT;+i+JPj>>tN^lTpJ+A@(ojG60JN{by^VMo?>+u(RfB!Dc2M)n+7y9(C{Kndawg^no#67v$? zkM7&5Jf35tS_FLVB*&ZMS(~^F%1Ej8j*xJEp5Yx|H}sD0IYj`g^2;zdOLYzOA*|h4 zA9P5El5(Ok;+1(H%B`yusNH3nv+~JMeg3#Ww227VimHXEW6X?v{YPdskXxYwJX|1? z7cLae02^ZzhzYL@W(|yXJ@7sratBvEz!&icgwS|qY+fo0@+6C#A63w zm0^(I-@+Fu{1cdT>`(f#{Xp>9vCGx}yHP28wnd@kE^J=RS7Lk$Zo6^~IWfcp{=55h z^Ld&}whXM54NEG0LD4(2bs_6CrgRl={=^;k{=^+6(X5?8#P>V)iwM1XglFMhtO4)r z@Dn8|D4cW}o5Yb4rPKRS3xCj0wb1Xe0XU8|d#Nx+f!UMV8~Ow*ue1AP6aEnyq*vho z@b*8~IDyUegzawhRkP`!=_l*$w^-0<8d-lB>-}(&j3hAaHZq`U)Cp_TE(cJA>Hw@X_qtd?d4+QO6?T>PR)0dW}_( ztcVaYLb2{qZMjvypjauXirmWNh}kFiy=Tu5jf6p@%;+O#3xBjWsgb=z?KcLwWYA^q4e$av)elyJ{IJ6?vOgyOM_fedTT4qqfGVdH-Y$5lSr{9*m+s9 zwSrx-95Gnpp<4MU5&XEI<2A|GoTsXwEj1#JsV8~ZQ=^*AYm2{=a}n^YLY7`AMS?Vi z>VNgYKtWH~?wcQ6B;{hi8)4qTQJrVaCLbN5f4R|Uyjf+AzdCwYKYBS^$!VO3X#c-e z&BRIwTQJ1?wduYf(^^F5m>%w7Yq^uCdxF# zej;sJ31fYr7-96u>mqUgtLeY60P(6E#)^;rp_S#RO=*}1H2A>+TfZQzOpSEQt`rVz zv~4~=3Xgz%i@}@mN}M&{9~vENhb;ltXUp{LOcN9T2MZW?awHxl)AoB%cOCL_aN4ge zWmA%-L7{qM2n`rwlpmo*PZfY~?amDfhp-mdz8%l}Ob^qsvUeaJ6`)<$Y;h!g?Kv(| zy`)L)Vtc^ZOOBo`ABxz=c9Zy_Mke8z*QmSw#<)O_4dwpfS!pd+bnz22^h^H-6J}Jh z^?J&jB2&~D7QlXFhDF}{?a7IgXMnf?d38ST0JsG8$UT2Iec(DI*Yu>BhorHq$;Vwh zZlrXe=A|{+Pl)7ji$~hvo%#f_fZ?QqxiyNiQP1vzd02&+PQ_0eWJ{iJW~dsg9(ph) zf3N_9!@y+#I>vr`93VpkYMU~xXac0LEJ|?eVM&(t!Yqe@v=)X*^qa~;!bt}T>+5r5 zlBRNEiC-;#qF?zU(UDIP!L=#K`f?K_oo`K!GZXlMHXr*Y0kA$S*y9ea0yXKHAqQi! z6rLnzR0{4%F{RCOZqCFe(f6{9#qZ*LS5;0>1ID2nRwW=|CH{T4lf$(<26^1FfP_8} z9k#Rkh{=-8Z~JPn&Y-qf-F*EN1iDvh<}M`e1Li6~w@R$w`qv!?{NsFmT8kRMM}6zW zgMVdU$rRo{T7Cx^5&+a|bzq_5?-x*Gpu)BuE+XJ})B|1$Qopotf7S&4+l~R2#fq5L z`{xDT=|CM8Xb~j1{u~x?%wSIf>+S^uWq-TkFtxTKLOllV@$C=5iNfF?`SZC|e?B?` z7H@5dn?(Ng3U9<7*qt4YZT44CjQj~S6$G#W?9}h~08xD)czy!-e%M@U*biG}!MpCo z_+-Pdo4>!B#lYF_gb}*E{PTlZ2>1b4*u9ItPMaFz71$NTVe21%c{~F$;3c5V1|+cm zdYfhN#^8Re{`;e+v7N!(s@1iH&WP&0`Q9MTSoXj6o%-hrBA7INH}yo{P^+t|M*id999Hii^*@Q{yyGFVMV0I0f5QM10z61z~rz@+e^Dnz~T9Kj0Bc* znbI0bAp*mbg%WXm?-$5%`+L#Kn9i_4r?N2D7=RYdOaP79zo`jqos#iSf(8a&!PqJO z`%mWq2wb)sP=&`cY5ZzBnGJE|`wykQ{b*UCD?cz)T@OG&J-6v+sQ#=Mt328p%K;$u zfUld#W`wp5g3^*>;C?=Qgno;|gl!AhFMuQ!12I{JZreTak7y77C8dx!@b`;9X{6#a z!2ECH$&d8B4(onkHnP|^ko=|7U3mWYDm`U>g2|)=g6`b8FO%K^43?xU5Ttz4{+$b0ow=#7|546Z-y8yimIa!z{&`{E zOyJ4J-zJ6q9Cmpz zB~Bpy^RNH!VgJKv{@=s?Cn5j;RkJVO3WEu(v5!7{_<-hhN+^8KfSeX_1&+{~jK6_> zSCbr}&o@yBA7DJbmT;y1)6-xfQNvRf%MT-GW9F6pqgHu}0Lh<2p&|VfPyMIffgzs<}ktj$4CzTn*$XKe}1V$3b#C5 z5zfT)_vH}j4a>~hM`e@#5#?Pf_%u-2b4qKUJqoa1JH!Qu(RM3w(AVJ?OBh4-dV-`& zo4eiWavzKcxvsjaAzR$>|Fm`G@lfd9yQHM$GK0u&X6$3Htf7o8Wl3WxLUyumjrEck zjAiVi+y*z*y@a+w2r*+Akv01|C0PbpBERoc?t9@Jk1>IX>dneyQ=O z<_^Xlhj_i}!qom~s-vi>?w9fXJ(Ybx-bdV6Kqcv~gXfq0x2D-en(Yz~U&+}e#uSAa z!&EHQGBMAe*1wunj@+*B5lgWdPAYIu5<#ciW7#L=beSI~DH~6W6TL{;QbLbu?jPmK zy4bYD<-I7ZCahNW>l++hriZr zJPRZVUB6Ynmau;E)Gk4_?5XbXVC^mOL<>qQRzU7<0`x)E*QN8BotxF*)y|H3@zkda z3-(XoA^GO`37!Zj@?_H&@+1& zvd+Z*E7qBmCL?p1rGEY!6*c+hj(LvXEJ^j$N2`v-h&3)2@xO$y<2eEFFGkkkB^0v- z8qu@Wca9$UOls(K+nHO!ASSrZ1SVB0Wy{(o+C^^amOB84%*Md4`%XOhg{`&!V3*G~ zcWrllZ}zt}raq-xp~Z#}l)Y2=n^(O%r^Ne)&pX?pUAv~`zSqEue(m6U$w$$6!mo!F z_op6J=NVuI*elMXmq}+k+pXBKHAN)s>wNUtkE@#hD#ZNoGPN1MTbt41P5vZ+?xff6 zS&xQ3_uZCyw)-OCV79xW6!RAHmXHtUuCP@6*0e&smq46gP}C~9uIhcXj&{0`y1347 z{GFd9SqDr2^Jdhn7_GkTjQA9<1!1U^{b^FTU?Qvg(a6_V9%Q|_Taj2Y^8ix=IW5e6b9S_)6Ym*(w48R06{n=>iZ@OD~A{gV8Db< zeb!s~rEqn>(>|m8_20Lo@1CjZ$? zx-YDwX!k;dca=T++OxekKij-uy6bsIMAtv>#=c*+{>UG1zMi)PRfCB?o*Eqv zG;1A#Su%ND0O|7u$ivd9k*TK9UsqYX0R&ncJ0dRm)XzH;u{*DrW+6G8o^{xVy0qqCqOkPH&^545A1Zhd*)|E|P{ z8<--BnNL&P@yDaH9r?YkB}O@Vi=(g65k<9|v-t)&vI23(`A)P{^xH)VfY_v|A^Jcs z4YG#meT;y@fQ z31*(R)#1v+3stz4G+-+u?VE0ZCN;$>vl8U|R|l>}-Hf)qxst0E^!RR)ddLE5lRnd{ zWqn_C#U!!7QfqtZ_T3`IhhPs_Ai`~#l10?crEF+*KBY)6`Fz~5llHSUA{tEi4#O&1 zk+rwWXTzwDjhTg-i?D${u!gg>%2s)WteySS^e6wy60na0^NWgNDmK1%bwv0A5wju& z=GMo6`}Ua62ZEYbB7XXfmDP-`4+M4=%3tX(!PzC~CG)1m>qA_-ADWbVx0i%(OvE#E z341!}lQp@uDmBi3Bk2Pi07aqNj=Ug#79B&L_MQN}?Zn^8jB4_UOyJFnZN)VT& z(2t9=3vFs|nXTEy9@rei*||IdJpbIxWnb}tIZ%{&T!}LSE4w@zwkWBfdjTA;EFQR{ z=nWk5U?PY3Zz)1rWwS-?swYkIxZ?x{zvyQdU~`3AliUyi;&yf67v5#WpHtNnY>$bF z>9OI&>o>Q8_8N6HKo_(Rr<29!9L!M(OIyhECD?Pon3}OA!9My<)XfjhdM-Za%%a(_LZ&CScuH zTn5peC^H!oHq~-m^iYDY2j@g>#JAwJb$(TUdvM7vn4YFI;TByw4Ij0RlxsY2t}FXo zd&FnhR#(7qScZuT4olB)`eGO{n-rdjc^S6u-J1YfoTa0$)IFT3(WZK0%!FyR;bOZ+ z-^Kf3{rvqzDPR{>y_C^{TVi)*_PdxOuWbiEA zUs;60{}}z=rj@{?FqJd-K9OMDT#So9@<HbthyZzN%* z>2cSawJ;4CEN3xOHo$|6!i?{pV*UH-^UGZ}_tb(KmnNEec_{o@2K*3TC)z_xy9o3k zbi!aRp9|Iaxyr8=t6snRy~aa7Rsy0heGWbQy4@Yz>>`&I%PLJCw)jRg)d+G;Hy85e zBH<|FhOywnuuM~$8*wc1TZf3hp2P(*Q1 zLC5`!XsgaL5~6BqL28bN_N`{I?nzzzOwILKBT(YaG7X?l8oFgdet8=d1HtZq zqW|A%`EHWX4j3#|<0%+9D1XuQI%b9|=yt@%cOne+navfNWmJwcTJ%DusTSqZ)(j;F z-4lyUz{dw+c(O}msk(waqBryjZpCFlNIRf##sU37K~N((7;!{9dzjR%fDs1f89An4 z4*0_yL8!x-Eb2uhx^zM={;(_rf`j4ZxTLO_s?$i7A_=e3ieG;D@Z20P0(#cXCSy?) zQhr!M4>@1C9C`wA8P(D!0Wv0Lcr~cVnTV~xR9%67O};M%nMoQr-kBkvdbC$6kHYx` z3zuzw{SI6?K){X0vHV4(+m%SajCdzixskKOhP%$;u|$pEjXu45aC-~$CgxzN`NI^Q zo|C8C(h~g@+kiFgU5|FKqp9Sp3LSW!G6H0}7+XOKSpv5_Yr_W>YY(*`)J-c3}9JnRsQW%Z_5po3)yYFH4(HY0$snY#3Mrxl)pxiqPWIoik!i-KJ!M*+`z(;W^3Jvx;LlMg!M# zCz2J_)q>rxpN9S5CuZLVOc!hgoucsh_OkCsDQ)s*I?`IP6>1d17UF-M91_FnAn|r! z!6x=0zyFVa-HZS-@b%;-MO9D~Zaj6n^L4;9~7k zU6kZpZsE#b|HNVT2LQ%FEPONIj+UZd&;>Nc|3`^sL#hMh_MLk1J!fAb?f*7Ib5q9_*o@Fo z&?GX3x3QazL&aV|<)E&%@dHIOo0aP)2~^Nwe>g9HA@Q?K_9A!H*zt+vOHKjZ?^hHm zSKc18Gra8GR}PjmC`oOuHcBeV(fU7mCgGo59d{`r3qXu6FAggYYf;Go1BE96bn%E6 zA^K(8UP8FvjA3TYY=En3I1%GeH%J{sjbZZT-xB)pB~nz~V8!&Oc@CK_QgCedvNNpbp=rSYzfo@sM zG%8NRAOS0mv7dG@oz1Fn(h<~i3K3&nSBCi>o6sg0AQW3ATKFTUl9lB>`;hU%lhxA{ zD;Ouf6n-W-*r|>WkE9bfBn+{l`hY@}w@eW@tyIU$hR1K(Yw#zsi($NRFt)BrW<7!@ z!b7+=ULEeQ=YN8%DoIO47UILNTt4=#5xv2y>+eR^sljUB(4nd52g|DM0v3$;v-bYe z>I-GG*Yl6XHp_r=@*{sw@`eH4t-vm7r}tUulwonoiYnjWzsz$z(74Sl9QxP!-48;0 zKepb-C~}#ZTuama)ky&V=iXdRM0T)u>3)~(I-33jMaPn^MG*az6NhUlJZ+asuv+Qt zmMIy^m3mHbSQ+}bEL(JC27dk-JRwEYZx*nU_-hKVJP4y#FkpfCp?HH2`1qTSfb)5i z65LML7N#^J*o>Sa4bK}X-2omnvoWlcy+Ygn9m!&bdd)TUq>`S@`*b3lEsDMbtC1kv zD9k1=9~#9Tg5Zsr`TMsO^f+^;V;~isR-xhnN~ca;3EBq53#oMdn|fk%26bZOA*cCK z(N?}LZ9RgM`+J<-LTTP|?_X>!O%IUnoDRLWuqWz8Efesh0hP|=S{=Q5L9Iv}$eXJM zzs-;kC1*GzJ5S9Ut)}@ZC*mH?Gl>n|rFf&%S_c_G7qE%;?8yD~7DWwfHAKQ%38~D2 zPAmv>fd>JCCH4Qg#Kem|erFduI%8>`ikHRJR&OEqB1B1GJQi5`fO6VZvu?UP>HCqa$MDDHzkK#-}& zzqO=TxnOdaN9I*eV%O#qFDm{x^vkbhAva*(Dc>-6Efm#optz6MefFHkz9D7 z4k}9_y3f|Up{^(pR^B4*s>U!A5xaVUqxv&|C^21XI;HSW?&~+8JH@p_wnvq{nmvMN zal;JBCgAQFjnzeTJ@>Q(XG=c0(>L8b=G?rcfs>z$ZbvyMez+@Toz}>|^v3wb`l~O-hOx?1b7&__Yv<&LhZ4mmk=F%cF$_Vyk;9 z3w!cYewUqUq0?a2%7QIyfOj(q_`nwJ#44<4V{Qer@pl)Ea9*>}I4u|BGASSoPTIkJtyL{^30 z_#3Y>Rw@m70NUH)P;|qg39g@5<;kV0fNpS^zJ@DYGXWp199tLFq`p|>{~b_qkDM0N z*~?0fGunl=$ikb4r)6r9QFyP`cC)&e_CjK<) z7puORxjP!n1~E=k``??(iPBQ1UGVW2F+4#zxPF`+{<)EvxZ-)p?g;4tJcfXpHK1S7 z84u9Bob5r?>%VEAmrT$qRaf zT)=zGx>{%0`uyRai--Oh7N6X~rPxhWBua$M6uBHZKH-s>z%84}#rALOINoes)UReGbZ<46SD;}F6 z=_3^4S({Rat{QIdONvZgy$0%Ig_1IRmHEye4P&j`VhXwfc08hAyEvHAx2@B2D3^`M z&Z3ka8wx0VqDQ3$KT$?5tC)PKBFKi+SH?($kezg73zv^xv zU&OOA7ZDmP|q_doMWt@scZgTK3AXFCDkhK*kCs-Xo(Uz zofT#&TTqQt(x*N+*Xj|u_@rD##bHOAxxRqU-L7p?+BU^stJ79J&OB$m2t8P4_TtR} z0K-?IX|_WrkXHK!?5xt4Eic+ zSMv~?AP!WAUyUSF#`tpt3cHI9r+mD-goFU$t2O>mlI_nCnT&3b+>S@^ox%UkoJWc- zU~>aD3#C=6X^5DB5}iEDqeG{eZ?+<$wZ3*V~Tt%3eV}?La@4l zrha|+#k+v^_@n3&B{B~5qBM<~slQ4IKXUW_2TR&>`8V5U?PF88*&d_jZ=S8-rIi)N z9v1n28DVMqp;$!X{v#+naNF7R04Wuu-^M{;?co_L38QC9n(OV@R0p~<^rFPId+fqB9qQ#mu;71)C&CS^fCT=# z$-e5nfAf#)VmugU3?bf_eO1x_Ur+y2+1V#sKZ*FC1ufzILX5e5Vxbk6MoFP41qdp9yFQUNd2Pnb7tw8zWbHXVTvcjicSwjZJBcptfvxBRI5jWXIxm+ilyRMTvfH4c8JAeuA/XzwzOZ9+oj0jPHvjrHnjo2bY1vPXEj7RsEos3cBJDwLCvKuWGF/wLKeNAWRfYR3GhIqeUcDwvGic0itCEF2yQMboqVptSUrzrHAbIMLxMIDGtP7DPZ4n1xh3k9r8QDmbpna2BOhPCtLIyxDPo09WOCTz2wD2jlCdH4foeEQleikty3dOes1nHGIp4nQs8PnwlT39PfeufH084Dp5fFkHfcZNmlpAs1BOr3vJNCgGji8hHspVBD4xWM8zRyxxO5NmVcLqwzXhIRMkSh1NMyD0llIlyRCNRaWR2VPV9iRhH6x2T6vgzoiHibCOqqLNOiqpikT1U5VXuk6zObMcf9o0yQsWDIGs7h0ocKLQOQM66CuAsrwicU4KbU4JbY7DZBmwcxu8SN8SWWGCzH0SrNRBviiB6JoiWXQKi0xSI7rCafMgXOqaKlPEZDWgEyWNuHRXpmdf5Sulc4fkf4nyjRBkuOC2i7cN4lnkm5oy+ZxoJPkKeQxYg/sHzqTFDPsKH/mGIQI6XRakuQ1td+p1i0ZXMr0Dza+bEtImYLtgEqas0n2XdON6NwPAiDuVw1OmXAdgdexkcA0XKZ4h1G0XH6xiK7sGBwPkFo1qy9iKfvKnVklIpPTeNSM8dY3CzU2EuJSWur0yp35/21NeHea2+OEh6cFb58gzGhCiOtwLmwVC+RtE4nm+95hEuPT6HUYFR3s+FjH5Hk+T9upO4BuM/PAGn6L/o1aBw+Kc8ls2JMD7i/SkMMdkkl2W37NkAOOIXMgyJYc0ai2EU94U+4GneC3EUyF8fC2ri8YKjtOMCnqTvSQVzrCVEzEdQW2KSqcJGY8OOmmQR3UXUZNhGOILWmL/Ky7+4qvS2c+ZhrVreFjaq0LygnD2WOckxN4eqfCuE7muErj3dy2bfZwfu1gBOSIg/putORhdF+IDVdnSRdmAHviVmvD8ROg5xtI3WZgJPIo80PMUz8yJoSeSgwVaCJCQ4iERxIpAUDYORRBBPILlTJ0Ls+2SfbBffgzN4xbKqWV3qFn1acj63mIsYux4JYSRGc9MjHWC4PiXvAMXNlY0uSqvjVgMHLiutljkP7rK2DjUA3daZZ86ACY4W68+lri6ooQhlS53Nqas5pb4SdfWsznHcnGxeg7o6oH11NdeKr0hdnboxUnPMM6dMKxiHn0xcnWpeX1hczQnZlYpr+xRPb9ZxcdVXBZyyEd4rI2FzXzPNCVSE+Iqy9y4yr699unFKvgZfmHnmpIkhtfS2/yX+bWTVcEjdWWxjsmqbc4l4Jh6xP42vgdHuoHVGm0G/TPG5Evg8t234zGh1Scki7ORHXB0+rywquix8ZlTUxZHcHhw5kltuU8gBMwb6icJFF2lnd24YB2YYtAw/wwBuuKL1ARyYEZXPYECjMZQXd5/N7Q/h4DpyYnUR9cqAu7CImqt075BDeasAqRy6jtOv/TEcmCHQv98+j4SW0viyEmoGUe+IRegq5NO7bZu/jikCaULmp14zLf3cf9E1U6dG9g+K/Du5bUbiSGAc40nRGUWYjkm5Eoiyzetu4S1vQRbzy7al9Lpjs80rM7kUKpWZXKBuKteOg90S/6a2E7PXbe1TiVsze91MNtWD2KHWUIKM0dChSa16kqrnXiBJ1THFXMUigyxbVX8FhH+/wrHQ+wLz68sLQzH+Bcfb9iRzFTCicXfUcx9KyfrxC6uLTrbfTd2lt7ulrHSpYvBFONYu4J9uYTqWh2kVOp3GqJENEumb0oRg2bUFqyprXdT9jhgWDytpoWlcLmtvBVUr17jmRSxFsCMiZunZNPqnmroiZoNycTlexM5OZnNmfiKZq4h5bhJWj5BWp8hl60mC+sBWl1x6Voc+0rbPLTOFrmFunRT4XUoU0+XwauIOO01c91jiGg3ddo65Byd2NbDv7EhinrBfrZKRTkt7YLM/XjiUavpIbp88ANfscMXWOD3d5sStcaKY/2VDUj3/4wvw+D8=7Zxde5pKEIB/jZf68K1eRk2T9DQnaUxi25s+K6xAg6yFRWN+/VniouBsDTZAICe96MMOuIR3Zmdn9qulDuePZwFaOJfEwl5LkazHljpqKUpfM9j/sWC9ERgSF9iBa21E8k4wdp8wF0pcGrkWDjMPUkI86i6yQpP4PjZpRoaCgKyyj82Il33rAtkYCMYm8qB04lrU2Uh7urSTn2PXdpI3yxK/M0fJw1wQOsgiq5RIPW2pw4AQurmaPw6xF7NLuOABUabU+O63L0YreuZrshO0N5V9OuYn208IsE//umo6Gc4847cbkadfN79Hq98Xs59tZVP1EnkR58W/la4TgAGJfAvHlUgtdbByXIrHC2TGd1fMYpjMoXOPlWR2OXM9b0g8ErCyT3z20CDnH88/cokDih9TquMfc4bJHNNgzR7hdzWuFm6XcpeXVzstdxPVOSkN68mDiFuWva16R49dcIBH6On9wNRzsjTKQqkClBSFDzFLHCxdxuvPYOWXwYY0IA84QdtS1L7RVZGxh5zJsWzpuFsid1nJghcYsSwCr5YFXgPg3XnsYQsiv8d3NsOGaTK5hUJnWwdQj9XtTyWpRDWoWs3UYAA1EOrgoAw1cM+S0UBZnHXj7Th/icLPeLK6o9HyZnL1ebycRcO20gegXwZ8pOtO+5E/e56KPLuW07MrZRFPVJkifu8GlEmGxKfI9WMrLxA9A9+zNBH6njJVjVLR9zt6Bv42KE7RV0T0tdLow451gsI5p180/ipdiyHloS1rJdF2aIS/Rku8uB6YT6P19f2Pn1dtaOpzHIab7tRywwWiplO0uSPcm5kiczfMHp7OKgzIlQpdjTC5kQH+IPKpO4/xO8i3vHfkavbY5w3gS2MPAxhnvYh71pAU72BK8ynApfQgV7VSrl3Atbk0VSknze2YzKv6Q88LrZF93r6/uyBXET0Z4ytRfwh4YsvGY14kAXWITXzkne6kgyzx3TNfCFlwzr8wpWs+OoYiSvYTUhRQflfWmQD7yVCarBytkpBEgYkPGBEPetlLbUwP0FF4NhgTOKjhAHuIusvskFvhtg9j9VGAGOgpin9bcD5UVivogVZgwFbQKykHEneTMEz5iufR64hWEvNBloKQr1qWMOYYeiSymOg8T/9XF7JyDdEqAO0nN8BmgMyHg4FcXZjW0VyhSw0w7z6aElZocq4oTa4ySlOgS20wT2GcJuJZSJwmBgr96r+Yrkjw0JiOX1dr1/wTq0xRvSGEzsLGQNUgVEHqWy1UmFLcEy+aFz9kX6GlvjlUOC81tBm/RWOgGm8K9ay3Gt2uL6eDrvXln9n6K5mGrmB4cuyw7/7U6OZfoU8VQoUdlef60WOr+SPsWq75DFmTOn0VAi9kSkM4SAGAF463LKIyCLU0FRLdTppmQtd+WTyhp724YuVL5CO7BNMtja0hdTQ1C9fod9SuyGLL8Q9CvDrAe2JZLOliTlfagGOs7UZz1uUacIYTEDfbzPY9GLMeG7PR3/17Y95wYuJuYR4emK09Y0PuJv75Ra5yp9ctCW0PoL3F84WHaPGDNNUMLgpI18ya4bjY/eVlY0xZVXL3fbqAahGrOoVUBTMN98Pru8Zw1eXcXEtbUyXkCpOODVfJIXThRXaTCetSHQg3OMsQeANde8ndisd39dL4iubhDY+9dzBlF3Z8McKbdZm76G3zAHvh9hmgFMaI7s+1pxfncPYCdSDPtX1WNJk62MvUQUzcZSHNCb8xdy3refpfpOqsMdRKtbKkV9lyYD65dBlI0l6GxDw0fF+zVqRC1N2XUPcr9VEwteSkfUybzNmQ68UZppZW4pqa1t+KaOv1og0TS27VB6ep6oVZq7/zgEkmxzz1muSm9fqThrklJz2fu6QxoAUmXTM/LViE4WEbmfFHbBx2cxyIwKzrRhvmoEuHhMUHHtXMddW+X0wS0BTv6+FFM2mLrLtmtGEWA/PT1GLk/31mmm9qUzQzX54Oc+ydwL51Eh86sYOd0k+WXGbNfixIrdnfFNkf/Y0//Fz4zut5Lowe07dG67/SR2YF/6Hdli+u4E/PN4sOTeCy3Av9+RuuievTlFvdN4pusgE3qWMzRcd/tlM4qKm/X5MhjvKSejekQL3PhrTF8ArbguFzmbZ1nJXUVvn70zd5ld99uaqy9Q2D+A9972lJsH1u/+iVvAoXVLU/FF2ywgXb1j8ULtBST9nXeCeJnV7fyoH9lK10mNN8KP14X1xLhQ9ul+sT+8yNHsjYG03a7rD3TbCWcEj8mWsxhbgo/vX2jAo4dFCPeH57SFlcsD0Uhvw6e8LOA6amw+/M2Del954eZ4bHbCzpdfRu1lQES6Bl4XLHApICocbhnAlQ7N/vJKuKrN7LRVa0gqY0sHCSZOEhP85ZSIw3WixIAAeHPhrR0dsIhG1I6nekbjnaNtrfroPJ/Q9f/hV8vnw6Pb3QJoKtBPBQH2kWb32VhsMDLez4QzfqcoSY0gdjWJUeQCNUS474pZDj2/6Mu5LxQxWkB6JGoZbXJs7dH9/P+rfawD25e3ryNam3vha0iRD51pTEe0HmRS9AfuOjf/b5y6ItuJrUKWnliZA/NH4z5YzemwYSX3NIAeouTHilBlhxd/DtJvrenR6snv4H7VtZe5s4FP01fkw+FhvbjzF2lpk2ScfTpM3LfDIIowYQFcLL/PqRQKzCcdIYnHSSl8DVgjnn6ujqSvR0099cEBC6n7ENvZ6m2JuePu1pmjocG+wft2xTi6EIw5IgW1QqDHP0LxRGRVhjZMOoUpFi7FEUVo0WDgJo0YoNEILX1WoO9qpPDcESSoa5BTzZeo9s6qbW0UAp7JcQLd3syaoiSnyQVRaGyAU2XpdM+qynmwRjml75GxN6HLwMF4apeXtnX9yB7/6Gajd35s33k7Sz85c0yV+BwID+ctcmcLW/f5Dg29Wn+defV6rpuYZooqyAFwu8kM8BZW8LyQpZULw63WZ4EhwHNuR9qj19snYRhfMQWLx0zTyI2Vzqe6LYBpGb13WQ55nYwyTpSHccaFgWs0eU4EdYKrGH4wVjQZ88870FPitIKNyUWBc4XEDsQ0q2rIoo1frivbe1+3XhIWpGu1vyjqweEE65zLsugGcXAvtmHq4f/Nn1l8Xi0p2YN+vZ5OvlxbyBh7Pbq9dhL8HtaDvgNhbGwGgRbuOtoa1KaEtQs8Ee8ksrJt52QoD1COl+zKsEOTigQhDVUY2RAAewRcjVcRVz1ZAxb4JcU9rCXJMwpyB6PJTQ1JwdqvYADpucfWwMddCms9e1pQn5br1dl5DH1IWkFY3vClVdOzaq/Y40xEPh5fH1RH9zejKQ8PdhFCWxS40G1hkLNuEzROQJfI87g/a1tzaFGm9mCo3SYuV00CIBg8Fb8/+hRMAdItRkeAEUcHU3PI72gl8t+ZULAtvjBQecZ9ksO7L7TUNipC10o92gssbI4NhDYiQxcg8iv8wI8DmcwSIKE2CUg1DSyfxbR1sbHhvtsYT2JxTEmxLc7xheXT82vFn6poTvrmiRvSOtYgY8tAy48jOYGP76hCOBLOCdiQIf2ba3a0Yu6FEaZoC2KJDi9qNHmKq8TCVxQJEPn/BtmYyqNItJs2EefY+kSZOA2iFps0909HBhBpPbS+THfwZabE0bEjm/dVw0rMVFow7Dokb85TEzJWCJgwXgbQ8Y+lgWHDhOU+ijG/pYt1sEfVTz+i4zDI2gy7mdL9CP38G8W0fy+PohJ2vOLBK8RyiP7pVNGZp0UWSjFSuwPBClzzR+xnwTZ+KBR3hiMWnl2OTWbAGVtQ6LB2dVYsMZQejYoNyq4QEBPAkrVQpKM6MPyBKx6fasl4hkug3C+LJtFCzrZh8FJxmWvETrh5vdPzwKQbDrZyVxg9TS9HBsX25DHvlFOF9UMjrSvqr9M3PYYEvQzqzvzo+1o0uCnOlqz4/tvgUGcKH9Zn58jgi0kkjr/+nDR9diOTfC3NNBy5iAhZesZ0oiU8P3RYExIJaIiwfKixY1aRiXHRFIuoopFkG0mnqdlbjulCe8KIHwHHupMyfl3PIZrwpDANczewnn2ZsMJ4Vjwryg8FXu+Xk5W+fR7V/QAxThYNZQWzOLm4gCQs/4YYkd3SVLgua2MLB/sWUEl35VYPQzLpalKmyJs2LUVGr0htPnLWDaGh392l5GQy5d07scHHIqSxoDsORIu3yj4gc5ijm/xcJQ8MaQVwqO6oNJqYvV69b0EY6JBZ8AQYDL3mAJ6X4l4YA8STZJ8FmVKzVTJ5reYsR9OU/41w4tjGrkp+8jGtX4z3/FK5axch6hJph5KuhDLT/UskW11Gtbj3pjiqdLuWzIi74HvYQbRL/xiqequPvey9Jz/G66Eb0kN9vSzS0kbKQnI3L68mNxZeV9ait3r/JmmtS29OabH9n8rLemvZM1fPjDu/b968mPlW/OnZXrNqQQa9K7ayPkQ3o/pPeQ0nvS3x+otqW8jSNDzvN+CO8rhDc7Dl8S3qcEqW3dVUc7sgaH1139bvYQXW3v/1n8cG6+nG8j0xw36K7JddfmbgR463xvP5Lc7m1se+ZfCPCbpUhPSf4ZPUJquT15p7TVA0z1RW/TCQOjQUy0A4hJI93POED5im8NOoJ1aOyHtd8lqnKyOPSSbCnFHNs4DDGhH8PnpTyPVOV0rCv5n7p/MCnjU2XYIfPPOJH5uu9H9n6u00lue9B/KRN6x0TIue7nnsxUHJ7+Vkzzt2ROOkPY4s4auy0+zEsDlOLzRn32Hw==7Zpdc5s4FIZ/jS+bAQQYLhvno9OZ7GTqbbbdmx0FFFsTjByQY7u/fiVb4kNSCLbBcZskM4l1EAL0nPfo6OABGM1W1xmcT29IjJKBY8WrAbgYOI5tWz77xy3rrSXgLW6YZDgWnUrDGP9CwmgJ6wLHKK91pIQkFM/rxoikKYpozQazjCzr3R5IUr/qHE6QZhhHMNGt/+CYTsVTeFZp/4LwZEqLBxZHZlB2FoZ8CmOyrJjA5QCMMkLo9tNsNUIJnzw5L9HTl6evV9ercxz+m9w/X8Lnp+Gn7WBXu5xSPEKGUtrt0ILlM0wWYr7Es9K1nMCMLNIY8UGsAThfTjFF4zmM+NElcxlme8BJMiIJyZgpJSniJpLSKzjDCfeav/GMeYBj/YWW7O83MoMp6xLDfLoZ1maNKZ0l4iM/VbiRHbB2y4cXk/SMMopWFfRiMq4RmSGarVkXcdQVZwi/dgXlZekkXhhubdOKgwDZEQrHnBQjl5PPPoj534EF6JrFwAGxh4LYbQekgUGNFW/cQkpRlm4sjsWtOc3II6pcOnDuge/3CJBptUbQ1wm6UrtVgoWgOyfoni7B4+Px6nQCnY7tGuj4fcHxNDjZIqVsHhsY2S8x6pbFfeC5nqWzfwgiFEU9MvKdGqPQwMikoN5CoK8xuru5ecd8HFAXke29NaHh2yUM+69PTo+IwLCOaKgTckxxzg77QhRoiFiOyx76/coIhHVGbx7nQg3RZIFyhsh6ZF7Ld2CHpg3nm9+DGR6LiAMMsrFMyVtfSOR+tcKEbUkpxCnKDsfh+6ORZR2MY5LAPBdXzR8RjaayIfemVp87prDFamSCBnqDZv8G0I6GwxTWjoujxRYWpfFnXtZhrYh7M+aBH60w/cHBnHmi9VNMJv98sRLMNo21aBwGhbHI1j/kuLzxc3P9oSfb5VU3LXnZ3WjmZJFFqKGfyHgpzCaoaTyRd6G4Vu3SfaNavjCgl7YMJZDi53qNzOQP4gq3BPMMoshLLfvMqzuf7dUH2T65OK9allKGUvMnbdHdTo02EHMiuK50m/MOecMtqyuOcNbS37cjlt5fzOoBgmhRETAL4gX3fM07jy+kjvUQtNSDXK5PRBC+FSpLozJEWzm4vpKqKuN0pAZXqc7YvncENeglmA81dKMG+7TUYKuJ4m+mhqHVfFtq/3C3/o51DLXpxbRbEmuCY8km1Tbrsiij12lggie8rBIxB2c5Njjn+SqOYPJZHJjhOOZjGxPsegp+mml1i4KOSTr9bUxblNyMUVOZu1dmWkuQRey0K5GzjKPm2LlvnO44anoto2ZwUkHTCdScuniHvnNODbT0XH2t8kLk7Cz46EXIMWKAIUXMairqf8QhdXtv1Qga3k8ba5ZdxKFvc/T1KY5/XYDl+P72P//O/v7dhFSD+H7ebqovn4Ft4NPX+00zH72ofIeZJshD3sDpTy/9O9JLJSenpY66qP0bOcmL7byeH1jw2mMvdLrreWOEOpEF3VZcz1brrm2X8yIPkD7cU4lMu+GgeVuj9pf31dW2xiwfveT/IZ9O0mFwWuoBijOqTr+veuTmvnf1vFJEsF3zffWrHudN1bPfZrIUTKG5XtTTuKi8Kp/w1OVzZlV+9izIFXciQ77Xj5jUd5sHLi2sWX5Pe9u9/LY7uPwf7Zpdb6M4FIZ/TS5b+QMIXHbSmVlp1aqadnZm9mblgpuwJTgCp0n216+dmAR/pCEJpKmmGqmDD8YEP+e8Pj7Qw4Px/GtBJqMbltCsh0Ay7+HrHkIQgkD8Jy2LlSWULWkYFmmiOm0M9+l/VBmBsk7ThJZaR85YxtOJboxZntOYazZSFGymd3timX7XCRlSy3Afk8y2/kgTPlJP4YON/Q+aDkd8/cDqzJhUnZWhHJGEzWom/LmHBwVjfHU0ng9oJievmpeb8jv4l9G+3/8zGz3+83D3cH97sRrsyz6XrB+hoDlvd2jF8oVkUzVf6ln5oprAgk3zhMpBQA9/mo1STu8nJJZnZ8JlhO0pzbIBy1ghTDnLqTSxnH8h4zSTXvOQjoUHIHBLZ+LvNzYmueiSkHK0HBaKxoiPM3UoL1VuBEPRbvjwapJeaMHpvIZeTcZXysaUFwvRRZ311BXKrz1FebZxEj+KVrZRzUFQoDoS5ZjD9cibyRcHav73YIHbZtFDOPFpmHjNgLzCQGMlG3eEc1rkSwsC0lrygj3T2q1D9IiDoEOAIlY1goFN0KtiVyOIuiLonS/B0+PxdTqhTQd6DjqdhZdvwSmmORfz+AojuI1RuyweQ9/zgc3+KYxpHHfIKEAao8jByBVBXleMAovRXzc3vzEfhPUggv5bE+q/XcJw+PqEOkSE+zqivk0IuXQORl0hCi1EIscVD/37hhGOdEZvrnORhWg4paVABJ6F18od2LFpw6flv6MZnooIwo6wAa7krSsk1X61xkRsSTlJc1ocjyMIBgMAjsYxzEhZqruWz5THo6pR7U1BlzumqMFq5IKGO4MG3wG0k+FwydppcTQoJ9A8uZJlHdGKpTenUvjpPOU/JZhLX7V+qcmUx9dzxWzZWKjGcVAEi2LxsxpXNn4t79/3q/bmrstWddv9aJZsWsR0d8bLSTGkr42n8i6aaNUu2zfq5QsH+spW0Izw9EWvkbn8Qd3hjqUyg1jnpQBe+rrzQV8fZPXk6rp6WcoYysyfrEV3NTXWQMKJyKLWbSI7lK/8ZHPFgdjw99WIG+9fz+oRAdGgpuMOiC3uucs7Tx9ILcdD2DAequX6TAIiAJGxNBpDNA0HLzBSVWOclqLBM6ozMPBPEA0N6mMf0XBQNMDzigZoJorvLBr64PWfZfaP9uuPwCmizS543rHECjiRbHJrs14VZew6DcnSoSyrxMLBRY6NP8l8NY1JdqVOjNMkkWM7E2w9BT/PtLpBQccVOt1tTO2iaDPVNOZux0xbCbLSTlhTzo2OurXzUJ1uWTX9hqoZnpVootDMqdfv0PfOqbGVnpuvVbYoZ2viY9eJ76kATDgVVldR/0OHzO090Ag63k87a5ad6VDl0Xvr0JEb9QNyuHemQ/isdAhipEuHuSFvqkJr+TKyqrazN4iQ8z7bfhf03L+rrWyMT39cXwSLK/z3t5vvJbydZk+LVlZxXbV2vPPyWo+lS8+P6vF0AS7hrg2YbN3RQgT9UqW7iDJsR5mbwHmV0DygeyEOD13sI2OghgW01jzbXuqX6zuYFCymZbltpa859lst+qf5zGwLnvpCjn3bz9qo0ruB2S+I31KK0EFa5NeVCOyQoZYVx2uqONF5KY7xMQkyv0BoqjgQ6QNh01W7Vhz79fn6U64P1VmeDU8nOqK5+QZ6hXjzJTn+/D8=7Vpdc5s6EP01fmwHiU8/Nm7a5s5NJrfONM1TRwHFKAHkEXJs99dXsoUBidrEBsd3ksmMwy5CmD17VquDB/YoXXxlaBpf0ggnA2hFi4H9eQAhAJYn/knPcu0JpCUdE0YiNah0jMlvrJyW8s5IhPPaQE5pwsm07gxpluGQ13yIMTqvD3ugSf2uUzTBhmMcosT03pKIx+opXKv0f8NkEvPNA6szKSoGK0ceo4jOKy77fGCPGKV8fZQuRjiRwSvicnET336f3k3PfesxeAK31xN+82E92ZeXXLJ5BIYzvvfUd/dX83/8X/8Nb6eP367AxdgbAnWJ9YySmYqXela+LALI6CyLsJzEGthn85hwPJ6iUJ6di5QRvgeSJCOaUCZcGc2wdNGMf0EpSWTW3JBUZAC0rvBcfH6nKcrEkAjl8WpaIIyYp4k6lJeqNAKBsFs+vArSM2YcLyrQq2B8xTTFnC3FEHXWUVeovHZUKOZlkrjD4doXVxIEFpmDVGJONjOXwRcHKv4vwAJ0gsXuwG+JdQ0TaVwjzjHLVh5oSW/OGX3CBdwDaAfw3vY8LQ2EP3JxEDk9AujWAfQbAPSbAAR9AQgNANks4wKDLTiCXnA0QLoPXMe1TJAeghCHYY8gebAG0qa6VlACRYGtouT0BZJtgPTj8vINAwTtOo2A99oIOSdfB2HrOqiWw77As/06eIGJ3QanKnZg2Bd4rgGeaLDEQ79dgtlDjWCvXgI9A6PJDOcCI+tJ5Ljs/w9qAUVQz1Z/B4N4LEiamj9oNUAC+4LENyAR+yGOSIbZ4Wh43mhkWSeLhqMTpGkFakLD7guN4B2NkhtN5eqoaAx39wM4iz5JrUBYYYLynMiCjheE/5S4fHSVdadiKY8/LxRkK2OpjMMwEVCw5c9iXmncre7vu4Vd3nVlFbd9GZg5nbEQ7+5xOWITzHd3WjiqSShmalS3VA3IFz6GE8TJc114aUoHdYdrSmRnsOlELfDRrVcC6NYnWT+5uq6qdWhT6X2RsWlfh8aYSCQRWlaGTeWAfMtX1skCbS3d1zOWyb+J6gE6QQvRppkQf0nPXdl5fCJ1zAe3JR/8k+KDZw21dVGboi0bHE/vQLWJOmKDoykywHePwIYWstk7G/ZhQ3BabAB6l/h/Y0Ngbf9ebnPj1Xo8OAbbTI3zmkYG4USvyY1NePFqwJRHUEImUlsJRYKLFts+k+0qCVHySZ1ISRTJuRv763oHfppddQuhpok7ve03gSmDtquaWux2RNpokFXtBJXKWdbR5tq5b53uuGrCllXTPamqCQO9pzZyqnVPbRvtud+uq+6s+JjK8NmMJJxkwtmk4r+Xoe2vzBpkLwCOWoZMudgsQ4JJY2VSxmM6oRlKzkuvFvlyzL+UTlWAHzHnSxVhNOO0Hv6OSwXwWtYKr2WtaF0EDgPD1IUvsoRk+MMPInhAH3IDm7ej4uuvyTZy16up+MDUjNst4fsuqAdqY3tsm15pCQdteXmcNRzob2h1MavtCq7/5MV4C9jRzgcUmaipb/3uTEzR/p0OndDBPjE21JOrMzZA7zhs0HSAl7JBmOVPDtfDyx9u2ud/AA==7Vpdc+I2FP01PLJjSf7iMSGEbadps8smDX1TbAW0ayzGFgH66ythGduSlziAgc5mmGGsa/ka33PP1dE1HdSfrYYJnk/vWEiiDrTCVQfddCDs2a74loZ1ZnAtOzNMEhpmJlAYRvRfooyWsi5oSNLKRM5YxOm8agxYHJOAV2w4SdiyOu2FRdW7zvGEGIZRgCPT+jcN+TSz+o5V2D8TOpnmdwaWOjPD+WRlSKc4ZMuSCQ06qJ8wxrOj2apPIhm7PC645z6MfyNfxwysx8G0+3R/1e1mzm7fc8n2ERIS8+O6hpnrVxwtVLzUs/J1HsCELeKQSCdWB10vp5ST0RwH8uxSZIywvdAo6rOIJcIUs5hIE4v5LZ7RSGbNNzoTGQCtP8lSfH9lMxyLKSFOpxu3QAymfBapQ3mpSiPgi3HDh1dBeiUJJ6sS9CoYQ8JmhCdrMUWdtdUVKq9thfKySBKn18ts01KCwDxzsErMydZzEXxxoOL/DizQUbB4O/A7Yl3BRA7uMeckiTcWaElryhP2g+RwdyDy4TNyXS0NhD10iB/aLQLoVAH0agD06gAEbQFoGwAmi5gLDHbgCFrB0QDp2XdsxzJBevEDEgQtguTCCkjb6lpCCeQFtoyS3RZIjgHS493dLwwQRFUaAffcCLkXXwdh4zqolsO2wENeFTzfxG6LUxk70GsLPM8ATwgs8dC/LsFQTyPY2Uugb2A0WZBUYGT9EDku5f9BElAE9XrzORjEU0FSJ/6gVQMJbAuSngGJ2A9xTGOSHI6G6/b7lnWxaNg6QepWoDo0UFto5ArxA44yEc4HBzDgMFAgcXgluwViFEQ4Taks6WRF+ZME5pOjRmMVTHl8s1KYbQZrNTgMFIFFsn7K/crBeHN/z8nHxV03o/y270MzZYskIDvmKZXLcTIhu/wprUXCShPFzI3ypqoG+tyWkAhz+lptvdTlg7rDPaNSG2y1qAU+OdVaAJ2qk+zJ1XXlbofmSldGxrY9C43hSCQRXpemzeWEdMdP1tkCkZbvmcci+7dRPYAQDdo29YT4SXq+lZ2nJ9KR+eA15EPvovjgWj1tZdRcNGWD7eoaVHN0JDbYWk8GeM4J2NCgcfbBhn3YkOugS6ED0IXi/40OvrX7dzn10qvxfHAKupltznsWGowTapMb+/D87YDZIcERncj2SiAyXIhsdC0FKw1wdKVOzGgYSt+1CruqwS9TVzfo1dRxp7UtJzA7oc3Kpha7NyJtKGRVPEGpdBaFtL547luoj1w27YZl07uoqgl9XVQbOdVYVCNDn3vNZPXRio/ZHL5e0Ih3aSysdZ38jzq0+7VZTesLgFPWIWg2W9qVbwfu8PcQf2eqQ85F1SGAYLV06DvyplVIf3MP3Xa29gBqP7iq3g5WV+7t8+Cv73j45dtoMRx+/wetbmh3XzKUMrZahN54lWUfKcULMhb8G1fod5IlGpnU+LyeXw+uBt5D9Dhyfv/yx+NDl3cva2Pju9VE09/8NF6f81q61sbHX513RbXuDxDWPGEBSdOfrc+l/D3XUn2KxRc6bj1C5eUXOWam7dFcF8Piz2kZxMU//NDgPw==7Z3Rcps4FIafJpdhkAQCLpM0bWfb7qZNO9vuTQeDbNgAYjC24z59hS0MWAQ7MUZsVrnImAMWRv/5dMSRBBfoJn58l7lp8In6JLqAuv94gd5cQGgAwP4XhvXWYOvm1jDLQn9rApXhPvxFuFHn1kXok3njwJzSKA/TptGjSUK8vGFzs4yumodNadQ8a+rOiGC499xItP4d+nnAr8LUK/t7Es6C8sxA53titzyYG+aB69NVzYRuL9BNRmm+/RQ/3pCoqLuyXuIr78fnN95tguKrf4P53XQB08ttYW+f85XdJWQkyV9cdPrwKZtOYvNj+mX94evbL+Yf8c9LiLZlL91owSssI3O6yDzCrBmZEnbOzefJuqgCN/En9JHXR74uK3m+CuPITdjW9Tx3s5y7AasmdB3QLPxFk9yNmKUweEEY+R/dNV0UF5NnhJQbtWO/MjPbC5iR/aDwlzvZnGvz/eKAMCHZ13VKeAnFYXSR+MTnX9qJVXxjFrnzOf88fyC5F/CNKSuK/1Zgs+0jK5oLsiRZTh5rbsYr/h2hMcmzosL4XtvmPsQhQpbGKVpVPolLZIKaP1q6wVngHMx2hVdasw9c7udIb3RIv6fvKghzcp+6XrG9Yu1EoVUeR6VCPVT9NIyiGxrRbHNGNJ1Ooedt/CmjD6S2x8cTbOJWsbpd/Gi1jKZYhu5oumXquz9Rul2jUZeuLKZ/5UxBuYTkK5o9MCOrjDBfj0BBAnyTWG0KOthC7lkVBE5TQggYcNJlw4JsgkzEZ8GLb5JoQle3leF6Y2A76i1qu4ht8mwa5qsiojJLQqvG+m0YlU0zSfzyCK+QPfS2Rn4IeL5mvEE53AyxXzIjnSVy/Yoq6vSBjERuHi6b8b9/NS1BTYZNVoTK6Vzh18AP61ADsumzFX2n0GeNij4khkCl5q6JPKgmwn2ryb96R0N2KVVDoINmS3DpOEjXHGfXm8LNIreXyEvZ85HdzzrBbcQQXPWcwmRK/+cNN9rr+pa3vtJabSRGWcX5Mzi3B+Ic4D3OmZs4MkEXoz2TJN2dTiFeKgWwpWHZlDuK8qd9+CDlpS7DU+7YMiEvr7vmNkvCPLhIgoyGdUa67RttrNtwgvB5E1l792G2IR11AwiaKdSfgTocCHUkdNxtUybqUHCbNFgzadxI4f4U7mMI7YY4wKR4fwbvhjTei6Qdksa7ODhF84BkkmH33XmwK2IwjiFreaVzrBJuHVnRwxzLS7hBYNaHL/eGk8/NsZhwC5MoTIphkmWY5SEdxWCJzKi9f0Nu4hHQrtJuJ9EuLe1mO8DRbGm0i1k3JlviuxFVxD9NPDaQdOKxiu+nJNrxUPEdWHvEW8iy68TDQYnHXQNq8XYuqXTWx5RuN8AIWFfRvcOVD7M+WHTfZx1b2NasinVgDgu7GN6TbWAfC+ljyr4hJH/SE1YDa6eQbg02sCaSbiENyYrqljiwlnsMtXyTgFOwi7CPIaxbamjtJNihNNgBNDUMq7A+LOxQcBsmdK7C+hOkO+aQYR3pywB9T9589q1Z9jNZXv1pR5di89y1aGu3XkpQ8rUs26q7C295RH/oZyUXMPdSbrYtOIOBW5wBA+dM3iA2+//1dVydTv/SZVwI6Bp0qj/bkEzxK10O1CVePVp3+XI9WLce1/tqhGOH00ykQ+ZGu1htnu0WvOu6G7Ga5qPIqL8oy9YP6cJ6I6AZNdIdSzLpr3Tp0SCkO9JIh1Am6WLmZkmjRTyGmD4i0pFhNVGXTDoQu+YK9WNRLx81IoF1IDWqA7EL782Y0qmCvS6TaTqaXu/BS4b9tS7oPwn2ljky7ZUnrQuPTBtoJpQFe8uDAyYR9YoBc3l9+QMTW/vhWebq0nYpVMf8BICH6pkLk9yQga36lNaB+RV75psHfxTgSiS4j5T5eTBnLfC4bseR6qQ/eb95EHskrZOOgNOY/WKcbUlK54U3uI8Lw1juy8fEvA21OvKO7GQ7gor5lzOPZIV66GCo2VBSqEfi4sUq1CvsW3r02Gh5sOXApItLBxXpR5Pe+0P2jibdRMx3Bli50nndLTfl8jiXclNeZNSlI6wSaycgPFRiTURYx3Y9WKNhEW55IOdDmKpI3Yq5Ae1GB112Lr3lgV0K+aORl5aKAyY2NatCHg4ctsVcnE+m7iLKFfat2JtAlzVe/v6bHX5wvwd3i3u69n7E/3z7633LTFZBrya4B9Trbf5oo+9VbNy5eU6yZGOButGqWU9TTA29KZpVvj2g/rIA2CKTeS6ZjlhZ0OTmgEx9jDH3Vdl7aUbLaanslrpG56prMVH15HRedo35nv83KpD7cot7u1E4K3zZY9VJmP26qLHiaVpXfEcc+v4mrrYp2UTySeYaDEl57wbYn73fpq41pLpHPA7rFJJeFGr6mhlvjYykrhec6PPFJF+nRByrU1AdEhobh4UG5pBKH/HwimdRdSDz0lM92vtxXjYwYtoljdykuAOhG2LSlGbic9cUMAeFto8ABvWjNNusXkm2vc+q3uuGbn8D \ No newline at end of file From fffcb8165235caa0507685c9e6d7865d09d55922 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 5 May 2022 11:43:48 +0200 Subject: [PATCH 0096/1953] docs: update the content of the report 1. Explain why the current situation is a problem. 2. We are beyond a simple introduction now, it's a real proposal. 3. Explain why you think it is solid, and fix a grammatical error. 4. The Rust rationale does not really belong to the initial paragraph. Also, I rephrased it to highlight the contrast with Go and the Kata community's past experience switching to Rust for the agent. Fixes:#4193 Signed-off-by: Christophe de Dinechin --- docs/design/architecture_3.0/README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/design/architecture_3.0/README.md b/docs/design/architecture_3.0/README.md index 6211796474df..7e31a8ce0ffb 100644 --- a/docs/design/architecture_3.0/README.md +++ b/docs/design/architecture_3.0/README.md @@ -1,17 +1,33 @@ # Kata 3.0 Architecture ## Overview -In cloud-native scenarios, there is an increased demand for container startup speed, resource consumption, stability, and security. To achieve this, we would like to introduce you to a solid and secure Rust version kata-runtime. We chose Rust because it is designed to prevent developers from accidentally introducing flaws in their code that can lead to buffer overflows, missing pointer checks, integer range errors, or other memory-related vulnerabilities. Besides, in contrast to Go, Rust makes a variety of design trade-offs in order to obtain the fastest feasible execution performance. +In cloud-native scenarios, there is an increased demand for container startup speed, resource consumption, stability, and security, areas where the present Kata Containers runtime is challenged relative to other runtimes. To achieve this, we propose a solid, field-tested and secure Rust version of the kata-runtime. + Also, we provide the following designs: - Turn key solution with builtin Dragonball Sandbox - Async io to reduce resource consumption - Extensible framework for multiple services, runtimes and hypervisors - Lifecycle management for sandbox and container associated resources + +### Rationale for choosing Rust + +We chose Rust because it is designed as a system language with a focus on efficiency. +In contrast to Go, Rust makes a variety of design trade-offs in order to obtain +good execution performance, with innovative techniques that, in contrast to C or +C++, provide reasonable protection against common memory errors (buffer +overflow, invalid pointers, range errors), error checking (ensuring errors are +dealt with), thread safety, ownership of resources, and more. + +These benefits were verified in our project when the Kata Containers guest agent +was rewritten in Rust. We notably saw a significant reduction in memory usage +with the Rust-based implementation. + + ## Design ### Architecture ![architecture](./images/architecture.png) ### Built-in VMM -#### Why Need Built-in VMM +#### Current Kata 2.x architecture ![not_builtin_vmm](./images/not_built_in_vmm.png) As shown in the figure, runtime and VMM are separate processes. The runtime process forks the VMM process and interacts through the inter-process RPC. Typically, process interaction consumes more resources than peers within the process, and it will result in relatively low efficiency. At the same time, the cost of resource operation and maintenance should be considered. For example, when performing resource recovery under abnormal conditions, the exception of any process must be detected by others and activate the appropriate resource recovery process. If there are additional processes, the recovery becomes even more difficult. #### How To Support Built-in VMM @@ -52,7 +68,7 @@ The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thr └─ container stdin io thread(M * tokio task) ``` ### Extensible Framework -The Rust version kata-runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration. +The Kata 3.x runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration. ![framework](./images/framework.png) ### Resource Manager From 026aaeecccd280851681511ee07302e68c146cb5 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 16 May 2022 13:24:55 +0800 Subject: [PATCH 0097/1953] docs: add FAQ to the report 1.provide answers for the qeustions will be frequently asked 2.format the document Fixes:#4193 Signed-off-by: Zhongtao Hu Signed-off-by: Chao Wu --- docs/design/architecture_3.0/README.md | 108 +++++++++++++++++-------- 1 file changed, 74 insertions(+), 34 deletions(-) diff --git a/docs/design/architecture_3.0/README.md b/docs/design/architecture_3.0/README.md index 7e31a8ce0ffb..9c5dd9657dff 100644 --- a/docs/design/architecture_3.0/README.md +++ b/docs/design/architecture_3.0/README.md @@ -4,8 +4,8 @@ In cloud-native scenarios, there is an increased demand for container startup sp Also, we provide the following designs: -- Turn key solution with builtin Dragonball Sandbox -- Async io to reduce resource consumption +- Turn key solution with builtin `Dragonball` Sandbox +- Async I/O to reduce resource consumption - Extensible framework for multiple services, runtimes and hypervisors - Lifecycle management for sandbox and container associated resources @@ -31,7 +31,7 @@ with the Rust-based implementation. ![not_builtin_vmm](./images/not_built_in_vmm.png) As shown in the figure, runtime and VMM are separate processes. The runtime process forks the VMM process and interacts through the inter-process RPC. Typically, process interaction consumes more resources than peers within the process, and it will result in relatively low efficiency. At the same time, the cost of resource operation and maintenance should be considered. For example, when performing resource recovery under abnormal conditions, the exception of any process must be detected by others and activate the appropriate resource recovery process. If there are additional processes, the recovery becomes even more difficult. #### How To Support Built-in VMM -We provide Dragonball Sandbox to enable built-in VMM by integrating VMM's function into the Rust library. We could perform VMM-related functionalities by using the library. Because runtime and VMM are in the same process, there is a benefit in terms of message processing speed and API synchronization. It can also guarantee the consistency of the runtime and the VMM life cycle, reducing resource recovery and exception handling maintenance, as shown in the figure: +We provide `Dragonball` Sandbox to enable built-in VMM by integrating VMM's function into the Rust library. We could perform VMM-related functionalities by using the library. Because runtime and VMM are in the same process, there is a benefit in terms of message processing speed and API synchronization. It can also guarantee the consistency of the runtime and the VMM life cycle, reducing resource recovery and exception handling maintenance, as shown in the figure: ![builtin_vmm](./images/built_in_vmm.png) ### Async Support #### Why Need Async @@ -48,11 +48,11 @@ We provide Dragonball Sandbox to enable built-in VMM by integrating VMM's functi - Add 3 I/O threads with a new container - In Sync mode, implementing a timeout mechanism is challenging. For example, in TTRPC API interaction, the timeout mechanism is difficult to align with Golang #### How To Support Async -The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the tokio thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block io and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable. +The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the `tokio` thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block I/O and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable. **For N tokio worker threads and M containers** -- Sync runtime(both OS thread and tokio task are OS thread but without tokio worker thread) OS thread number: 4 + 12*M +- Sync runtime(both OS thread and `tokio` task are OS thread but without `tokio` worker thread) OS thread number: 4 + 12*M - Async runtime(only OS thread is OS thread) OS thread number: 2 + N ```shell ├─ main(OS thread) @@ -72,38 +72,78 @@ The Kata 3.x runtime is designed with the extension of service, runtime, and hyp ![framework](./images/framework.png) ### Resource Manager -In our case, there will be a variety of resources, and every resource has several subtypes. Especially for Virt-Container, every subtype of resource has different operations. And there may be dependencies, such as the share-fs rootfs and the share-fs volume will use share-fs resources to share files to the VM. Currently, network and share-fs are regarded as sandbox resources, while rootfs, volume, and cgroup are regarded as container resources. Also, we abstract a common interface for each resource and use subclass operations to evaluate the differences between different subtypes. +In our case, there will be a variety of resources, and every resource has several subtypes. Especially for `Virt-Container`, every subtype of resource has different operations. And there may be dependencies, such as the share-fs rootfs and the share-fs volume will use share-fs resources to share files to the VM. Currently, network and share-fs are regarded as sandbox resources, while rootfs, volume, and cgroup are regarded as container resources. Also, we abstract a common interface for each resource and use subclass operations to evaluate the differences between different subtypes. ![resource manager](./images/resourceManager.png) ## Roadmap -- Stage 1: provide basic features（current delivered） -- Stage 2: support common features +- Stage 1 (June): provide basic features (current delivered) +- Stage 2 (September): support common features - Stage 3: support full features -| **Class** | **Sub-Class** | **Development Stage** | -| --- | --- | --- | -| service | task service | Stage 1 | -| | extend service | Stage 3 | -| | image service | Stage 3 | -| runtime handler | Virt-Container | Stage 1 | -| | Wasm-Container | Stage 3 | -| | Linux-Container | Stage 3 | -| Endpoint | Veth Endpoint | Stage 1 | -| | Physical Endpoint | Stage 2 | -| | Tap Endpoint | Stage 2 | -| | Tuntap Endpoint | Stage 2 | -| | IPVlan Endpoint | Stage 3 | -| | MacVlan Endpoint | Stage 3 | -| | MacVtap Endpoint | Stage 3 | -| | VhostUserEndpoint | Stage 3 | -| Network Interworking Model | Tc filter | Stage 1 | -| | Route | Stage 1 | -| | MacVtap | Stage 3 | -| Storage | virtiofs | Stage 1 | -| | nydus | Stage 2 | -| hypervisor | Dragonball | Stage 1 | -| | Qemu | Stage 2 | -| | Acrn | Stage 3 | -| | CloudHypervisor | Stage 3 | -| | Firecracker | Stage 3 | +| **Class** | **Sub-Class** | **Development Stage** | +| -------------------------- | ------------------- | --------------------- | +| Service | task service | Stage 1 | +| | extend service | Stage 3 | +| | image service | Stage 3 | +| Runtime handler | `Virt-Container` | Stage 1 | +| | `Wasm-Container` | Stage 3 | +| | `Linux-Container` | Stage 3 | +| Endpoint | VETH Endpoint | Stage 1 | +| | Physical Endpoint | Stage 2 | +| | Tap Endpoint | Stage 2 | +| | `Tuntap` Endpoint | Stage 2 | +| | `IPVlan` Endpoint | Stage 3 | +| | `MacVlan` Endpoint | Stage 3 | +| | MACVTAP Endpoint | Stage 3 | +| | `VhostUserEndpoint` | Stage 3 | +| Network Interworking Model | Tc filter | Stage 1 | +| | Route | Stage 1 | +| | `MacVtap` | Stage 3 | +| Storage | Virtio-fs | Stage 1 | +| | `nydus` | Stage 2 | +| Hypervisor | `Dragonball` | Stage 1 | +| | QEMU | Stage 2 | +| | ACRN | Stage 3 | +| | Cloud Hypervisor | Stage 3 | +| | Firecracker | Stage 3 | + +## FAQ + +- Are the "service", "message dispatcher" and "runtime handler" all part of the single Kata 3.x runtime binary? + + Yes. They are components in Kata 3.x runtime. And they will be packed into one binary. + 1. Service is an interface, which is responsible for handling multiple services like task service, image service and etc. + 2. Message dispatcher, it is used to match multiple requests from the service module. + 3. Runtime handler is used to deal with the operation for sandbox and container. +- What is the name of the Kata 3.x runtime binary? + + Apparently we can't use `containerd-shim-v2-kata` because it's already used. We are facing the hardest issue of "naming" again. Any suggestions are welcomed. + Internally we use `containerd-shim-v2-rund`. + +- Is the Kata 3.x design compatible with the containerd shimv2 architecture? + + Yes. It is designed to follow the functionality of go version kata. And it implements the `containerd shim v2` interface/protocol. + +- How will users migrate to the Kata 3.x architecture? + + The migration plan will be provided before the Kata 3.x is merging into the main branch. + +- Is `Dragonball` limited to its own built-in VMM? Can the `Dragonball` system be configured to work using an external `Dragonball` VMM/hypervisor? + + The `Dragonball` could work as an external hypervisor. However, stability and performance is challenging in this case. Built in VMM could optimise the container overhead, and it's easy to maintain stability. + + `runD` is the `containerd-shim-v2` counterpart of `runC` and can run a pod/containers. `Dragonball` is a `microvm`/VMM that is designed to run container workloads. Instead of microvm/VMM, we sometimes refer to it as secure sandbox. + +- QEMU, Cloud Hypervisor and Firecracker support are planned, but how that would work. Are they working in separate process? + + Yes. They are unable to work as built in VMM. + +- What is `upcall`? + + `Dbs-upcall` is a `vsock-based` direct communication tool between VMM and guests. The server side of the `upcall` is a driver in guest kernel (kernel patches are needed for this feature) and it'll start to serve the requests once the kernel has started. And the client side is in VMM , it'll be a thread that communicates with VSOCK through `uds`. We have accomplished device hotplug / hot-unplug directly through `upcall` in order to avoid virtualization of ACPI to minimize virtual machine's overhead. And there could be many other usage through this direct communication channel. It's already open source. + https://github.com/openanolis/dragonball-sandbox/tree/main/crates/dbs-upcall + +- What is the security boundary for the monolithic / "Built-in VMM" case? + + It has the security boundary of virtualization. More details will be provided in next stage. \ No newline at end of file From 2bb1eeaecc71e5e7857912effb5ed9da1c2298ab Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 24 May 2022 11:14:33 +0800 Subject: [PATCH 0098/1953] docs: further questions related to upcall add questions and answers for upcall Fixes:#4193 Signed-off-by: Zhongtao Hu --- docs/design/architecture_3.0/README.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/design/architecture_3.0/README.md b/docs/design/architecture_3.0/README.md index 9c5dd9657dff..562404bd51b4 100644 --- a/docs/design/architecture_3.0/README.md +++ b/docs/design/architecture_3.0/README.md @@ -133,7 +133,7 @@ In our case, there will be a variety of resources, and every resource has severa The `Dragonball` could work as an external hypervisor. However, stability and performance is challenging in this case. Built in VMM could optimise the container overhead, and it's easy to maintain stability. - `runD` is the `containerd-shim-v2` counterpart of `runC` and can run a pod/containers. `Dragonball` is a `microvm`/VMM that is designed to run container workloads. Instead of microvm/VMM, we sometimes refer to it as secure sandbox. + `runD` is the `containerd-shim-v2` counterpart of `runC` and can run a pod/containers. `Dragonball` is a `microvm`/VMM that is designed to run container workloads. Instead of `microvm`/VMM, we sometimes refer to it as secure sandbox. - QEMU, Cloud Hypervisor and Firecracker support are planned, but how that would work. Are they working in separate process? @@ -141,9 +141,30 @@ In our case, there will be a variety of resources, and every resource has severa - What is `upcall`? + The `upcall` is used to hotplug CPU/memory/MMIO devices, and it solves two issues. + 1. avoid dependency on PCI/ACPI + 2. avoid dependency on `udevd` within guest and get deterministic results for hotplug operations. So `upcall` is an alternative to ACPI based CPU/memory/device hotplug. And we may cooperate with the community to add support for ACPI based CPU/memory/device hotplug if needed. + `Dbs-upcall` is a `vsock-based` direct communication tool between VMM and guests. The server side of the `upcall` is a driver in guest kernel (kernel patches are needed for this feature) and it'll start to serve the requests once the kernel has started. And the client side is in VMM , it'll be a thread that communicates with VSOCK through `uds`. We have accomplished device hotplug / hot-unplug directly through `upcall` in order to avoid virtualization of ACPI to minimize virtual machine's overhead. And there could be many other usage through this direct communication channel. It's already open source. https://github.com/openanolis/dragonball-sandbox/tree/main/crates/dbs-upcall +- The URL below says the kernel patches work with 4.19, but do they also work with 5.15+ ? + + Forward compatibility should be achievable, we have ported it to 5.10 based kernel. + +- Are these patches platform-specific or would they work for any architecture that supports VSOCK? + + It's almost platform independent, but some message related to CPU hotplug are platform dependent. + +- Could the kernel driver be replaced with a userland daemon in the guest using loopback VSOCK? + + We need to create device nodes for hot-added CPU/memory/devices, so it's not easy for userspace daemon to do these tasks. + +- The fact that `upcall` allows communication between the VMM and the guest suggests that this architecture might be incompatible with https://github.com/confidential-containers where the VMM should have no knowledge of what happens inside the VM. + + 1. `TDX` doesn't support CPU/memory hotplug yet. + 2. For ACPI based device hotplug, it depends on ACPI `DSDT` table, and the guest kernel will execute `ASL` code to handle during handling those hotplug event. And it should be easier to audit VSOCK based communication than ACPI `ASL` methods. + - What is the security boundary for the monolithic / "Built-in VMM" case? It has the security boundary of virtualization. More details will be provided in next stage. \ No newline at end of file From 87d38ae49fa0dc69d0956e50522d1cc207d0975b Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Fri, 10 Jun 2022 16:39:52 +0800 Subject: [PATCH 0099/1953] Doc: add document for Dragonball API add detailed explanation for Dragonball API Signed-off-by: Chao Wu --- src/dragonball/docs/api.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/dragonball/docs/api.md b/src/dragonball/docs/api.md index cd2bc2db8e7f..dab49835ae24 100644 --- a/src/dragonball/docs/api.md +++ b/src/dragonball/docs/api.md @@ -3,5 +3,25 @@ We provide plenty API for Kata runtime to interact with `Dragonball` virtual machine manager. This document provides the introduction for each of them. -TODO: Details will be added in the Part III PR for `Dragonball` +## `ConfigureBootSource` +Configure the boot source of the VM using `BootSourceConfig`. This action can only be called before the VM has booted. + +### Boot Source Config +1. `kernel_path`: Path of the kernel image. `Dragonball` only supports compressed kernel image for now. +2. `initrd_path`: Path of the initrd (could be None) +3. `boot_args`: Boot arguments passed to the kernel (could be None) + +## `SetVmConfiguration` +Set virtual machine configuration using `VmConfigInfo` to initialize VM. + +### VM Config Info +1. `vcpu_count`: Number of vCPU to start. Currently we only support up to 255 vCPUs. +2. `max_vcpu_count`: Max number of vCPU can be added through CPU hotplug. +3. `cpu_pm`: CPU power management. +4. `cpu_topology`: CPU topology information (including `threads_per_core`, `cores_per_die`, `dies_per_socket` and `sockets`). +5. `vpmu_feature`: `vPMU` feature level. +6. `mem_type`: Memory type that can be either `hugetlbfs` or `shmem`, default is `shmem`. +7. `mem_file_path` : Memory file path. +8. `mem_size_mib`: The memory size in MiB. The maximum memory size is 1TB. +9. `serial_path`: Optional sock path. From 3d20387a25a8910413185a8317d073f087e71a6e Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 17:17:02 +0800 Subject: [PATCH 0100/1953] dragonball: add virtio-blk device support Virtio-blk devices are supported. Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 1 + src/dragonball/src/api/v1/vmm_action.rs | 102 +++ .../src/device_manager/blk_dev_mgr.rs | 795 ++++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 45 +- src/dragonball/src/error.rs | 5 + 5 files changed, 945 insertions(+), 3 deletions(-) create mode 100644 src/dragonball/src/device_manager/blk_dev_mgr.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index d1b87beda4c9..fcedd492ced6 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -49,6 +49,7 @@ acpi = [] atomic-guest-memory = [] hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] +virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] [patch.'crates-io'] dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index d4d598da89a3..253a6854ce6e 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -17,6 +17,10 @@ use crate::event_manager::EventManager; use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; +#[cfg(feature = "virtio-blk")] +use crate::device_manager::blk_dev_mgr::{ + BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, +}; #[cfg(feature = "virtio-vsock")] use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; @@ -29,6 +33,10 @@ pub enum VmmActionError { #[error("the virtual machine instance ID is invalid")] InvalidVMID, + /// Failed to hotplug, due to Upcall not ready. + #[error("Upcall not ready, can't hotplug device.")] + UpcallNotReady, + /// The action `ConfigureBootSource` failed either because of bad user input or an internal /// error. #[error("failed to configure boot source for VM: {0}")] @@ -51,6 +59,11 @@ pub enum VmmActionError { /// The action `InsertVsockDevice` failed either because of bad user input or an internal error. #[error("failed to add virtio-vsock device: {0}")] Vsock(#[source] VsockDeviceError), + + #[cfg(feature = "virtio-blk")] + /// Block device related errors. + #[error("virtio-blk device error: {0}")] + Block(#[source] BlockDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -81,6 +94,20 @@ pub enum VmmAction { /// `VsockDeviceConfig` as input. This action can only be called before the microVM has /// booted. The response is sent using the `OutcomeSender`. InsertVsockDevice(VsockDeviceConfigInfo), + + #[cfg(feature = "virtio-blk")] + /// Add a new block device or update one that already exists using the `BlockDeviceConfig` as + /// input. This action can only be called before the microVM has booted. + InsertBlockDevice(BlockDeviceConfigInfo), + + #[cfg(feature = "virtio-blk")] + /// Remove a new block device for according to given drive_id + RemoveBlockDevice(String), + + #[cfg(feature = "virtio-blk")] + /// Update a block device, after microVM start. Currently, the only updatable properties + /// are the RX and TX rate limiters. + UpdateBlockDevice(BlockDeviceConfigUpdateInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -150,6 +177,18 @@ impl VmmService { } #[cfg(feature = "virtio-vsock")] VmmAction::InsertVsockDevice(vsock_cfg) => self.add_vsock_device(vmm, vsock_cfg), + #[cfg(feature = "virtio-blk")] + VmmAction::InsertBlockDevice(block_device_config) => { + self.add_block_device(vmm, event_mgr, block_device_config) + } + #[cfg(feature = "virtio-blk")] + VmmAction::UpdateBlockDevice(blk_update) => { + self.update_blk_rate_limiters(vmm, blk_update) + } + #[cfg(feature = "virtio-blk")] + VmmAction::RemoveBlockDevice(drive_id) => { + self.remove_block_device(vmm, event_mgr, &drive_id) + } }; debug!("send vmm response: {:?}", response); @@ -393,4 +432,67 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::Vsock) } + + #[cfg(feature = "virtio-blk")] + // Only call this function as part of the API. + // If the drive_id does not exist, a new Block Device Config is added to the list. + fn add_block_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: BlockDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicrovmError::UpcallNotReady = e { + return VmmActionError::UpcallNotReady; + } + VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot) + })?; + + BlockDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Block) + } + + #[cfg(feature = "virtio-blk")] + /// Updates configuration for an emulated net device as described in `config`. + fn update_blk_rate_limiters( + &mut self, + vmm: &mut Vmm, + config: BlockDeviceConfigUpdateInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Block) + } + + #[cfg(feature = "virtio-blk")] + // Only call this function as part of the API. + // If the drive_id does not exist, a new Block Device Config is added to the list. + fn remove_block_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + drive_id: &str, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|_| VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot))?; + + BlockDeviceMgr::remove_device(vm.device_manager_mut(), ctx, drive_id) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Block) + } } diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs new file mode 100644 index 000000000000..8a1b2d02fc88 --- /dev/null +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -0,0 +1,795 @@ +// Copyright 2020-2022 Alibaba, Inc. or its affiliates. All Rights Reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +//! Device manager for virtio-blk and vhost-user-blk devices. +use std::collections::{vec_deque, VecDeque}; +use std::convert::TryInto; +use std::fs::OpenOptions; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use dbs_virtio_devices as virtio; +use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, Ufile}; +use serde_derive::{Deserialize, Serialize}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo}; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::vm::KernelConfigInfo; + +use super::DbsMmioV2Device; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; + +macro_rules! info( + ($l:expr, $($args:tt)+) => { + slog::info!($l, $($args)+; slog::o!("subsystem" => "block_manager")) + }; +); + +macro_rules! error( + ($l:expr, $($args:tt)+) => { + slog::error!($l, $($args)+; slog::o!("subsystem" => "block_manager")) + }; +); + +macro_rules! get_bucket_update { + ($self:ident, $rate_limiter: ident, $metric: ident) => {{ + match &$self.$rate_limiter { + Some(rl_cfg) => { + let tb_cfg = &rl_cfg.$metric; + dbs_utils::rate_limiter::RateLimiter::make_bucket( + tb_cfg.size, + tb_cfg.one_time_burst, + tb_cfg.refill_time, + ) + // Updated active rate-limiter. + .map(dbs_utils::rate_limiter::BucketUpdate::Update) + // Updated/deactivated rate-limiter + .unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled) + } + // No update to the rate-limiter. + None => dbs_utils::rate_limiter::BucketUpdate::None, + } + }}; +} + +/// Default queue size for VirtIo block devices. +pub const QUEUE_SIZE: u16 = 128; + +/// Errors associated with the operations allowed on a drive. +#[derive(Debug, thiserror::Error)] +pub enum BlockDeviceError { + /// Invalid VM instance ID. + #[error("invalid VM instance id")] + InvalidVMID, + + /// The block device path is invalid. + #[error("invalid block device path '{0}'")] + InvalidBlockDevicePath(PathBuf), + + /// The block device type is invalid. + #[error("invalid block device type")] + InvalidBlockDeviceType, + + /// The block device path was already used for a different drive. + #[error("block device path '{0}' already exists")] + BlockDevicePathAlreadyExists(PathBuf), + + /// The device id doesn't exist. + #[error("invalid block device id '{0}'")] + InvalidDeviceId(String), + + /// Cannot perform the requested operation after booting the microVM. + #[error("block device does not support runtime update")] + UpdateNotAllowedPostBoot, + + /// A root block device was already added. + #[error("could not add multiple virtual machine root devices")] + RootBlockDeviceAlreadyAdded, + + /// Failed to send patch message to block epoll handler. + #[error("could not send patch message to the block epoll handler")] + BlockEpollHanderSendFail, + + /// Failure from device manager, + #[error("device manager errors: {0}")] + DeviceManager(#[from] DeviceMgrError), + + /// Failure from virtio subsystem. + #[error(transparent)] + Virtio(virtio::Error), + + /// Unable to seek the block device backing file due to invalid permissions or + /// the file was deleted/corrupted. + #[error("cannot create block device: {0}")] + CreateBlockDevice(#[source] virtio::Error), + + /// Cannot open the block device backing file. + #[error("cannot open the block device backing file: {0}")] + OpenBlockDevice(#[source] std::io::Error), + + /// Cannot initialize a MMIO Block Device or add a device to the MMIO Bus. + #[error("failure while registering block device: {0}")] + RegisterBlockDevice(#[source] DeviceMgrError), +} + +/// Type of low level storage device/protocol for virtio-blk devices. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub enum BlockDeviceType { + /// Unknown low level device type. + Unknown, + /// Vhost-user-blk based low level device. + /// SPOOL is a reliable NVMe virtualization system for the cloud environment. + /// You could learn more SPOOL here: https://www.usenix.org/conference/atc20/presentation/xue + Spool, + /// Local disk/file based low level device. + RawBlock, +} + +impl BlockDeviceType { + /// Get type of low level storage device/protocol by parsing `path`. + pub fn get_type(path: &str) -> BlockDeviceType { + // SPOOL path should be started with "spool", e.g. "spool:/device1" + if path.starts_with("spool:/") { + BlockDeviceType::Spool + } else { + BlockDeviceType::RawBlock + } + } +} + +/// Configuration information for a block device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct BlockDeviceConfigUpdateInfo { + /// Unique identifier of the drive. + pub drive_id: String, + /// Rate Limiter for I/O operations. + pub rate_limiter: Option, +} + +impl BlockDeviceConfigUpdateInfo { + /// Provides a `BucketUpdate` description for the bandwidth rate limiter. + pub fn bytes(&self) -> dbs_utils::rate_limiter::BucketUpdate { + get_bucket_update!(self, rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the ops rate limiter. + pub fn ops(&self) -> dbs_utils::rate_limiter::BucketUpdate { + get_bucket_update!(self, rate_limiter, ops) + } +} + +/// Configuration information for a block device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct BlockDeviceConfigInfo { + /// Unique identifier of the drive. + pub drive_id: String, + /// Type of low level storage/protocol. + pub device_type: BlockDeviceType, + /// Path of the drive. + pub path_on_host: PathBuf, + /// If set to true, it makes the current device the root block device. + /// Setting this flag to true will mount the block device in the + /// guest under /dev/vda unless the part_uuid is present. + pub is_root_device: bool, + /// Part-UUID. Represents the unique id of the boot partition of this device. + /// It is optional and it will be used only if the `is_root_device` field is true. + pub part_uuid: Option, + /// If set to true, the drive is opened in read-only mode. Otherwise, the + /// drive is opened as read-write. + pub is_read_only: bool, + /// If set to false, the drive is opened with buffered I/O mode. Otherwise, the + /// drive is opened with direct I/O mode. + pub is_direct: bool, + /// Don't close `path_on_host` file when dropping the device. + pub no_drop: bool, + /// Block device multi-queue + pub num_queues: usize, + /// Virtio queue size. + pub queue_size: u16, + /// Rate Limiter for I/O operations. + pub rate_limiter: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl std::default::Default for BlockDeviceConfigInfo { + fn default() -> Self { + Self { + drive_id: String::default(), + device_type: BlockDeviceType::RawBlock, + path_on_host: PathBuf::default(), + is_root_device: false, + part_uuid: None, + is_read_only: false, + is_direct: Self::default_direct(), + no_drop: Self::default_no_drop(), + num_queues: Self::default_num_queues(), + queue_size: 256, + rate_limiter: None, + use_shared_irq: None, + use_generic_irq: None, + } + } +} + +impl BlockDeviceConfigInfo { + /// Get default queue numbers + pub fn default_num_queues() -> usize { + 1 + } + + /// Get default value of is_direct switch + pub fn default_direct() -> bool { + true + } + + /// Get default value of no_drop switch + pub fn default_no_drop() -> bool { + false + } + + /// Get type of low level storage/protocol. + pub fn device_type(&self) -> BlockDeviceType { + self.device_type + } + + /// Returns a reference to `path_on_host`. + pub fn path_on_host(&self) -> &PathBuf { + &self.path_on_host + } + + /// Returns a reference to the part_uuid. + pub fn get_part_uuid(&self) -> Option<&String> { + self.part_uuid.as_ref() + } + + /// Checks whether the drive had read only permissions. + pub fn is_read_only(&self) -> bool { + self.is_read_only + } + + /// Checks whether the drive uses direct I/O + pub fn is_direct(&self) -> bool { + self.is_direct + } + + /// Get number and size of queues supported. + pub fn queue_sizes(&self) -> Vec { + (0..self.num_queues) + .map(|_| self.queue_size) + .collect::>() + } +} + +impl ConfigItem for BlockDeviceConfigInfo { + type Err = BlockDeviceError; + + fn id(&self) -> &str { + &self.drive_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BlockDeviceError> { + if self.drive_id == other.drive_id { + Ok(()) + } else if self.path_on_host == other.path_on_host { + Err(BlockDeviceError::BlockDevicePathAlreadyExists( + self.path_on_host.clone(), + )) + } else { + Ok(()) + } + } +} + +impl std::fmt::Debug for BlockDeviceInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.config) + } +} + +/// Block Device Info +pub type BlockDeviceInfo = DeviceConfigInfo; + +/// Wrapper for the collection that holds all the Block Devices Configs +//#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone)] +pub struct BlockDeviceMgr { + /// A list of `BlockDeviceInfo` objects. + info_list: VecDeque, + has_root_block: bool, + has_part_uuid_root: bool, + read_only_root: bool, + part_uuid: Option, + use_shared_irq: bool, +} + +impl BlockDeviceMgr { + /// returns a front-to-back iterator. + pub fn iter(&self) -> vec_deque::Iter { + self.info_list.iter() + } + + /// Checks whether any of the added BlockDevice is the root. + pub fn has_root_block_device(&self) -> bool { + self.has_root_block + } + + /// Checks whether the root device is configured using a part UUID. + pub fn has_part_uuid_root(&self) -> bool { + self.has_part_uuid_root + } + + /// Checks whether the root device has read-only permisssions. + pub fn is_read_only_root(&self) -> bool { + self.read_only_root + } + + /// Gets the index of the device with the specified `drive_id` if it exists in the list. + pub fn get_index_of_drive_id(&self, id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.id().eq(id)) + } + + /// Gets the 'BlockDeviceConfigInfo' of the device with the specified `drive_id` if it exists in the list. + pub fn get_config_of_drive_id(&self, drive_id: &str) -> Option { + match self.get_index_of_drive_id(drive_id) { + Some(index) => { + let config = self.info_list.get(index).unwrap().config.clone(); + Some(config) + } + None => None, + } + } + + /// Inserts `block_device_config` in the block device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + /// Inserting a secondary root block device will fail. + pub fn insert_device( + device_mgr: &mut DeviceManager, + mut ctx: DeviceOpContext, + config: BlockDeviceConfigInfo, + ) -> std::result::Result<(), BlockDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + return Err(BlockDeviceError::UpdateNotAllowedPostBoot); + } + + let mgr = &mut device_mgr.block_manager; + + // If the id of the drive already exists in the list, the operation is update. + match mgr.get_index_of_drive_id(config.id()) { + Some(index) => { + // No support for runtime update yet. + if ctx.is_hotplug { + Err(BlockDeviceError::BlockDevicePathAlreadyExists( + config.path_on_host.clone(), + )) + } else { + for (idx, info) in mgr.info_list.iter().enumerate() { + if idx != index { + info.config.check_conflicts(&config)?; + } + } + mgr.update(index, config) + } + } + None => { + for info in mgr.info_list.iter() { + info.config.check_conflicts(&config)?; + } + let config2 = config.clone(); + let index = mgr.create(config2)?; + if !ctx.is_hotplug { + return Ok(()); + } + + match config.device_type { + BlockDeviceType::RawBlock => { + let device = Self::create_blk_device(&config, &mut ctx) + .map_err(BlockDeviceError::Virtio)?; + let dev = DeviceManager::create_mmio_virtio_device( + device, + &mut ctx, + config.use_shared_irq.unwrap_or(mgr.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BlockDeviceError::DeviceManager)?; + mgr.update_device_by_index(index, Arc::clone(&dev))?; + // live-upgrade need save/restore device from info.device. + mgr.info_list[index].set_device(dev.clone()); + ctx.insert_hotplug_mmio_device(&dev, None).map_err(|e| { + let logger = ctx.logger().new(slog::o!()); + BlockDeviceMgr::remove_device(device_mgr, ctx, &config.drive_id) + .unwrap(); + error!( + logger, + "failed to hot-add virtio block device {}, {:?}", + &config.drive_id, + e + ); + BlockDeviceError::DeviceManager(e) + }) + } + _ => Err(BlockDeviceError::InvalidBlockDeviceType), + } + } + } + } + + /// Attaches all block devices from the BlockDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), BlockDeviceError> { + for info in self.info_list.iter_mut() { + match info.config.device_type { + BlockDeviceType::RawBlock => { + info!( + ctx.logger(), + "attach virtio-blk device, drive_id {}, path {}", + info.config.drive_id, + info.config.path_on_host.to_str().unwrap_or("") + ); + let device = Self::create_blk_device(&info.config, ctx) + .map_err(BlockDeviceError::Virtio)?; + let device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BlockDeviceError::RegisterBlockDevice)?; + info.device = Some(device); + } + _ => { + return Err(BlockDeviceError::OpenBlockDevice( + std::io::Error::from_raw_os_error(libc::EINVAL), + )); + } + } + } + + Ok(()) + } + + /// Removes all virtio-blk devices + pub fn remove_devices(&mut self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + while let Some(mut info) = self.info_list.pop_back() { + info!(ctx.logger(), "remove drive {}", info.config.drive_id); + if let Some(device) = info.device.take() { + DeviceManager::destroy_mmio_virtio_device(device, ctx)?; + } + } + + Ok(()) + } + + fn remove(&mut self, drive_id: &str) -> Option { + match self.get_index_of_drive_id(drive_id) { + Some(index) => self.info_list.remove(index), + None => None, + } + } + + /// remove a block device, it basically is the inverse operation of `insert_device`` + pub fn remove_device( + dev_mgr: &mut DeviceManager, + mut ctx: DeviceOpContext, + drive_id: &str, + ) -> std::result::Result<(), BlockDeviceError> { + if !cfg!(feature = "hotplug") { + return Err(BlockDeviceError::UpdateNotAllowedPostBoot); + } + + let mgr = &mut dev_mgr.block_manager; + match mgr.remove(drive_id) { + Some(mut info) => { + info!(ctx.logger(), "remove drive {}", info.config.drive_id); + if let Some(device) = info.device.take() { + DeviceManager::destroy_mmio_virtio_device(device, &mut ctx) + .map_err(BlockDeviceError::DeviceManager)?; + } + } + None => return Err(BlockDeviceError::InvalidDeviceId(drive_id.to_owned())), + } + + Ok(()) + } + + fn create_blk_device( + cfg: &BlockDeviceConfigInfo, + ctx: &mut DeviceOpContext, + ) -> std::result::Result>, virtio::Error> { + let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?; + + // Safe to unwrap() because we have verified it when parsing device type. + //let path = cfg.path_on_host.to_str().unwrap(); + let mut block_files: Vec> = vec![]; + + match cfg.device_type { + BlockDeviceType::RawBlock => { + let custom_flags = if cfg.is_direct() { + info!( + ctx.logger(), + "Open block device \"{}\" in direct mode.", + cfg.path_on_host().display() + ); + libc::O_DIRECT + } else { + info!( + ctx.logger(), + "Open block device \"{}\" in buffer mode.", + cfg.path_on_host().display(), + ); + 0 + }; + let io_uring_supported = IoUring::is_supported(); + for i in 0..cfg.num_queues { + let queue_size = cfg.queue_sizes()[i] as u32; + let file = OpenOptions::new() + .read(true) + .custom_flags(custom_flags) + .write(!cfg.is_read_only()) + .open(cfg.path_on_host())?; + info!(ctx.logger(), "Queue {}: block file opened", i); + + if io_uring_supported { + info!( + ctx.logger(), + "Queue {}: Using io_uring Raw disk file, queue size {}.", i, queue_size + ); + let io_engine = IoUring::new(file.as_raw_fd(), queue_size)?; + block_files.push(Box::new(LocalFile::new(file, cfg.no_drop, io_engine)?)); + } else { + info!( + ctx.logger(), + "Queue {}: Since io_uring_supported is not enabled, change to default support of Aio Raw disk file, queue size {}", i, queue_size + ); + let io_engine = Aio::new(file.as_raw_fd(), queue_size)?; + block_files.push(Box::new(LocalFile::new(file, cfg.no_drop, io_engine)?)); + } + } + } + _ => { + error!( + ctx.logger(), + "invalid block device type: {:?}", cfg.device_type + ); + return Err(virtio::Error::InvalidInput); + } + }; + + let mut limiters = vec![]; + for _i in 0..cfg.num_queues { + if let Some(limiter) = cfg.rate_limiter.clone().map(|mut v| { + v.resize(cfg.num_queues as u64); + v.try_into().unwrap() + }) { + limiters.push(limiter); + } + } + + Ok(Box::new(Block::new( + block_files, + cfg.is_read_only, + Arc::new(cfg.queue_sizes()), + epoll_mgr, + limiters, + )?)) + } + + /// Generated guest kernel commandline related to root block device. + pub fn generate_kernel_boot_args( + &self, + kernel_config: &mut KernelConfigInfo, + ) -> std::result::Result<(), DeviceMgrError> { + // Respect user configuration if kernel_cmdline contains "root=", + // special attention for the case when kernel command line starting with "root=xxx" + let old_kernel_cmdline = format!(" {}", kernel_config.kernel_cmdline().as_str()); + if !old_kernel_cmdline.contains(" root=") && self.has_root_block { + let cmdline = kernel_config.kernel_cmdline_mut(); + if let Some(ref uuid) = self.part_uuid { + cmdline + .insert("root", &format!("PART_UUID={}", uuid)) + .map_err(DeviceMgrError::Cmdline)?; + } else { + cmdline + .insert("root", "/dev/vda") + .map_err(DeviceMgrError::Cmdline)?; + } + if self.read_only_root { + if old_kernel_cmdline.contains(" rw") { + return Err(DeviceMgrError::InvalidOperation); + } + cmdline.insert_str("ro").map_err(DeviceMgrError::Cmdline)?; + } + } + + Ok(()) + } + + /// insert a block device's config. return index on success. + fn create( + &mut self, + block_device_config: BlockDeviceConfigInfo, + ) -> std::result::Result { + self.check_data_file_present(&block_device_config)?; + if self + .get_index_of_drive_path(&block_device_config.path_on_host) + .is_some() + { + return Err(BlockDeviceError::BlockDevicePathAlreadyExists( + block_device_config.path_on_host, + )); + } + + // check whether the Device Config belongs to a root device + // we need to satisfy the condition by which a VMM can only have on root device + if block_device_config.is_root_device { + if self.has_root_block { + return Err(BlockDeviceError::RootBlockDeviceAlreadyAdded); + } else { + self.has_root_block = true; + self.read_only_root = block_device_config.is_read_only; + self.has_part_uuid_root = block_device_config.part_uuid.is_some(); + self.part_uuid = block_device_config.part_uuid.clone(); + // Root Device should be the first in the list whether or not PART_UUID is specified + // in order to avoid bugs in case of switching from part_uuid boot scenarios to + // /dev/vda boot type. + self.info_list + .push_front(BlockDeviceInfo::new(block_device_config)); + Ok(0) + } + } else { + self.info_list + .push_back(BlockDeviceInfo::new(block_device_config)); + Ok(self.info_list.len() - 1) + } + } + + /// Updates a Block Device Config. The update fails if it would result in two + /// root block devices. + fn update( + &mut self, + mut index: usize, + new_config: BlockDeviceConfigInfo, + ) -> std::result::Result<(), BlockDeviceError> { + // Check if the path exists + self.check_data_file_present(&new_config)?; + if let Some(idx) = self.get_index_of_drive_path(&new_config.path_on_host) { + if idx != index { + return Err(BlockDeviceError::BlockDevicePathAlreadyExists( + new_config.path_on_host.clone(), + )); + } + } + + if self.info_list.get(index).is_none() { + return Err(InvalidDeviceId(index.to_string())); + } + // Check if the root block device is being updated. + if self.info_list[index].config.is_root_device { + self.has_root_block = new_config.is_root_device; + self.read_only_root = new_config.is_root_device && new_config.is_read_only; + self.has_part_uuid_root = new_config.part_uuid.is_some(); + self.part_uuid = new_config.part_uuid.clone(); + } else if new_config.is_root_device { + // Check if a second root block device is being added. + if self.has_root_block { + return Err(BlockDeviceError::RootBlockDeviceAlreadyAdded); + } else { + // One of the non-root blocks is becoming root. + self.has_root_block = true; + self.read_only_root = new_config.is_read_only; + self.has_part_uuid_root = new_config.part_uuid.is_some(); + self.part_uuid = new_config.part_uuid.clone(); + + // Make sure the root device is on the first position. + self.info_list.swap(0, index); + // Block config to be updated has moved to first position. + index = 0; + } + } + // Update the config. + self.info_list[index].config = new_config; + + Ok(()) + } + + fn check_data_file_present( + &self, + block_device_config: &BlockDeviceConfigInfo, + ) -> std::result::Result<(), BlockDeviceError> { + if block_device_config.device_type == BlockDeviceType::RawBlock + && !block_device_config.path_on_host.exists() + { + Err(BlockDeviceError::InvalidBlockDevicePath( + block_device_config.path_on_host.clone(), + )) + } else { + Ok(()) + } + } + + fn get_index_of_drive_path(&self, drive_path: &Path) -> Option { + self.info_list + .iter() + .position(|info| info.config.path_on_host.eq(drive_path)) + } + + /// update devce information in `info_list`. The caller of this method is + /// `insert_device` when hotplug is true. + pub fn update_device_by_index( + &mut self, + index: usize, + device: Arc, + ) -> Result<(), BlockDeviceError> { + if let Some(info) = self.info_list.get_mut(index) { + info.device = Some(device); + return Ok(()); + } + + Err(BlockDeviceError::InvalidDeviceId("".to_owned())) + } + + /// Update the ratelimiter settings of a virtio blk device. + pub fn update_device_ratelimiters( + device_mgr: &mut DeviceManager, + new_cfg: BlockDeviceConfigUpdateInfo, + ) -> std::result::Result<(), BlockDeviceError> { + let mgr = &mut device_mgr.block_manager; + match mgr.get_index_of_drive_id(&new_cfg.drive_id) { + Some(index) => { + let config = &mut mgr.info_list[index].config; + config.rate_limiter = new_cfg.rate_limiter.clone(); + let device = mgr.info_list[index] + .device + .as_mut() + .ok_or_else(|| BlockDeviceError::InvalidDeviceId("".to_owned()))?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(blk_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return blk_dev + .set_patch_rate_limiters(new_cfg.bytes(), new_cfg.ops()) + .map(|_p| ()) + .map_err(|_e| BlockDeviceError::BlockEpollHanderSendFail); + } + } + Ok(()) + } + None => Err(BlockDeviceError::InvalidDeviceId(new_cfg.drive_id)), + } + } +} + +impl Default for BlockDeviceMgr { + /// Constructor for the BlockDeviceConfigs. It initializes an empty LinkedList. + fn default() -> BlockDeviceMgr { + BlockDeviceMgr { + info_list: VecDeque::::new(), + has_root_block: false, + has_part_uuid_root: false, + read_only_root: false, + part_uuid: None, + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 07698d4b887d..11a742bbbab2 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -59,6 +59,12 @@ pub mod vsock_dev_mgr; #[cfg(feature = "virtio-vsock")] use self::vsock_dev_mgr::VsockDeviceMgr; +#[cfg(feature = "virtio-blk")] +/// virtio-block device manager +pub mod blk_dev_mgr; +#[cfg(feature = "virtio-blk")] +use self::blk_dev_mgr::BlockDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -71,21 +77,27 @@ pub enum DeviceMgrError { /// Invalid operation. #[error("invalid device manager operation")] InvalidOperation, + /// Failed to get device resource. #[error("failed to get device assigned resources")] GetDeviceResource, + /// Appending to kernel command line failed. #[error("failed to add kernel command line parameter for device: {0}")] Cmdline(#[source] linux_loader::cmdline::Error), + /// Failed to manage console devices. #[error(transparent)] ConsoleManager(console_manager::ConsoleManagerError), + /// Failed to create the device. #[error("failed to create virtual device: {0}")] CreateDevice(#[source] io::Error), + /// Failed to perform an operation on the bus. #[error(transparent)] IoManager(IoManagerError), + /// Failure from legacy device manager. #[error(transparent)] LegacyManager(legacy::Error), @@ -409,6 +421,11 @@ pub struct DeviceManager { pub(crate) mmio_device_info: HashMap<(DeviceType, String), MMIODeviceInfo>, #[cfg(feature = "virtio-vsock")] pub(crate) vsock_manager: VsockDeviceMgr, + + #[cfg(feature = "virtio-blk")] + // If there is a Root Block Device, this should be added as the first element of the list. + // This is necessary because we want the root to always be mounted on /dev/vda. + pub(crate) block_manager: BlockDeviceMgr, } impl DeviceManager { @@ -433,6 +450,8 @@ impl DeviceManager { mmio_device_info: HashMap::new(), #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), + #[cfg(feature = "virtio-blk")] + block_manager: BlockDeviceMgr::default(), } } @@ -553,9 +572,18 @@ impl DeviceManager { self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; + #[cfg(feature = "virtio-blk")] + self.block_manager + .attach_devices(&mut ctx) + .map_err(StartMicrovmError::BlockDeviceError)?; + #[cfg(feature = "virtio-vsock")] self.vsock_manager.attach_devices(&mut ctx)?; + #[cfg(feature = "virtio-blk")] + self.block_manager + .generate_kernel_boot_args(kernel_config) + .map_err(StartMicrovmError::DeviceManager)?; ctx.generate_kernel_boot_args(kernel_config) .map_err(StartMicrovmError::DeviceManager)?; @@ -570,10 +598,21 @@ impl DeviceManager { /// Remove all devices when shutdown the associated virtual machine pub fn remove_devices( &mut self, - _vm_as: GuestAddressSpaceImpl, - _epoll_mgr: EpollManager, - _address_space: Option<&AddressSpace>, + vm_as: GuestAddressSpaceImpl, + epoll_mgr: EpollManager, + address_space: Option<&AddressSpace>, ) -> Result<()> { + // create context for removing devices + let mut ctx = DeviceOpContext::new( + Some(epoll_mgr), + self, + Some(vm_as), + address_space.cloned(), + true, + ); + + #[cfg(feature = "virtio-blk")] + self.block_manager.remove_devices(&mut ctx)?; Ok(()) } } diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index f6b37c867d88..9c9c46564c43 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -158,6 +158,11 @@ pub enum StartMicroVmError { /// Upcall connect Error. #[error("failure while connecting the upcall client: {0}")] UpcallConnectError(#[source] dbs_upcall::UpcallClientError), + + #[cfg(feature = "virtio-blk")] + /// Virtio-blk errors. + #[error("virtio-blk errors: {0}")] + BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError), } /// Errors associated with starting the instance. From 948381bdbee2bd2589f40702d688d353ba325729 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 18:08:24 +0800 Subject: [PATCH 0101/1953] dragonball: add virtio-net device support Virtio-net devices are supported. Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 1 + src/dragonball/src/api/v1/vmm_action.rs | 71 ++++ src/dragonball/src/config_manager.rs | 22 + .../src/device_manager/blk_dev_mgr.rs | 25 +- src/dragonball/src/device_manager/mod.rs | 16 + .../src/device_manager/virtio_net_dev_mgr.rs | 386 ++++++++++++++++++ src/dragonball/src/error.rs | 5 + 7 files changed, 504 insertions(+), 22 deletions(-) create mode 100644 src/dragonball/src/device_manager/virtio_net_dev_mgr.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index fcedd492ced6..d44e115ca52e 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -50,6 +50,7 @@ atomic-guest-memory = [] hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] +virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] [patch.'crates-io'] dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 253a6854ce6e..0a021341ac4f 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -21,6 +21,11 @@ use crate::vmm::Vmm; use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, }; +#[cfg(feature = "virtio-net")] +use crate::device_manager::virtio_net_dev_mgr::{ + VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, + VirtioNetDeviceMgr, +}; #[cfg(feature = "virtio-vsock")] use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; @@ -64,6 +69,11 @@ pub enum VmmActionError { /// Block device related errors. #[error("virtio-blk device error: {0}")] Block(#[source] BlockDeviceError), + + #[cfg(feature = "virtio-net")] + /// Net device related errors. + #[error("virtio-net device error: {0}")] + VirtioNet(#[source] VirtioNetDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -108,6 +118,17 @@ pub enum VmmAction { /// Update a block device, after microVM start. Currently, the only updatable properties /// are the RX and TX rate limiters. UpdateBlockDevice(BlockDeviceConfigUpdateInfo), + + #[cfg(feature = "virtio-net")] + /// Add a new network interface config or update one that already exists using the + /// `NetworkInterfaceConfig` as input. This action can only be called before the microVM has + /// booted. The response is sent using the `OutcomeSender`. + InsertNetworkDevice(VirtioNetDeviceConfigInfo), + + #[cfg(feature = "virtio-net")] + /// Update a network interface, after microVM start. Currently, the only updatable properties + /// are the RX and TX rate limiters. + UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -189,6 +210,14 @@ impl VmmService { VmmAction::RemoveBlockDevice(drive_id) => { self.remove_block_device(vmm, event_mgr, &drive_id) } + #[cfg(feature = "virtio-net")] + VmmAction::InsertNetworkDevice(virtio_net_cfg) => { + self.add_virtio_net_device(vmm, event_mgr, virtio_net_cfg) + } + #[cfg(feature = "virtio-net")] + VmmAction::UpdateNetworkInterface(netif_update) => { + self.update_net_rate_limiters(vmm, netif_update) + } }; debug!("send vmm response: {:?}", response); @@ -495,4 +524,46 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::Block) } + + #[cfg(feature = "virtio-net")] + fn add_virtio_net_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: VirtioNetDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicrovmError::MicroVMAlreadyRunning = e { + VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot) + } else if let StartMicrovmError::UpcallNotReady = e { + VmmActionError::UpcallNotReady + } else { + VmmActionError::StartMicrovm(e) + } + })?; + + VirtioNetDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::VirtioNet) + } + + #[cfg(feature = "virtio-net")] + fn update_net_rate_limiters( + &mut self, + vmm: &mut Vmm, + config: VirtioNetDeviceConfigUpdateInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::VirtioNet) + } } diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs index e25b792e7bff..cbfb790413c5 100644 --- a/src/dragonball/src/config_manager.rs +++ b/src/dragonball/src/config_manager.rs @@ -10,6 +10,28 @@ use dbs_device::DeviceIo; use dbs_utils::rate_limiter::{RateLimiter, TokenBucket}; use serde_derive::{Deserialize, Serialize}; +macro_rules! get_bucket_update { + ($self:ident, $rate_limiter: ident, $metric: ident) => {{ + match &$self.$rate_limiter { + Some(rl_cfg) => { + let tb_cfg = &rl_cfg.$metric; + dbs_utils::rate_limiter::RateLimiter::make_bucket( + tb_cfg.size, + tb_cfg.one_time_burst, + tb_cfg.refill_time, + ) + // Updated active rate-limiter. + .map(dbs_utils::rate_limiter::BucketUpdate::Update) + // Updated/deactivated rate-limiter + .unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled) + } + // No update to the rate-limiter. + None => dbs_utils::rate_limiter::BucketUpdate::None, + } + }}; +} +pub(crate) use get_bucket_update; + /// Trait for generic configuration information. pub trait ConfigItem { /// Related errors. diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 8a1b2d02fc88..38e46cd59486 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -20,7 +20,9 @@ use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, U use serde_derive::{Deserialize, Serialize}; use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo}; +use crate::config_manager::{ + get_bucket_update, ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo, +}; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; use crate::vm::KernelConfigInfo; @@ -43,27 +45,6 @@ macro_rules! error( }; ); -macro_rules! get_bucket_update { - ($self:ident, $rate_limiter: ident, $metric: ident) => {{ - match &$self.$rate_limiter { - Some(rl_cfg) => { - let tb_cfg = &rl_cfg.$metric; - dbs_utils::rate_limiter::RateLimiter::make_bucket( - tb_cfg.size, - tb_cfg.one_time_burst, - tb_cfg.refill_time, - ) - // Updated active rate-limiter. - .map(dbs_utils::rate_limiter::BucketUpdate::Update) - // Updated/deactivated rate-limiter - .unwrap_or(dbs_utils::rate_limiter::BucketUpdate::Disabled) - } - // No update to the rate-limiter. - None => dbs_utils::rate_limiter::BucketUpdate::None, - } - }}; -} - /// Default queue size for VirtIo block devices. pub const QUEUE_SIZE: u16 = 128; diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 11a742bbbab2..9313919f85ae 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -65,6 +65,12 @@ pub mod blk_dev_mgr; #[cfg(feature = "virtio-blk")] use self::blk_dev_mgr::BlockDeviceMgr; +#[cfg(feature = "virtio-net")] +/// Device manager for virtio-net devices. +pub mod virtio_net_dev_mgr; +#[cfg(feature = "virtio-net")] +use self::virtio_net_dev_mgr::VirtioNetDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -426,6 +432,9 @@ pub struct DeviceManager { // If there is a Root Block Device, this should be added as the first element of the list. // This is necessary because we want the root to always be mounted on /dev/vda. pub(crate) block_manager: BlockDeviceMgr, + + #[cfg(feature = "virtio-net")] + pub(crate) virtio_net_manager: VirtioNetDeviceMgr, } impl DeviceManager { @@ -452,6 +461,8 @@ impl DeviceManager { vsock_manager: VsockDeviceMgr::default(), #[cfg(feature = "virtio-blk")] block_manager: BlockDeviceMgr::default(), + #[cfg(feature = "virtio-net")] + virtio_net_manager: VirtioNetDeviceMgr::default(), } } @@ -577,6 +588,11 @@ impl DeviceManager { .attach_devices(&mut ctx) .map_err(StartMicrovmError::BlockDeviceError)?; + #[cfg(feature = "virtio-net")] + self.virtio_net_manager + .attach_devices(&mut ctx) + .map_err(StartMicrovmError::VirtioNetDeviceError)?; + #[cfg(feature = "virtio-vsock")] self.vsock_manager.attach_devices(&mut ctx)?; diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs new file mode 100644 index 000000000000..65f29fbcd048 --- /dev/null +++ b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs @@ -0,0 +1,386 @@ +// Copyright 2020-2022 Alibaba, Inc. or its affiliates. All Rights Reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Portions Copyright 2017 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the THIRD-PARTY file. + +use std::convert::TryInto; +use std::sync::Arc; + +use dbs_utils::net::{MacAddr, Tap, TapError}; +use dbs_utils::rate_limiter::BucketUpdate; +use dbs_virtio_devices as virtio; +use dbs_virtio_devices::net::Net; +use dbs_virtio_devices::Error as VirtioError; +use serde_derive::{Deserialize, Serialize}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ + get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, +}; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; + +use super::DbsMmioV2Device; + +/// Default number of virtio queues, one rx/tx pair. +pub const NUM_QUEUES: usize = 2; +/// Default size of virtio queues. +pub const QUEUE_SIZE: u16 = 256; +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; + +/// Errors associated with virtio net device operations. +#[derive(Debug, thiserror::Error)] +pub enum VirtioNetDeviceError { + /// The virtual machine instance ID is invalid. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + + /// The iface ID is invalid. + #[error("invalid virtio-net iface id '{0}'")] + InvalidIfaceId(String), + + /// Invalid queue number configuration for virtio_net device. + #[error("invalid queue number {0} for virtio-net device")] + InvalidQueueNum(usize), + + /// Failure from device manager, + #[error("failure in device manager operations, {0}")] + DeviceManager(#[source] DeviceMgrError), + + /// The Context Identifier is already in use. + #[error("the device ID {0} already exists")] + DeviceIDAlreadyExist(String), + + /// The MAC address is already in use. + #[error("the guest MAC address {0} is already in use")] + GuestMacAddressInUse(String), + + /// The host device name is already in use. + #[error("the host device name {0} is already in use")] + HostDeviceNameInUse(String), + + /// Cannot open/create tap device. + #[error("cannot open TAP device")] + OpenTap(#[source] TapError), + + /// Failure from virtio subsystem. + #[error(transparent)] + Virtio(VirtioError), + + /// Failed to send patch message to net epoll handler. + #[error("could not send patch message to the net epoll handler")] + NetEpollHanderSendFail, + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// Split this at some point. + /// Internal errors are due to resource exhaustion. + /// Users errors are due to invalid permissions. + #[error("cannot create network device: {0}")] + CreateNetDevice(#[source] VirtioError), + + /// Cannot initialize a MMIO Network Device or add a device to the MMIO Bus. + #[error("failure while registering network device: {0}")] + RegisterNetDevice(#[source] DeviceMgrError), +} + +/// Configuration information for virtio net devices. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct VirtioNetDeviceConfigUpdateInfo { + /// ID of the guest network interface. + pub iface_id: String, + /// Rate Limiter for received packages. + pub rx_rate_limiter: Option, + /// Rate Limiter for transmitted packages. + pub tx_rate_limiter: Option, +} + +impl VirtioNetDeviceConfigUpdateInfo { + /// Provides a `BucketUpdate` description for the RX bandwidth rate limiter. + pub fn rx_bytes(&self) -> BucketUpdate { + get_bucket_update!(self, rx_rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the RX ops rate limiter. + pub fn rx_ops(&self) -> BucketUpdate { + get_bucket_update!(self, rx_rate_limiter, ops) + } + /// Provides a `BucketUpdate` description for the TX bandwidth rate limiter. + pub fn tx_bytes(&self) -> BucketUpdate { + get_bucket_update!(self, tx_rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the TX ops rate limiter. + pub fn tx_ops(&self) -> BucketUpdate { + get_bucket_update!(self, tx_rate_limiter, ops) + } +} + +/// Configuration information for virtio net devices. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, Default)] +pub struct VirtioNetDeviceConfigInfo { + /// ID of the guest network interface. + pub iface_id: String, + /// Host level path for the guest network interface. + pub host_dev_name: String, + /// Number of virtqueues to use. + pub num_queues: usize, + /// Size of each virtqueue. Unit: byte. + pub queue_size: u16, + /// Guest MAC address. + pub guest_mac: Option, + /// Rate Limiter for received packages. + pub rx_rate_limiter: Option, + /// Rate Limiter for transmitted packages. + pub tx_rate_limiter: Option, + /// allow duplicate mac + pub allow_duplicate_mac: bool, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl VirtioNetDeviceConfigInfo { + /// Returns the tap device that `host_dev_name` refers to. + pub fn open_tap(&self) -> std::result::Result { + Tap::open_named(self.host_dev_name.as_str(), false).map_err(VirtioNetDeviceError::OpenTap) + } + + /// Returns a reference to the mac address. It the mac address is not configured, it + /// return None. + pub fn guest_mac(&self) -> Option<&MacAddr> { + self.guest_mac.as_ref() + } + + ///Rx and Tx queue and max queue sizes + pub fn queue_sizes(&self) -> Vec { + let mut queue_size = self.queue_size; + if queue_size == 0 { + queue_size = QUEUE_SIZE; + } + let num_queues = if self.num_queues > 0 { + self.num_queues + } else { + NUM_QUEUES + }; + + (0..num_queues).map(|_| queue_size).collect::>() + } +} + +impl ConfigItem for VirtioNetDeviceConfigInfo { + type Err = VirtioNetDeviceError; + + fn id(&self) -> &str { + &self.iface_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), VirtioNetDeviceError> { + if self.iface_id == other.iface_id { + Err(VirtioNetDeviceError::DeviceIDAlreadyExist( + self.iface_id.clone(), + )) + } else if !other.allow_duplicate_mac + && self.guest_mac.is_some() + && self.guest_mac == other.guest_mac + { + Err(VirtioNetDeviceError::GuestMacAddressInUse( + self.guest_mac.as_ref().unwrap().to_string(), + )) + } else if self.host_dev_name == other.host_dev_name { + Err(VirtioNetDeviceError::HostDeviceNameInUse( + self.host_dev_name.clone(), + )) + } else { + Ok(()) + } + } +} + +/// Virtio Net Device Info +pub type VirtioNetDeviceInfo = DeviceConfigInfo; + +/// Device manager to manage all virtio net devices. +pub struct VirtioNetDeviceMgr { + pub(crate) info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl VirtioNetDeviceMgr { + /// Gets the index of the device with the specified `drive_id` if it exists in the list. + pub fn get_index_of_iface_id(&self, if_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.iface_id.eq(if_id)) + } + + /// Insert or update a virtio net device into the manager. + pub fn insert_device( + device_mgr: &mut DeviceManager, + mut ctx: DeviceOpContext, + config: VirtioNetDeviceConfigInfo, + ) -> std::result::Result<(), VirtioNetDeviceError> { + if config.num_queues % 2 != 0 { + return Err(VirtioNetDeviceError::InvalidQueueNum(config.num_queues)); + } + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + return Err(VirtioNetDeviceError::UpdateNotAllowedPostBoot); + } + + let mgr = &mut device_mgr.virtio_net_manager; + + slog::info!( + ctx.logger(), + "add virtio-net device configuration"; + "subsystem" => "net_dev_mgr", + "id" => &config.iface_id, + "host_dev_name" => &config.host_dev_name, + ); + + let device_index = mgr.info_list.insert_or_update(&config)?; + + if ctx.is_hotplug { + slog::info!( + ctx.logger(), + "attach virtio-net device"; + "subsystem" => "net_dev_mgr", + "id" => &config.iface_id, + "host_dev_name" => &config.host_dev_name, + ); + + match Self::create_device(&config, &mut ctx) { + Ok(device) => { + let dev = DeviceManager::create_mmio_virtio_device( + device, + &mut ctx, + config.use_shared_irq.unwrap_or(mgr.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(VirtioNetDeviceError::DeviceManager)?; + ctx.insert_hotplug_mmio_device(&dev.clone(), None) + .map_err(VirtioNetDeviceError::DeviceManager)?; + // live-upgrade need save/restore device from info.device. + mgr.info_list[device_index].set_device(dev); + } + Err(e) => { + mgr.info_list.remove(device_index); + return Err(VirtioNetDeviceError::Virtio(e)); + } + } + } + + Ok(()) + } + + /// Update the ratelimiter settings of a virtio net device. + pub fn update_device_ratelimiters( + device_mgr: &mut DeviceManager, + new_cfg: VirtioNetDeviceConfigUpdateInfo, + ) -> std::result::Result<(), VirtioNetDeviceError> { + let mgr = &mut device_mgr.virtio_net_manager; + match mgr.get_index_of_iface_id(&new_cfg.iface_id) { + Some(index) => { + let config = &mut mgr.info_list[index].config; + config.rx_rate_limiter = new_cfg.rx_rate_limiter.clone(); + config.tx_rate_limiter = new_cfg.tx_rate_limiter.clone(); + let device = mgr.info_list[index].device.as_mut().ok_or_else(|| { + VirtioNetDeviceError::InvalidIfaceId(new_cfg.iface_id.clone()) + })?; + + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(net_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return net_dev + .set_patch_rate_limiters( + new_cfg.rx_bytes(), + new_cfg.rx_ops(), + new_cfg.tx_bytes(), + new_cfg.tx_ops(), + ) + .map(|_p| ()) + .map_err(|_e| VirtioNetDeviceError::NetEpollHanderSendFail); + } + } + Ok(()) + } + None => Err(VirtioNetDeviceError::InvalidIfaceId( + new_cfg.iface_id.clone(), + )), + } + } + + /// Attach all configured vsock device to the virtual machine instance. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), VirtioNetDeviceError> { + for info in self.info_list.iter_mut() { + slog::info!( + ctx.logger(), + "attach virtio-net device"; + "subsystem" => "net_dev_mgr", + "id" => &info.config.iface_id, + "host_dev_name" => &info.config.host_dev_name, + ); + + let device = Self::create_device(&info.config, ctx) + .map_err(VirtioNetDeviceError::CreateNetDevice)?; + let device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(VirtioNetDeviceError::RegisterNetDevice)?; + info.set_device(device); + } + + Ok(()) + } + + fn create_device( + cfg: &VirtioNetDeviceConfigInfo, + ctx: &mut DeviceOpContext, + ) -> std::result::Result>, virtio::Error> { + let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?; + let rx_rate_limiter = match cfg.rx_rate_limiter.as_ref() { + Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?), + None => None, + }; + let tx_rate_limiter = match cfg.tx_rate_limiter.as_ref() { + Some(rl) => Some(rl.try_into().map_err(virtio::Error::IOError)?), + None => None, + }; + + let net_device = Net::new( + cfg.host_dev_name.clone(), + cfg.guest_mac(), + Arc::new(cfg.queue_sizes()), + epoll_mgr, + rx_rate_limiter, + tx_rate_limiter, + )?; + + Ok(Box::new(net_device)) + } +} + +impl Default for VirtioNetDeviceMgr { + /// Create a new virtio net device manager. + fn default() -> Self { + VirtioNetDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 9c9c46564c43..667405c5ecd8 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -163,6 +163,11 @@ pub enum StartMicroVmError { /// Virtio-blk errors. #[error("virtio-blk errors: {0}")] BlockDeviceError(#[source] device_manager::blk_dev_mgr::BlockDeviceError), + + #[cfg(feature = "virtio-net")] + /// Virtio-net errors. + #[error("virtio-net errors: {0}")] + VirtioNetDeviceError(#[source] device_manager::virtio_net_dev_mgr::VirtioNetDeviceError), } /// Errors associated with starting the instance. From 11b3f95140c7d840e0cac66355fa22c92b2777b5 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 21:24:09 +0800 Subject: [PATCH 0102/1953] dragonball: add virtio-fs device support Virtio-fs devices are supported. Fixes: #4257 Signed-off-by: wllenyj --- src/dragonball/Cargo.toml | 2 + src/dragonball/src/api/v1/vmm_action.rs | 95 ++++ .../src/device_manager/blk_dev_mgr.rs | 2 +- .../src/device_manager/fs_dev_mgr.rs | 523 ++++++++++++++++++ .../device_manager/memory_region_handler.rs | 110 ++++ src/dragonball/src/device_manager/mod.rs | 23 + src/dragonball/src/error.rs | 5 + 7 files changed, 759 insertions(+), 1 deletion(-) create mode 100644 src/dragonball/src/device_manager/fs_dev_mgr.rs create mode 100644 src/dragonball/src/device_manager/memory_region_handler.rs diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index d44e115ca52e..0f4aa582f99a 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -51,6 +51,8 @@ hotplug = ["virtio-vsock"] virtio-vsock = ["dbs-virtio-devices/virtio-vsock", "virtio-queue"] virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] +# virtio-fs only work on atomic-guest-memory +virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] [patch.'crates-io'] dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 0a021341ac4f..dfba2faf13d9 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -21,6 +21,10 @@ use crate::vmm::Vmm; use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, }; +#[cfg(feature = "virtio-fs")] +use crate::device_manager::fs_dev_mgr::{ + FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo, +}; #[cfg(feature = "virtio-net")] use crate::device_manager::virtio_net_dev_mgr::{ VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, @@ -74,6 +78,11 @@ pub enum VmmActionError { /// Net device related errors. #[error("virtio-net device error: {0}")] VirtioNet(#[source] VirtioNetDeviceError), + + #[cfg(feature = "virtio-fs")] + /// The action `InsertFsDevice` failed either because of bad user input or an internal error. + #[error("virtio-fs device: {0}")] + FsDevice(#[source] FsDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -129,6 +138,22 @@ pub enum VmmAction { /// Update a network interface, after microVM start. Currently, the only updatable properties /// are the RX and TX rate limiters. UpdateNetworkInterface(VirtioNetDeviceConfigUpdateInfo), + + #[cfg(feature = "virtio-fs")] + /// Add a new shared fs device or update one that already exists using the + /// `FsDeviceConfig` as input. This action can only be called before the microVM has + /// booted. + InsertFsDevice(FsDeviceConfigInfo), + + #[cfg(feature = "virtio-fs")] + /// Attach a new virtiofs Backend fs or detach an existing virtiofs Backend fs using the + /// `FsMountConfig` as input. This action can only be called _after_ the microVM has + /// booted. + ManipulateFsBackendFs(FsMountConfigInfo), + + #[cfg(feature = "virtio-fs")] + /// Update fs rate limiter, after microVM start. + UpdateFsDevice(FsDeviceConfigUpdateInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -218,6 +243,17 @@ impl VmmService { VmmAction::UpdateNetworkInterface(netif_update) => { self.update_net_rate_limiters(vmm, netif_update) } + #[cfg(feature = "virtio-fs")] + VmmAction::InsertFsDevice(fs_cfg) => self.add_fs_device(vmm, fs_cfg), + + #[cfg(feature = "virtio-fs")] + VmmAction::ManipulateFsBackendFs(fs_mount_cfg) => { + self.manipulate_fs_backend_fs(vmm, fs_mount_cfg) + } + #[cfg(feature = "virtio-fs")] + VmmAction::UpdateFsDevice(fs_update_cfg) => { + self.update_fs_rate_limiters(vmm, fs_update_cfg) + } }; debug!("send vmm response: {:?}", response); @@ -566,4 +602,63 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::VirtioNet) } + + #[cfg(feature = "virtio-fs")] + fn add_fs_device(&mut self, vmm: &mut Vmm, config: FsDeviceConfigInfo) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + let hotplug = vm.is_vm_initialized(); + if !cfg!(feature = "hotplug") && hotplug { + return Err(VmmActionError::FsDevice( + FsDeviceError::UpdateNotAllowedPostBoot, + )); + } + + let ctx = vm.create_device_op_context(None).map_err(|e| { + info!("create device op context error: {:?}", e); + VmmActionError::FsDevice(FsDeviceError::UpdateNotAllowedPostBoot) + })?; + FsDeviceMgr::insert_device(vm.device_manager_mut(), ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::FsDevice) + } + + #[cfg(feature = "virtio-fs")] + fn manipulate_fs_backend_fs( + &self, + vmm: &mut Vmm, + config: FsMountConfigInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + if !vm.is_vm_initialized() { + return Err(VmmActionError::FsDevice(FsDeviceError::MicroVMNotRunning)); + } + + FsDeviceMgr::manipulate_backend_fs(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::FsDevice) + } + + #[cfg(feature = "virtio-fs")] + fn update_fs_rate_limiters( + &self, + vmm: &mut Vmm, + config: FsDeviceConfigUpdateInfo, + ) -> VmmRequestResult { + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + + if !vm.is_vm_initialized() { + return Err(VmmActionError::FsDevice(FsDeviceError::MicroVMNotRunning)); + } + + FsDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::FsDevice) + } } diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 38e46cd59486..a624f68db682 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -176,7 +176,7 @@ pub struct BlockDeviceConfigInfo { pub no_drop: bool, /// Block device multi-queue pub num_queues: usize, - /// Virtio queue size. + /// Virtio queue size. Size: byte pub queue_size: u16, /// Rate Limiter for I/O operations. pub rate_limiter: Option, diff --git a/src/dragonball/src/device_manager/fs_dev_mgr.rs b/src/dragonball/src/device_manager/fs_dev_mgr.rs new file mode 100644 index 000000000000..c8fb55366737 --- /dev/null +++ b/src/dragonball/src/device_manager/fs_dev_mgr.rs @@ -0,0 +1,523 @@ +// Copyright 2020-2022 Alibaba Cloud. All Rights Reserved. +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::convert::TryInto; + +use dbs_utils::epoll_manager::EpollManager; +use dbs_virtio_devices::{self as virtio, Error as VirtIoError}; +use serde_derive::{Deserialize, Serialize}; +use slog::{error, info}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ + get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, +}; +use crate::device_manager::{ + DbsMmioV2Device, DeviceManager, DeviceMgrError, DeviceOpContext, DeviceVirtioRegionHandler, +}; + +use super::DbsVirtioDevice; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = true; +// Default cache size is 2 Gi since this is a typical VM memory size. +const DEFAULT_CACHE_SIZE: u64 = 2 * 1024 * 1024 * 1024; + +/// Errors associated with `FsDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum FsDeviceError { + /// Invalid fs, "virtio" or "vhostuser" is allowed. + #[error("the fs type is invalid, virtio or vhostuser is allowed")] + InvalidFs, + + /// Cannot access address space. + #[error("Cannot access address space.")] + AddressSpaceNotInitialized, + + /// Cannot convert RateLimterConfigInfo into RateLimiter. + #[error("failure while converting RateLimterConfigInfo into RateLimiter: {0}")] + RateLimterConfigInfoTryInto(#[source] std::io::Error), + + /// The fs device tag was already used for a different fs. + #[error("VirtioFs device tag {0} already exists")] + FsDeviceTagAlreadyExists(String), + + /// The fs device path was already used for a different fs. + #[error("VirtioFs device tag {0} already exists")] + FsDevicePathAlreadyExists(String), + + /// The update is not allowed after booting the microvm. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The attachbackendfs operation fails. + #[error("Fs device attach a backend fs failed")] + AttachBackendFailed(String), + + /// attach backend fs must be done when vm is running. + #[error("vm is not running when attaching a backend fs")] + MicroVMNotRunning, + + /// The mount tag doesn't exist. + #[error("fs tag'{0}' doesn't exist")] + TagNotExists(String), + + /// Failed to send patch message to VirtioFs epoll handler. + #[error("could not send patch message to the VirtioFs epoll handler")] + VirtioFsEpollHanderSendFail, + + /// Creating a shared-fs device fails (if the vhost-user socket cannot be open.) + #[error("cannot create shared-fs device: {0}")] + CreateFsDevice(#[source] VirtIoError), + + /// Cannot initialize a shared-fs device or add a device to the MMIO Bus. + #[error("failure while registering shared-fs device: {0}")] + RegisterFsDevice(#[source] DeviceMgrError), + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a vhost-user-fs device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct FsDeviceConfigInfo { + /// vhost-user socket path. + pub sock_path: String, + /// virtiofs mount tag name used inside the guest. + /// used as the device name during mount. + pub tag: String, + /// Number of virtqueues to use. + pub num_queues: usize, + /// Size of each virtqueue. Unit: byte. + pub queue_size: u16, + /// DAX cache window size + pub cache_size: u64, + /// Number of thread pool workers. + pub thread_pool_size: u16, + /// The caching policy the file system should use (auto, always or never). + /// This cache policy is set for virtio-fs, visit https://gitlab.com/virtio-fs/virtiofsd to get further information. + pub cache_policy: String, + /// Writeback cache + pub writeback_cache: bool, + /// Enable no_open or not + pub no_open: bool, + /// Enable xattr or not + pub xattr: bool, + /// Drop CAP_SYS_RESOURCE or not + pub drop_sys_resource: bool, + /// virtio fs or vhostuser fs. + pub mode: String, + /// Enable kill_priv_v2 or not + pub fuse_killpriv_v2: bool, + /// Enable no_readdir or not + pub no_readdir: bool, + /// Rate Limiter for I/O operations. + pub rate_limiter: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl std::default::Default for FsDeviceConfigInfo { + fn default() -> Self { + Self { + sock_path: String::default(), + tag: String::default(), + num_queues: 1, + queue_size: 1024, + cache_size: DEFAULT_CACHE_SIZE, + thread_pool_size: 0, + cache_policy: Self::default_cache_policy(), + writeback_cache: Self::default_writeback_cache(), + no_open: Self::default_no_open(), + fuse_killpriv_v2: Self::default_fuse_killpriv_v2(), + no_readdir: Self::default_no_readdir(), + xattr: Self::default_xattr(), + drop_sys_resource: Self::default_drop_sys_resource(), + mode: Self::default_fs_mode(), + rate_limiter: Some(RateLimiterConfigInfo::default()), + use_shared_irq: None, + use_generic_irq: None, + } + } +} + +impl FsDeviceConfigInfo { + /// The default mode is set to 'virtio' for 'virtio-fs' device. + pub fn default_fs_mode() -> String { + String::from("virtio") + } + + /// The default cache policy + pub fn default_cache_policy() -> String { + "always".to_string() + } + + /// The default setting of writeback cache + pub fn default_writeback_cache() -> bool { + true + } + + /// The default setting of no_open + pub fn default_no_open() -> bool { + true + } + + /// The default setting of killpriv_v2 + pub fn default_fuse_killpriv_v2() -> bool { + false + } + + /// The default setting of xattr + pub fn default_xattr() -> bool { + false + } + + /// The default setting of drop_sys_resource + pub fn default_drop_sys_resource() -> bool { + false + } + + /// The default setting of no_readdir + pub fn default_no_readdir() -> bool { + false + } + + /// The default setting of rate limiter + pub fn default_fs_rate_limiter() -> Option { + None + } +} + +/// Configuration information for virtio-fs. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct FsDeviceConfigUpdateInfo { + /// virtiofs mount tag name used inside the guest. + /// used as the device name during mount. + pub tag: String, + /// Rate Limiter for I/O operations. + pub rate_limiter: Option, +} + +impl FsDeviceConfigUpdateInfo { + /// Provides a `BucketUpdate` description for the bandwidth rate limiter. + pub fn bytes(&self) -> dbs_utils::rate_limiter::BucketUpdate { + get_bucket_update!(self, rate_limiter, bandwidth) + } + /// Provides a `BucketUpdate` description for the ops rate limiter. + pub fn ops(&self) -> dbs_utils::rate_limiter::BucketUpdate { + get_bucket_update!(self, rate_limiter, ops) + } +} + +impl ConfigItem for FsDeviceConfigInfo { + type Err = FsDeviceError; + + fn id(&self) -> &str { + &self.tag + } + + fn check_conflicts(&self, other: &Self) -> Result<(), FsDeviceError> { + if self.tag == other.tag { + Err(FsDeviceError::FsDeviceTagAlreadyExists(self.tag.clone())) + } else if self.mode.as_str() == "vhostuser" && self.sock_path == other.sock_path { + Err(FsDeviceError::FsDevicePathAlreadyExists( + self.sock_path.clone(), + )) + } else { + Ok(()) + } + } +} + +/// Configuration information of manipulating backend fs for a virtiofs device. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct FsMountConfigInfo { + /// Mount operations, mount, update, umount + pub ops: String, + /// The backend fs type to mount. + pub fstype: Option, + /// the source file/directory the backend fs points to + pub source: Option, + /// where the backend fs gets mounted + pub mountpoint: String, + /// backend fs config content in json format + pub config: Option, + /// virtiofs mount tag name used inside the guest. + /// used as the device name during mount. + pub tag: String, + /// Path to file that contains file lists that should be prefetched by rafs + pub prefetch_list_path: Option, + /// What size file supports dax + pub dax_threshold_size_kb: Option, +} + +pub(crate) type FsDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for FsDeviceInfo { + type Err = FsDeviceError; + fn id(&self) -> &str { + &self.config.tag + } + + fn check_conflicts(&self, other: &Self) -> Result<(), FsDeviceError> { + if self.config.tag == other.config.tag { + Err(FsDeviceError::FsDeviceTagAlreadyExists( + self.config.tag.clone(), + )) + } else if self.config.sock_path == other.config.sock_path { + Err(FsDeviceError::FsDevicePathAlreadyExists( + self.config.sock_path.clone(), + )) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Fs Devices Configs +pub struct FsDeviceMgr { + /// A list of `FsDeviceConfig` objects. + pub(crate) info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl FsDeviceMgr { + /// Inserts `fs_cfg` in the shared-fs device configuration list. + pub fn insert_device( + device_mgr: &mut DeviceManager, + ctx: DeviceOpContext, + fs_cfg: FsDeviceConfigInfo, + ) -> std::result::Result<(), FsDeviceError> { + // It's too complicated to manage life cycle of shared-fs service process for hotplug. + if ctx.is_hotplug { + error!( + ctx.logger(), + "no support of shared-fs device hotplug"; + "subsystem" => "shared-fs", + "tag" => &fs_cfg.tag, + ); + return Err(FsDeviceError::UpdateNotAllowedPostBoot); + } + + info!( + ctx.logger(), + "add shared-fs device configuration"; + "subsystem" => "shared-fs", + "tag" => &fs_cfg.tag, + ); + device_mgr + .fs_manager + .lock() + .unwrap() + .info_list + .insert_or_update(&fs_cfg)?; + + Ok(()) + } + + /// Attaches all vhost-user-fs devices from the FsDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), FsDeviceError> { + let epoll_mgr = ctx + .epoll_mgr + .clone() + .ok_or(FsDeviceError::CreateFsDevice(virtio::Error::InvalidInput))?; + + for info in self.info_list.iter_mut() { + let device = Self::create_fs_device(&info.config, ctx, epoll_mgr.clone())?; + let mmio_device = DeviceManager::create_mmio_virtio_device( + device, + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(FsDeviceError::RegisterFsDevice)?; + + info.set_device(mmio_device); + } + + Ok(()) + } + + fn create_fs_device( + config: &FsDeviceConfigInfo, + ctx: &mut DeviceOpContext, + epoll_mgr: EpollManager, + ) -> std::result::Result { + match config.mode.as_str() { + "virtio" => Self::attach_virtio_fs_devices(config, ctx, epoll_mgr), + _ => Err(FsDeviceError::CreateFsDevice(virtio::Error::InvalidInput)), + } + } + + fn attach_virtio_fs_devices( + config: &FsDeviceConfigInfo, + ctx: &mut DeviceOpContext, + epoll_mgr: EpollManager, + ) -> std::result::Result { + info!( + ctx.logger(), + "add virtio-fs device configuration"; + "subsystem" => "virito-fs", + "tag" => &config.tag, + "dax_window_size" => &config.cache_size, + ); + + let limiter = if let Some(rlc) = config.rate_limiter.clone() { + Some( + rlc.try_into() + .map_err(FsDeviceError::RateLimterConfigInfoTryInto)?, + ) + } else { + None + }; + + let vm_as = ctx.get_vm_as().map_err(|e| { + error!(ctx.logger(), "virtio-fs get vm_as error: {:?}", e; + "subsystem" => "virito-fs"); + FsDeviceError::DeviceManager(e) + })?; + let address_space = match ctx.address_space.as_ref() { + Some(address_space) => address_space.clone(), + None => { + error!(ctx.logger(), "virtio-fs get address_space error"; "subsystem" => "virito-fs"); + return Err(FsDeviceError::AddressSpaceNotInitialized); + } + }; + let handler = DeviceVirtioRegionHandler { + vm_as, + address_space, + }; + + let device = Box::new( + virtio::fs::VirtioFs::new( + &config.tag, + config.num_queues, + config.queue_size, + config.cache_size, + &config.cache_policy, + config.thread_pool_size, + config.writeback_cache, + config.no_open, + config.fuse_killpriv_v2, + config.xattr, + config.drop_sys_resource, + config.no_readdir, + Box::new(handler), + epoll_mgr, + limiter, + ) + .map_err(FsDeviceError::CreateFsDevice)?, + ); + + Ok(device) + } + + /// Attach a backend fs to a VirtioFs device or detach a backend + /// fs from a Virtiofs device + pub fn manipulate_backend_fs( + device_mgr: &mut DeviceManager, + config: FsMountConfigInfo, + ) -> std::result::Result<(), FsDeviceError> { + let mut found = false; + + let mgr = &mut device_mgr.fs_manager.lock().unwrap(); + for info in mgr + .info_list + .iter() + .filter(|info| info.config.tag.as_str() == config.tag.as_str()) + { + found = true; + if let Some(device) = info.device.as_ref() { + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let mut guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device_mut(); + if let Some(virtio_fs_dev) = inner_dev + .as_any_mut() + .downcast_mut::>() + { + return virtio_fs_dev + .manipulate_backend_fs( + config.source, + config.fstype, + &config.mountpoint, + config.config, + &config.ops, + config.prefetch_list_path, + config.dax_threshold_size_kb, + ) + .map(|_p| ()) + .map_err(|e| FsDeviceError::AttachBackendFailed(e.to_string())); + } + } + } + } + if !found { + Err(FsDeviceError::AttachBackendFailed( + "fs tag not found".to_string(), + )) + } else { + Ok(()) + } + } + + /// Gets the index of the device with the specified `tag` if it exists in the list. + pub fn get_index_of_tag(&self, tag: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.id().eq(tag)) + } + + /// Update the ratelimiter settings of a virtio fs device. + pub fn update_device_ratelimiters( + device_mgr: &mut DeviceManager, + new_cfg: FsDeviceConfigUpdateInfo, + ) -> std::result::Result<(), FsDeviceError> { + let mgr = &mut device_mgr.fs_manager.lock().unwrap(); + match mgr.get_index_of_tag(&new_cfg.tag) { + Some(index) => { + let config = &mut mgr.info_list[index].config; + config.rate_limiter = new_cfg.rate_limiter.clone(); + let device = mgr.info_list[index] + .device + .as_mut() + .ok_or_else(|| FsDeviceError::TagNotExists("".to_owned()))?; + + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(fs_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return fs_dev + .set_patch_rate_limiters(new_cfg.bytes(), new_cfg.ops()) + .map(|_p| ()) + .map_err(|_e| FsDeviceError::VirtioFsEpollHanderSendFail); + } + } + Ok(()) + } + None => Err(FsDeviceError::TagNotExists(new_cfg.tag)), + } + } +} + +impl Default for FsDeviceMgr { + /// Create a new `FsDeviceMgr` object.. + fn default() -> Self { + FsDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} diff --git a/src/dragonball/src/device_manager/memory_region_handler.rs b/src/dragonball/src/device_manager/memory_region_handler.rs new file mode 100644 index 000000000000..2be149ef97f6 --- /dev/null +++ b/src/dragonball/src/device_manager/memory_region_handler.rs @@ -0,0 +1,110 @@ +// Copyright 2022 Alibaba, Inc. or its affiliates. All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::io; +use std::sync::Arc; + +use dbs_address_space::{AddressSpace, AddressSpaceRegion, AddressSpaceRegionType}; +use dbs_virtio_devices::{Error as VirtIoError, VirtioRegionHandler}; +use log::{debug, error}; +use vm_memory::{FileOffset, GuestAddressSpace, GuestMemoryRegion, GuestRegionMmap}; + +use crate::address_space_manager::GuestAddressSpaceImpl; + +/// This struct implements the VirtioRegionHandler trait, which inserts the memory +/// region of the virtio device into vm_as and address_space. +/// +/// * After region is inserted into the vm_as, the virtio device can read guest memory +/// data using vm_as.get_slice with GuestAddress. +/// +/// * Insert virtio memory into address_space so that the correct guest last address can +/// be found when initializing the e820 table. The e820 table is a table that describes +/// guest memory prepared before the guest startup. we need to config the correct guest +/// memory address and length in the table. The virtio device memory belongs to the MMIO +/// space and does not belong to the Guest Memory space. Therefore, it cannot be configured +/// into the e820 table. When creating AddressSpaceRegion we use +/// AddressSpaceRegionType::ReservedMemory type, in this way, address_space will know that +/// this region a special memory, it will don't put the this memory in e820 table. +/// +/// This function relies on the atomic-guest-memory feature. Without this feature enabled, memory +/// regions cannot be inserted into vm_as. Because the insert_region interface of vm_as does +/// not insert regions in place, but returns an array of inserted regions. We need to manually +/// replace this array of regions with vm_as, and that's what atomic-guest-memory feature does. +/// So we rely on the atomic-guest-memory feature here +pub struct DeviceVirtioRegionHandler { + pub(crate) vm_as: GuestAddressSpaceImpl, + pub(crate) address_space: AddressSpace, +} + +impl DeviceVirtioRegionHandler { + fn insert_address_space( + &mut self, + region: Arc, + ) -> std::result::Result<(), VirtIoError> { + let file_offset = match region.file_offset() { + // TODO: use from_arc + Some(f) => Some(FileOffset::new(f.file().try_clone()?, 0)), + None => None, + }; + + let as_region = Arc::new(AddressSpaceRegion::build( + AddressSpaceRegionType::DAXMemory, + region.start_addr(), + region.size() as u64, + None, + file_offset, + region.flags(), + false, + )); + + self.address_space.insert_region(as_region).map_err(|e| { + error!("inserting address apace error: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + region.start_addr().0, + region.len() + ), + )) + })?; + Ok(()) + } + + fn insert_vm_as( + &mut self, + region: Arc, + ) -> std::result::Result<(), VirtIoError> { + let vm_as_new = self.vm_as.memory().insert_region(region).map_err(|e| { + error!( + "DeviceVirtioRegionHandler failed to insert guest memory region: {:?}.", + e + ); + VirtIoError::InsertMmap(e) + })?; + // Do not expect poisoned lock here, so safe to unwrap(). + self.vm_as.lock().unwrap().replace(vm_as_new); + + Ok(()) + } +} + +impl VirtioRegionHandler for DeviceVirtioRegionHandler { + fn insert_region( + &mut self, + region: Arc, + ) -> std::result::Result<(), VirtIoError> { + debug!( + "add geust memory region to address_space/vm_as, new region: {:?}", + region + ); + + self.insert_address_space(region.clone())?; + self.insert_vm_as(region)?; + + Ok(()) + } +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 9313919f85ae..3cb0477e3d5c 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -71,6 +71,16 @@ pub mod virtio_net_dev_mgr; #[cfg(feature = "virtio-net")] use self::virtio_net_dev_mgr::VirtioNetDeviceMgr; +#[cfg(feature = "virtio-fs")] +/// virtio-block device manager +pub mod fs_dev_mgr; +#[cfg(feature = "virtio-fs")] +use self::fs_dev_mgr::FsDeviceMgr; +#[cfg(feature = "virtio-fs")] +mod memory_region_handler; +#[cfg(feature = "virtio-fs")] +pub use self::memory_region_handler::*; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -435,6 +445,9 @@ pub struct DeviceManager { #[cfg(feature = "virtio-net")] pub(crate) virtio_net_manager: VirtioNetDeviceMgr, + + #[cfg(feature = "virtio-fs")] + fs_manager: Arc>, } impl DeviceManager { @@ -463,6 +476,8 @@ impl DeviceManager { block_manager: BlockDeviceMgr::default(), #[cfg(feature = "virtio-net")] virtio_net_manager: VirtioNetDeviceMgr::default(), + #[cfg(feature = "virtio-fs")] + fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), } } @@ -588,6 +603,14 @@ impl DeviceManager { .attach_devices(&mut ctx) .map_err(StartMicrovmError::BlockDeviceError)?; + #[cfg(feature = "virtio-fs")] + { + let mut fs_manager = self.fs_manager.lock().unwrap(); + fs_manager + .attach_devices(&mut ctx) + .map_err(StartMicrovmError::FsDeviceError)?; + } + #[cfg(feature = "virtio-net")] self.virtio_net_manager .attach_devices(&mut ctx) diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 667405c5ecd8..2c6fbf6c4402 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -168,6 +168,11 @@ pub enum StartMicroVmError { /// Virtio-net errors. #[error("virtio-net errors: {0}")] VirtioNetDeviceError(#[source] device_manager::virtio_net_dev_mgr::VirtioNetDeviceError), + + #[cfg(feature = "virtio-fs")] + /// Virtio-fs errors. + #[error("virtio-fs errors: {0}")] + FsDeviceError(#[source] device_manager::fs_dev_mgr::FsDeviceError), } /// Errors associated with starting the instance. From 38957fe00b71a79fe1e2f16063a334e6ee79074c Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Sun, 3 Jul 2022 23:34:26 +0800 Subject: [PATCH 0103/1953] UT: fix compile error in unit tests fix compile error in unit tests for DummyConfigInfo. Signed-off-by: Chao Wu --- src/dragonball/src/address_space_manager.rs | 4 ++-- src/dragonball/src/api/v1/vmm_action.rs | 1 - src/dragonball/src/config_manager.rs | 3 +-- src/dragonball/src/vcpu/vcpu_impl.rs | 9 --------- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/dragonball/src/address_space_manager.rs b/src/dragonball/src/address_space_manager.rs index 5a54c0da01c1..9992833e0c0f 100644 --- a/src/dragonball/src/address_space_manager.rs +++ b/src/dragonball/src/address_space_manager.rs @@ -867,8 +867,8 @@ mod tests { assert_eq!(&builder.get_next_mem_file(), "/tmp/shmem"); assert_eq!(builder.mem_index, 3); - builder.set_prealloc(true); - builder.set_dirty_page_logging(true); + builder.toggle_prealloc(true); + builder.toggle_dirty_page_logging(true); assert!(builder.mem_prealloc); assert!(builder.dirty_page_logging); } diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index dfba2faf13d9..2e284d79ba19 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -10,7 +10,6 @@ use std::fs::File; use std::sync::mpsc::{Receiver, Sender, TryRecvError}; use log::{debug, error, info, warn}; -use vmm_sys_util::eventfd::EventFd; use crate::error::{Result, StartMicrovmError, StopMicrovmError}; use crate::event_manager::EventManager; diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs index cbfb790413c5..a80fd14c726f 100644 --- a/src/dragonball/src/config_manager.rs +++ b/src/dragonball/src/config_manager.rs @@ -30,7 +30,6 @@ macro_rules! get_bucket_update { } }}; } -pub(crate) use get_bucket_update; /// Trait for generic configuration information. pub trait ConfigItem { @@ -414,7 +413,7 @@ mod tests { Exist, } - #[derive(Clone, Debug)] + #[derive(Clone, Debug, Default)] pub struct DummyConfigInfo { id: String, content: String, diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index 2dbccf66d059..513fa435f931 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -966,15 +966,6 @@ pub mod tests { fn test_vcpu_check_io_port_info() { let (vcpu, _receiver) = create_vcpu(); - // boot complete signal - let res = vcpu - .check_io_port_info( - MAGIC_IOPORT_SIGNAL_GUEST_BOOT_COMPLETE, - &[MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE], - ) - .unwrap(); - assert!(res); - // debug info signal let res = vcpu .check_io_port_info(MAGIC_IOPORT_DEBUG_INFO, &[0, 0, 0, 0]) From dd003ebe0e690e85cf7d68e05a7e4dc782237f35 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 4 Jul 2022 11:37:42 +0800 Subject: [PATCH 0104/1953] Dragonball: change error name and fix compile error Change error name from `StartMicrovm` to `StartMicroVm`, `StartMicrovmError` to `StartMicroVmError`. Besides, we fix a compile error in config_manager. Signed-off-by: Chao Wu --- src/dragonball/src/api/v1/vmm_action.rs | 18 ++--- src/dragonball/src/config_manager.rs | 2 + .../src/device_manager/blk_dev_mgr.rs | 5 +- .../src/device_manager/fs_dev_mgr.rs | 3 +- src/dragonball/src/device_manager/mod.rs | 28 +++---- .../src/device_manager/virtio_net_dev_mgr.rs | 3 +- .../src/device_manager/vsock_dev_mgr.rs | 20 ++--- src/dragonball/src/lib.rs | 2 +- src/dragonball/src/vm/mod.rs | 74 +++++++++---------- src/dragonball/src/vm/x86_64.rs | 36 ++++----- 10 files changed, 97 insertions(+), 94 deletions(-) diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 2e284d79ba19..6dedbfc28e1d 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -11,7 +11,7 @@ use std::sync::mpsc::{Receiver, Sender, TryRecvError}; use log::{debug, error, info, warn}; -use crate::error::{Result, StartMicrovmError, StopMicrovmError}; +use crate::error::{Result, StartMicroVmError, StopMicrovmError}; use crate::event_manager::EventManager; use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; @@ -310,8 +310,8 @@ impl VmmService { } fn start_microvm(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> VmmRequestResult { - use self::StartMicrovmError::MicroVMAlreadyRunning; - use self::VmmActionError::StartMicrovm; + use self::StartMicroVmError::MicroVMAlreadyRunning; + use self::VmmActionError::StartMicroVm; let vmm_seccomp_filter = vmm.vmm_seccomp_filter(); let vcpu_seccomp_filter = vmm.vcpu_seccomp_filter(); @@ -319,12 +319,12 @@ impl VmmService { .get_vm_by_id_mut("") .ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { - return Err(StartMicrovm(MicroVMAlreadyRunning)); + return Err(StartMicroVm(MicroVMAlreadyRunning)); } vm.start_microvm(event_mgr, vmm_seccomp_filter, vcpu_seccomp_filter) .map(|_| VmmData::Empty) - .map_err(StartMicrovm) + .map_err(StartMicroVm) } fn shutdown_microvm(&mut self, vmm: &mut Vmm) -> VmmRequestResult { @@ -512,7 +512,7 @@ impl VmmService { let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|e| { - if let StartMicrovmError::UpcallNotReady = e { + if let StartMicroVmError::UpcallNotReady = e { return VmmActionError::UpcallNotReady; } VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot) @@ -573,12 +573,12 @@ impl VmmService { let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|e| { - if let StartMicrovmError::MicroVMAlreadyRunning = e { + if let StartMicroVmError::MicroVMAlreadyRunning = e { VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot) - } else if let StartMicrovmError::UpcallNotReady = e { + } else if let StartMicroVmError::UpcallNotReady = e { VmmActionError::UpcallNotReady } else { - VmmActionError::StartMicrovm(e) + VmmActionError::StartMicroVm(e) } })?; diff --git a/src/dragonball/src/config_manager.rs b/src/dragonball/src/config_manager.rs index a80fd14c726f..f855be12662d 100644 --- a/src/dragonball/src/config_manager.rs +++ b/src/dragonball/src/config_manager.rs @@ -10,6 +10,8 @@ use dbs_device::DeviceIo; use dbs_utils::rate_limiter::{RateLimiter, TokenBucket}; use serde_derive::{Deserialize, Serialize}; +/// Get bucket update for rate limiter. +#[macro_export] macro_rules! get_bucket_update { ($self:ident, $rate_limiter: ident, $metric: ident) => {{ match &$self.$rate_limiter { diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index a624f68db682..0d35c4f95f03 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -20,10 +20,9 @@ use dbs_virtio_devices::block::{aio::Aio, io_uring::IoUring, Block, LocalFile, U use serde_derive::{Deserialize, Serialize}; use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::config_manager::{ - get_bucket_update, ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo, -}; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo}; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::get_bucket_update; use crate::vm::KernelConfigInfo; use super::DbsMmioV2Device; diff --git a/src/dragonball/src/device_manager/fs_dev_mgr.rs b/src/dragonball/src/device_manager/fs_dev_mgr.rs index c8fb55366737..d699dcbde573 100644 --- a/src/dragonball/src/device_manager/fs_dev_mgr.rs +++ b/src/dragonball/src/device_manager/fs_dev_mgr.rs @@ -12,11 +12,12 @@ use slog::{error, info}; use crate::address_space_manager::GuestAddressSpaceImpl; use crate::config_manager::{ - get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, + ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, }; use crate::device_manager::{ DbsMmioV2Device, DeviceManager, DeviceMgrError, DeviceOpContext, DeviceVirtioRegionHandler, }; +use crate::get_bucket_update; use super::DbsVirtioDevice; diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 3cb0477e3d5c..53a1e2df92d4 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -40,7 +40,7 @@ use dbs_upcall::{ use dbs_virtio_devices::vsock::backend::VsockInnerConnector; use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::error::StartMicrovmError; +use crate::error::StartMicroVmError; use crate::resource_manager::ResourceManager; use crate::vm::{KernelConfigInfo, Vm}; use crate::IoManagerCached; @@ -503,7 +503,7 @@ impl DeviceManager { pub fn create_legacy_devices( &mut self, ctx: &mut DeviceOpContext, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { #[cfg(target_arch = "x86_64")] { let mut tx = ctx.io_context.begin_tx(); @@ -517,7 +517,7 @@ impl DeviceManager { } Err(e) => { ctx.io_context.cancel_tx(tx); - return Err(StartMicrovmError::LegacyDevice(e)); + return Err(StartMicroVmError::LegacyDevice(e)); } } } @@ -531,10 +531,10 @@ impl DeviceManager { dmesg_fifo: Option>, com1_sock_path: Option, _ctx: &mut DeviceOpContext, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { // Connect serial ports to the console and dmesg_fifo. self.set_guest_kernel_log_stream(dmesg_fifo) - .map_err(|_| StartMicrovmError::EventFd)?; + .map_err(|_| StartMicroVmError::EventFd)?; info!(self.logger, "init console path: {:?}", com1_sock_path); if let Some(path) = com1_sock_path { @@ -542,13 +542,13 @@ impl DeviceManager { let com1 = legacy_manager.get_com1_serial(); self.con_manager .create_socket_console(com1, path) - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; } } else if let Some(legacy_manager) = self.legacy_manager.as_ref() { let com1 = legacy_manager.get_com1_serial(); self.con_manager .create_stdio_console(com1) - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; } Ok(()) @@ -586,7 +586,7 @@ impl DeviceManager { com1_sock_path: Option, dmesg_fifo: Option>, address_space: Option<&AddressSpace>, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { let mut ctx = DeviceOpContext::new( Some(epoll_mgr), self, @@ -601,20 +601,20 @@ impl DeviceManager { #[cfg(feature = "virtio-blk")] self.block_manager .attach_devices(&mut ctx) - .map_err(StartMicrovmError::BlockDeviceError)?; + .map_err(StartMicroVmError::BlockDeviceError)?; #[cfg(feature = "virtio-fs")] { let mut fs_manager = self.fs_manager.lock().unwrap(); fs_manager .attach_devices(&mut ctx) - .map_err(StartMicrovmError::FsDeviceError)?; + .map_err(StartMicroVmError::FsDeviceError)?; } #[cfg(feature = "virtio-net")] self.virtio_net_manager .attach_devices(&mut ctx) - .map_err(StartMicrovmError::VirtioNetDeviceError)?; + .map_err(StartMicroVmError::VirtioNetDeviceError)?; #[cfg(feature = "virtio-vsock")] self.vsock_manager.attach_devices(&mut ctx)?; @@ -622,15 +622,15 @@ impl DeviceManager { #[cfg(feature = "virtio-blk")] self.block_manager .generate_kernel_boot_args(kernel_config) - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; ctx.generate_kernel_boot_args(kernel_config) - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; Ok(()) } /// Start all registered devices when booting the associated virtual machine. - pub fn start_devices(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub fn start_devices(&mut self) -> std::result::Result<(), StartMicroVmError> { Ok(()) } diff --git a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs index 65f29fbcd048..3e81f2948773 100644 --- a/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs +++ b/src/dragonball/src/device_manager/virtio_net_dev_mgr.rs @@ -18,9 +18,10 @@ use serde_derive::{Deserialize, Serialize}; use crate::address_space_manager::GuestAddressSpaceImpl; use crate::config_manager::{ - get_bucket_update, ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, + ConfigItem, DeviceConfigInfo, DeviceConfigInfos, RateLimiterConfigInfo, }; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::get_bucket_update; use super::DbsMmioV2Device; diff --git a/src/dragonball/src/device_manager/vsock_dev_mgr.rs b/src/dragonball/src/device_manager/vsock_dev_mgr.rs index cec58d7de105..5e49bbfb375b 100644 --- a/src/dragonball/src/device_manager/vsock_dev_mgr.rs +++ b/src/dragonball/src/device_manager/vsock_dev_mgr.rs @@ -17,7 +17,7 @@ use dbs_virtio_devices::vsock::Vsock; use dbs_virtio_devices::Error as VirtioError; use serde_derive::{Deserialize, Serialize}; -use super::StartMicrovmError; +use super::StartMicroVmError; use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; use crate::device_manager::{DeviceManager, DeviceOpContext}; @@ -185,11 +185,11 @@ impl VsockDeviceMgr { pub fn attach_devices( &mut self, ctx: &mut DeviceOpContext, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { let epoll_mgr = ctx .epoll_mgr .clone() - .ok_or(StartMicrovmError::CreateVsockDevice( + .ok_or(StartMicroVmError::CreateVsockDevice( virtio::Error::InvalidInput, ))?; @@ -209,32 +209,32 @@ impl VsockDeviceMgr { epoll_mgr.clone(), ) .map_err(VirtioError::VirtioVsockError) - .map_err(StartMicrovmError::CreateVsockDevice)?, + .map_err(StartMicroVmError::CreateVsockDevice)?, ); if let Some(uds_path) = info.config.uds_path.as_ref() { let unix_backend = VsockUnixStreamBackend::new(uds_path.clone()) .map_err(VirtioError::VirtioVsockError) - .map_err(StartMicrovmError::CreateVsockDevice)?; + .map_err(StartMicroVmError::CreateVsockDevice)?; device .add_backend(Box::new(unix_backend), true) .map_err(VirtioError::VirtioVsockError) - .map_err(StartMicrovmError::CreateVsockDevice)?; + .map_err(StartMicroVmError::CreateVsockDevice)?; } if let Some(tcp_addr) = info.config.tcp_addr.as_ref() { let tcp_backend = VsockTcpBackend::new(tcp_addr.clone()) .map_err(VirtioError::VirtioVsockError) - .map_err(StartMicrovmError::CreateVsockDevice)?; + .map_err(StartMicroVmError::CreateVsockDevice)?; device .add_backend(Box::new(tcp_backend), false) .map_err(VirtioError::VirtioVsockError) - .map_err(StartMicrovmError::CreateVsockDevice)?; + .map_err(StartMicroVmError::CreateVsockDevice)?; } // add inner backend to the the first added vsock device if let Some(inner_backend) = self.default_inner_backend.take() { device .add_backend(Box::new(inner_backend), false) .map_err(VirtioError::VirtioVsockError) - .map_err(StartMicrovmError::CreateVsockDevice)?; + .map_err(StartMicroVmError::CreateVsockDevice)?; } let device = DeviceManager::create_mmio_virtio_device_with_features( device, @@ -243,7 +243,7 @@ impl VsockDeviceMgr { info.config.use_shared_irq.unwrap_or(self.use_shared_irq), info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), ) - .map_err(StartMicrovmError::RegisterVsockDevice)?; + .map_err(StartMicroVmError::RegisterVsockDevice)?; info.device = Some(device); } diff --git a/src/dragonball/src/lib.rs b/src/dragonball/src/lib.rs index bd58159ac208..7371e8213a2b 100644 --- a/src/dragonball/src/lib.rs +++ b/src/dragonball/src/lib.rs @@ -36,7 +36,7 @@ mod event_manager; mod io_manager; mod vmm; -pub use self::error::StartMicrovmError; +pub use self::error::StartMicroVmError; pub use self::io_manager::IoManagerCached; pub use self::vmm::Vmm; diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 4063b77708e4..8c3cb21c9745 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -30,7 +30,7 @@ use crate::address_space_manager::{ use crate::api::v1::{InstanceInfo, InstanceState}; use crate::device_manager::console_manager::DmesgWriter; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; -use crate::error::{LoadInitrdError, Result, StartMicrovmError, StopMicrovmError}; +use crate::error::{LoadInitrdError, Result, StartMicroVmError, StopMicrovmError}; use crate::event_manager::EventManager; use crate::kvm_context::KvmContext; use crate::resource_manager::ResourceManager; @@ -357,7 +357,7 @@ impl Vm { pub fn create_device_op_context( &mut self, epoll_mgr: Option, - ) -> std::result::Result { + ) -> std::result::Result { if !self.is_vm_initialized() { Ok(DeviceOpContext::create_boot_ctx(self, epoll_mgr)) } else { @@ -365,9 +365,9 @@ impl Vm { } } - pub(crate) fn check_health(&self) -> std::result::Result<(), StartMicrovmError> { + pub(crate) fn check_health(&self) -> std::result::Result<(), StartMicroVmError> { if self.kernel_config.is_none() { - return Err(StartMicrovmError::MissingKernelConfig); + return Err(StartMicroVmError::MissingKernelConfig); } Ok(()) } @@ -432,25 +432,25 @@ impl Vm { pub(crate) fn init_devices( &mut self, epoll_manager: EpollManager, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: initializing devices ..."); let com1_sock_path = self.vm_config.serial_path.clone(); let kernel_config = self .kernel_config .as_mut() - .ok_or(StartMicrovmError::MissingKernelConfig)?; + .ok_or(StartMicroVmError::MissingKernelConfig)?; info!(self.logger, "VM: create interrupt manager"); self.device_manager .create_interrupt_manager() - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; info!(self.logger, "VM: create devices"); let vm_as = self.address_space .get_vm_as() - .ok_or(StartMicrovmError::AddressManagerError( + .ok_or(StartMicroVmError::AddressManagerError( AddressManagerError::GuestMemoryNotInitialized, ))?; self.device_manager.create_devices( @@ -501,7 +501,7 @@ impl Vm { Box::new(DmesgWriter::new(&self.logger)) } - pub(crate) fn init_guest_memory(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub(crate) fn init_guest_memory(&mut self) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: initializing guest memory..."); // We are not allowing reinitialization of vm guest memory. if self.address_space.is_initialized() { @@ -512,7 +512,7 @@ impl Vm { let mem_size = (self.vm_config.mem_size_mib as u64) << 20; let reserve_memory_bytes = self.vm_config.reserve_memory_bytes; if reserve_memory_bytes > (mem_size >> 1) as u64 { - return Err(StartMicrovmError::ConfigureInvalid(String::from( + return Err(StartMicroVmError::ConfigureInvalid(String::from( "invalid reserve_memory_bytes", ))); } @@ -553,11 +553,11 @@ impl Vm { ); let mut address_space_param = AddressSpaceMgrBuilder::new(&mem_type, &mem_file_path) - .map_err(StartMicrovmError::AddressManagerError)?; + .map_err(StartMicroVmError::AddressManagerError)?; address_space_param.set_kvm_vm_fd(self.vm_fd.clone()); self.address_space .create_address_space(&self.resource_manager, &numa_regions, address_space_param) - .map_err(StartMicrovmError::AddressManagerError)?; + .map_err(StartMicroVmError::AddressManagerError)?; info!(self.logger, "VM: initializing guest memory done"); Ok(()) @@ -566,18 +566,18 @@ impl Vm { fn init_configure_system( &mut self, vm_as: &GuestAddressSpaceImpl, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { let vm_memory = vm_as.memory(); let kernel_config = self .kernel_config .as_ref() - .ok_or(StartMicrovmError::MissingKernelConfig)?; + .ok_or(StartMicroVmError::MissingKernelConfig)?; //let cmdline = kernel_config.cmdline.clone(); let initrd: Option = match kernel_config.initrd_file() { Some(f) => { let initrd_file = f.try_clone(); if initrd_file.is_err() { - return Err(StartMicrovmError::InitrdLoader( + return Err(StartMicroVmError::InitrdLoader( LoadInitrdError::ReadInitrd(io::Error::from(io::ErrorKind::InvalidData)), )); } @@ -638,12 +638,12 @@ impl Vm { fn load_kernel( &mut self, vm_memory: &GuestMemoryImpl, - ) -> std::result::Result { + ) -> std::result::Result { // This is the easy way out of consuming the value of the kernel_cmdline. let kernel_config = self .kernel_config .as_mut() - .ok_or(StartMicrovmError::MissingKernelConfig)?; + .ok_or(StartMicroVmError::MissingKernelConfig)?; let high_mem_addr = GuestAddress(dbs_boot::get_kernel_start()); #[cfg(target_arch = "x86_64")] @@ -653,7 +653,7 @@ impl Vm { kernel_config.kernel_file_mut(), Some(high_mem_addr), ) - .map_err(StartMicrovmError::KernelLoader); + .map_err(StartMicroVmError::KernelLoader); #[cfg(target_arch = "aarch64")] return linux_loader::loader::pe::PE::load( @@ -662,7 +662,7 @@ impl Vm { kernel_config.kernel_file_mut(), Some(high_mem_addr), ) - .map_err(StartMicrovmError::KernelLoader); + .map_err(StartMicroVmError::KernelLoader); } /// Set up the initial microVM state and start the vCPU threads. @@ -674,10 +674,10 @@ impl Vm { event_mgr: &mut EventManager, vmm_seccomp_filter: BpfProgram, vcpu_seccomp_filter: BpfProgram, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: received instance start command"); if self.is_vm_initialized() { - return Err(StartMicrovmError::MicroVMAlreadyRunning); + return Err(StartMicroVmError::MicroVMAlreadyRunning); } let request_ts = TimestampUs::default(); @@ -697,12 +697,12 @@ impl Vm { let vm_as = self .vm_as() .cloned() - .ok_or(StartMicrovmError::AddressManagerError( + .ok_or(StartMicroVmError::AddressManagerError( AddressManagerError::GuestMemoryNotInitialized, ))?; self.init_vcpu_manager(vm_as.clone(), vcpu_seccomp_filter) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; self.init_microvm(event_mgr.epoll_manager(), vm_as.clone(), request_ts)?; self.init_configure_system(&vm_as)?; self.init_upcall()?; @@ -712,9 +712,9 @@ impl Vm { info!(self.logger, "VM: start vcpus"); self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .start_boot_vcpus(vmm_seccomp_filter) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; // Use expect() to crash if the other thread poisoned this lock. self.shared_info @@ -730,29 +730,29 @@ impl Vm { #[cfg(feature = "hotplug")] impl Vm { /// initialize upcall client for guest os - fn new_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + fn new_upcall(&mut self) -> std::result::Result<(), StartMicroVmError> { // get vsock inner connector for upcall let inner_connector = self .device_manager .get_vsock_inner_connector() - .ok_or(StartMicrovmError::UpcallMissVsock)?; + .ok_or(StartMicroVmError::UpcallMissVsock)?; let mut upcall_client = UpcallClient::new( inner_connector, self.epoll_manager.clone(), DevMgrService::default(), ) - .map_err(StartMicrovmError::UpcallInitError)?; + .map_err(StartMicroVmError::UpcallInitError)?; upcall_client .connect() - .map_err(StartMicrovmError::UpcallConnectError)?; + .map_err(StartMicroVmError::UpcallConnectError)?; self.upcall_client = Some(Arc::new(upcall_client)); info!(self.logger, "upcall client init success"); Ok(()) } - fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + fn init_upcall(&mut self) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM upcall init"); if let Err(e) = self.new_upcall() { info!( @@ -762,7 +762,7 @@ impl Vm { Err(e) } else { self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .set_upcall_channel(self.upcall_client().clone()); Ok(()) } @@ -776,27 +776,27 @@ impl Vm { fn create_device_hotplug_context( &self, epoll_mgr: Option, - ) -> std::result::Result { + ) -> std::result::Result { if self.upcall_client().is_none() { - Err(StartMicrovmError::UpcallMissVsock) + Err(StartMicroVmError::UpcallMissVsock) } else if self.is_upcall_client_ready() { Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) } else { - Err(StartMicrovmError::UpcallNotReady) + Err(StartMicroVmError::UpcallNotReady) } } } #[cfg(not(feature = "hotplug"))] impl Vm { - fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + fn init_upcall(&mut self) -> std::result::Result<(), StartMicroVmError> { Ok(()) } fn create_device_hotplug_context( &self, _epoll_mgr: Option, - ) -> std::result::Result { - Err(StartMicrovmError::MicroVMAlreadyRunning) + ) -> std::result::Result { + Err(StartMicroVmError::MicroVMAlreadyRunning) } } diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index 6bd74acc09d9..3fcebdc20fe8 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -20,7 +20,7 @@ use slog::info; use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; -use crate::error::{Error, Result, StartMicrovmError}; +use crate::error::{Error, Result, StartMicroVmError}; use crate::event_manager::EventManager; use crate::vm::{Vm, VmError}; @@ -183,7 +183,7 @@ impl Vm { epoll_mgr: EpollManager, vm_as: GuestAddressSpaceImpl, request_ts: TimestampUs, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: start initializing microvm ..."); self.init_tss()?; @@ -195,9 +195,9 @@ impl Vm { let reset_event_fd = self.device_manager.get_reset_eventfd().unwrap(); self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .set_reset_event_fd(reset_event_fd) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; if self.vm_config.cpu_pm == "on" { // TODO: add cpu_pm support. issue #4590. @@ -207,9 +207,9 @@ impl Vm { let vm_memory = vm_as.memory(); let kernel_loader_result = self.load_kernel(vm_memory.deref())?; self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; info!(self.logger, "VM: initializing microvm done"); Ok(()) @@ -224,10 +224,10 @@ impl Vm { vm_memory: &GuestMemoryImpl, cmdline: &Cmdline, initrd: Option, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { let cmdline_addr = GuestAddress(dbs_boot::layout::CMDLINE_START); linux_loader::loader::load_cmdline(vm_memory, cmdline_addr, cmdline) - .map_err(StartMicrovmError::LoadCommandline)?; + .map_err(StartMicroVmError::LoadCommandline)?; configure_system( vm_memory, @@ -239,27 +239,27 @@ impl Vm { self.vm_config.max_vcpu_count, self.vm_config.reserve_memory_bytes, ) - .map_err(StartMicrovmError::ConfigureSystem) + .map_err(StartMicroVmError::ConfigureSystem) } /// Initializes the guest memory. - pub(crate) fn init_tss(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub(crate) fn init_tss(&mut self) -> std::result::Result<(), StartMicroVmError> { self.vm_fd .set_tss_address(dbs_boot::layout::KVM_TSS_ADDRESS.try_into().unwrap()) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::VmSetup(e))) } /// Creates the irq chip and an in-kernel device model for the PIT. pub(crate) fn setup_interrupt_controller( &mut self, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { self.vm_fd .create_irq_chip() - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::VmSetup(e))) } /// Creates an in-kernel device model for the PIT. - pub(crate) fn create_pit(&self) -> std::result::Result<(), StartMicrovmError> { + pub(crate) fn create_pit(&self) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: create pit"); // We need to enable the emulation of a dummy speaker port stub so that writing to port 0x61 // (i.e. KVM_SPEAKER_BASE_ADDRESS) does not trigger an exit to user space. @@ -272,20 +272,20 @@ impl Vm { // correct amount of memory from our pointer, and we verify the return result. self.vm_fd .create_pit2(pit_config) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::VmSetup(e))) } pub(crate) fn register_events( &mut self, event_mgr: &mut EventManager, - ) -> std::result::Result<(), StartMicrovmError> { + ) -> std::result::Result<(), StartMicroVmError> { let reset_evt = self .device_manager .get_reset_eventfd() - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; event_mgr .register_exit_eventfd(&reset_evt) - .map_err(|_| StartMicrovmError::RegisterEvent)?; + .map_err(|_| StartMicroVmError::RegisterEvent)?; self.reset_eventfd = Some(reset_evt); Ok(()) From d88b1bf01c152cae589cb13c238c3c85cc67dcb0 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 26 May 2022 17:29:50 +0800 Subject: [PATCH 0105/1953] dragonball: update vsock dependency 1. fix vsock device init failed 2. fix VsockDeviceConfigInfo not found Signed-off-by: wllenyj --- src/dragonball/src/api/v1/mod.rs | 4 +--- src/dragonball/src/api/v1/vmm_action.rs | 8 ++++---- src/dragonball/src/device_manager/vsock_dev_mgr.rs | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs index 2077c982e95f..99e3075ebb41 100644 --- a/src/dragonball/src/api/v1/mod.rs +++ b/src/dragonball/src/api/v1/mod.rs @@ -4,9 +4,7 @@ //! API Version 1 related data structures to configure the vmm. mod vmm_action; -pub use self::vmm_action::{ - VmmAction, VmmActionError, VmmData, VmmRequest, VmmResponse, VmmService, -}; +pub use self::vmm_action::*; /// Wrapper for configuring the microVM boot source. mod boot_source; diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 6dedbfc28e1d..2962994b4bb1 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -17,20 +17,20 @@ use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; #[cfg(feature = "virtio-blk")] -use crate::device_manager::blk_dev_mgr::{ +pub use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, }; #[cfg(feature = "virtio-fs")] -use crate::device_manager::fs_dev_mgr::{ +pub use crate::device_manager::fs_dev_mgr::{ FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo, }; #[cfg(feature = "virtio-net")] -use crate::device_manager::virtio_net_dev_mgr::{ +pub use crate::device_manager::virtio_net_dev_mgr::{ VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, VirtioNetDeviceMgr, }; #[cfg(feature = "virtio-vsock")] -use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; +pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; use super::*; diff --git a/src/dragonball/src/device_manager/vsock_dev_mgr.rs b/src/dragonball/src/device_manager/vsock_dev_mgr.rs index 5e49bbfb375b..4f0f07413400 100644 --- a/src/dragonball/src/device_manager/vsock_dev_mgr.rs +++ b/src/dragonball/src/device_manager/vsock_dev_mgr.rs @@ -88,6 +88,20 @@ pub struct VsockDeviceConfigInfo { pub use_generic_irq: Option, } +impl Default for VsockDeviceConfigInfo { + fn default() -> Self { + Self { + id: String::default(), + guest_cid: 0, + uds_path: None, + tcp_addr: None, + queue_size: Vec::from(QUEUE_SIZES), + use_shared_irq: None, + use_generic_irq: None, + } + } +} + impl VsockDeviceConfigInfo { /// Get number and size of queues supported. pub fn queue_sizes(&self) -> Vec { From bde6609b93c33db1c86e7cf3d40a0b9abce5110f Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 4 Jul 2022 19:47:14 +0800 Subject: [PATCH 0106/1953] hotplug: add room for other hotplug solution Add room in the code for other hotplug solution without upcall Signed-off-by: Chao Wu --- src/dragonball/src/device_manager/mod.rs | 4 ++-- src/dragonball/src/error.rs | 4 ++-- src/dragonball/src/vcpu/vcpu_manager.rs | 12 +++++++----- src/dragonball/src/vm/mod.rs | 22 ++++++++++++++++++---- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 53a1e2df92d4..af19fdeac35b 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -323,11 +323,11 @@ impl DeviceOpContext { } } -#[cfg(not(feature = "hotplug"))] +#[cfg(all(feature = "hotplug", not(feature = "dbs-upcall")))] impl DeviceOpContext { pub(crate) fn insert_hotplug_mmio_device( &self, - _dev: &Arc, + _dev: &Arc, _callback: Option<()>, ) -> Result<()> { Err(DeviceMgrError::InvalidOperation) diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 2c6fbf6c4402..b4ee40119456 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -149,12 +149,12 @@ pub enum StartMicroVmError { #[error("vCPU related error: {0}")] Vcpu(#[source] vcpu::VcpuManagerError), - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] /// Upcall initialize Error. #[error("failure while initializing the upcall client: {0}")] UpcallInitError(#[source] dbs_upcall::UpcallClientError), - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] /// Upcall connect Error. #[error("failure while connecting the upcall client: {0}")] UpcallConnectError(#[source] dbs_upcall::UpcallClientError), diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 278a88ecea1e..f6f3e93ffac2 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -147,6 +147,7 @@ pub enum VcpuResizeError { #[error("Removable vcpu not enough, removable vcpu num: {0}, number to remove: {1}, present vcpu count {2}")] LackRemovableVcpus(u16, u16, u16), + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] /// Cannot update the configuration by upcall channel. #[error("cannot update the configuration by upcall channel: {0}")] Upcall(#[source] dbs_upcall::UpcallClientError), @@ -782,13 +783,14 @@ impl VcpuManager { #[cfg(feature = "hotplug")] mod hotplug { - use std::cmp::Ordering; - - use dbs_upcall::{CpuDevRequest, DevMgrRequest}; - + #[cfg(feature = "dbs-upcall")] use super::*; + #[cfg(feature = "dbs-upcall")] + use dbs_upcall::{CpuDevRequest, DevMgrRequest}; + #[cfg(feature = "dbs-upcall")] + use std::cmp::Ordering; - #[cfg(all(target_arch = "x86_64"))] + #[cfg(all(target_arch = "x86_64", feature = "dbs-upcall"))] use dbs_boot::mptable::APIC_VERSION; #[cfg(all(target_arch = "aarch64"))] const APIC_VERSION: u8 = 0; diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 8c3cb21c9745..31b41baed277 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -20,7 +20,7 @@ use slog::{error, info}; use vm_memory::{Bytes, GuestAddress, GuestAddressSpace}; use vmm_sys_util::eventfd::EventFd; -#[cfg(feature = "hotplug")] +#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] use dbs_upcall::{DevMgrService, UpcallClient}; use crate::address_space_manager::{ @@ -195,7 +195,7 @@ pub struct Vm { #[cfg(target_arch = "aarch64")] irqchip_handle: Option>, - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_client: Option>>, } @@ -240,7 +240,7 @@ impl Vm { #[cfg(target_arch = "aarch64")] irqchip_handle: None, - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_client: None, }) } @@ -305,7 +305,7 @@ impl Vm { /// returns true if system upcall service is ready pub fn is_upcall_client_ready(&self) -> bool { - #[cfg(feature = "hotplug")] + #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] { if let Some(upcall_client) = self.upcall_client() { return upcall_client.is_ready(); @@ -705,6 +705,7 @@ impl Vm { .map_err(StartMicroVmError::Vcpu)?; self.init_microvm(event_mgr.epoll_manager(), vm_as.clone(), request_ts)?; self.init_configure_system(&vm_as)?; + #[cfg(feature = "dbs-upcall")] self.init_upcall()?; info!(self.logger, "VM: register events"); @@ -730,6 +731,7 @@ impl Vm { #[cfg(feature = "hotplug")] impl Vm { /// initialize upcall client for guest os + #[cfg(feature = "dbs-upcall")] fn new_upcall(&mut self) -> std::result::Result<(), StartMicroVmError> { // get vsock inner connector for upcall let inner_connector = self @@ -752,6 +754,7 @@ impl Vm { Ok(()) } + #[cfg(feature = "dbs-upcall")] fn init_upcall(&mut self) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM upcall init"); if let Err(e) = self.new_upcall() { @@ -769,10 +772,12 @@ impl Vm { } /// Get upcall client. + #[cfg(feature = "dbs-upcall")] pub fn upcall_client(&self) -> &Option>> { &self.upcall_client } + #[cfg(feature = "dbs-upcall")] fn create_device_hotplug_context( &self, epoll_mgr: Option, @@ -785,6 +790,15 @@ impl Vm { Err(StartMicroVmError::UpcallNotReady) } } + + // We will support hotplug without upcall in future stages. + #[cfg(not(feature = "dbs-upcall"))] + fn create_device_hotplug_context( + &self, + _epoll_mgr: Option, + ) -> std::result::Result { + Err(StartMicroVmError::MicroVMAlreadyRunning) + } } #[cfg(not(feature = "hotplug"))] From cb54ac6c6e2182b32acda86980073c5621b5dc53 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 4 Jul 2022 20:27:04 +0800 Subject: [PATCH 0107/1953] memory: remove reserve_memory_bytes This is currently an unsupported feature and we will remove it from the current code. Signed-off-by: Chao Wu --- src/dragonball/src/api/v1/vmm_action.rs | 10 ---------- src/dragonball/src/vm/mod.rs | 13 +------------ src/dragonball/src/vm/x86_64.rs | 13 ------------- 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 2962994b4bb1..1bd40cad01f6 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -428,16 +428,6 @@ impl VmmService { config.mem_file_path = machine_config.mem_file_path.clone(); - let reserve_memory_bytes = machine_config.reserve_memory_bytes; - // Reserved memory must be 2MB aligned and less than half of the total memory. - if reserve_memory_bytes % 0x200000 != 0 - || reserve_memory_bytes > (config.mem_size_mib as u64) << 20 - { - return Err(MachineConfig(InvalidReservedMemorySize( - reserve_memory_bytes as usize >> 20, - ))); - } - config.reserve_memory_bytes = reserve_memory_bytes; if config.mem_type == "hugetlbfs" && config.mem_file_path.is_empty() { return Err(MachineConfig(InvalidMemFilePath("".to_owned()))); } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 31b41baed277..e7248a8113a9 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -125,8 +125,6 @@ pub struct VmConfigInfo { pub mem_file_path: String, /// The memory size in MiB. pub mem_size_mib: usize, - /// reserve memory bytes - pub reserve_memory_bytes: u64, /// sock path pub serial_path: Option, @@ -149,7 +147,6 @@ impl Default for VmConfigInfo { mem_type: String::from("shmem"), mem_file_path: String::from(""), mem_size_mib: 128, - reserve_memory_bytes: 0, serial_path: None, } } @@ -510,12 +507,6 @@ impl Vm { // vcpu boot up require local memory. reserve 100 MiB memory let mem_size = (self.vm_config.mem_size_mib as u64) << 20; - let reserve_memory_bytes = self.vm_config.reserve_memory_bytes; - if reserve_memory_bytes > (mem_size >> 1) as u64 { - return Err(StartMicroVmError::ConfigureInvalid(String::from( - "invalid reserve_memory_bytes", - ))); - } let mem_type = self.vm_config.mem_type.clone(); let mut mem_file_path = String::from(""); @@ -543,12 +534,10 @@ impl Vm { info!( self.logger, - "VM: mem_type:{} mem_file_path:{}, mem_size:{}, reserve_memory_bytes:{}, \ - numa_regions:{:?}", + "VM: mem_type:{} mem_file_path:{}, mem_size:{}, numa_regions:{:?}", mem_type, mem_file_path, mem_size, - reserve_memory_bytes, numa_regions, ); diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index 3fcebdc20fe8..96ca0acb1fc2 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -48,7 +48,6 @@ fn configure_system( initrd: &Option, boot_cpus: u8, max_cpus: u8, - rsv_mem_bytes: u64, ) -> super::Result<()> { const KERNEL_BOOT_FLAG_MAGIC: u16 = 0xaa55; const KERNEL_HDR_MAGIC: u32 = 0x5372_6448; @@ -111,17 +110,6 @@ fn configure_system( } } - // reserve memory from microVM. - if rsv_mem_bytes > 0 { - add_e820_entry( - &mut params.0, - mem_end.raw_value().max(mmio_end.raw_value()) + 1, - rsv_mem_bytes, - bootparam::E820_RESERVED, - ) - .map_err(Error::BootSystem)?; - } - let zero_page_addr = GuestAddress(layout::ZERO_PAGE_START); guest_mem .checked_offset(zero_page_addr, mem::size_of::()) @@ -237,7 +225,6 @@ impl Vm { &initrd, self.vm_config.vcpu_count, self.vm_config.max_vcpu_count, - self.vm_config.reserve_memory_bytes, ) .map_err(StartMicroVmError::ConfigureSystem) } From b646d7cb37d77fb1b928490c27e64942254da596 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 4 Jul 2022 20:59:45 +0800 Subject: [PATCH 0108/1953] config: remove ht_enabled Since cpu topology could tell whether hyper thread is enabled or not, we removed ht_enabled config from VmConfigInfo Signed-off-by: Chao Wu --- src/dragonball/src/vm/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index e7248a8113a9..7afdefedcc43 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -110,8 +110,6 @@ pub struct VmConfigInfo { pub vcpu_count: u8, /// Max number of vcpu can be added pub max_vcpu_count: u8, - /// Enable or disable hyperthreading. - pub ht_enabled: bool, /// cpu power management. pub cpu_pm: String, /// cpu topology information @@ -135,7 +133,6 @@ impl Default for VmConfigInfo { VmConfigInfo { vcpu_count: 1, max_vcpu_count: 1, - ht_enabled: false, cpu_pm: String::from("on"), cpu_topology: CpuTopology { threads_per_core: 1, From 5ea35ddcdce13a79c2b2fed02fbc1bf6c8d602a9 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Mon, 4 Jul 2022 23:50:25 +0800 Subject: [PATCH 0109/1953] refractor: remove redundant by_id remove redundant by_id in get_vm_by_id_mut and get_vm_by_id. They are optimized to get_vm_mut and get_vm. Signed-off-by: Chao Wu --- src/dragonball/src/api/v1/vmm_action.rs | 48 +++++++------------------ src/dragonball/src/event_manager.rs | 2 +- src/dragonball/src/vmm.rs | 6 ++-- 3 files changed, 16 insertions(+), 40 deletions(-) diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 1bd40cad01f6..ae9db4fb4c2e 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -279,9 +279,7 @@ impl VmmService { }; use super::VmmActionError::BootSource; - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(BootSource(UpdateNotAllowedPostBoot)); } @@ -315,9 +313,7 @@ impl VmmService { let vmm_seccomp_filter = vmm.vmm_seccomp_filter(); let vcpu_seccomp_filter = vmm.vcpu_seccomp_filter(); - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(StartMicroVm(MicroVMAlreadyRunning)); } @@ -342,9 +338,7 @@ impl VmmService { use self::VmConfigError::*; use self::VmmActionError::MachineConfig; - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(MachineConfig(UpdateNotAllowedPostBoot)); } @@ -455,9 +449,7 @@ impl VmmService { #[cfg(feature = "virtio-vsock")] fn add_vsock_device(&self, vmm: &mut Vmm, config: VsockDeviceConfigInfo) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(VmmActionError::Vsock( VsockDeviceError::UpdateNotAllowedPostBoot, @@ -496,9 +488,7 @@ impl VmmService { event_mgr: &mut EventManager, config: BlockDeviceConfigInfo, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|e| { @@ -520,9 +510,7 @@ impl VmmService { vmm: &mut Vmm, config: BlockDeviceConfigUpdateInfo, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; BlockDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) .map(|_| VmmData::Empty) @@ -538,9 +526,7 @@ impl VmmService { event_mgr: &mut EventManager, drive_id: &str, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|_| VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot))?; @@ -557,9 +543,7 @@ impl VmmService { event_mgr: &mut EventManager, config: VirtioNetDeviceConfigInfo, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|e| { @@ -583,9 +567,7 @@ impl VmmService { vmm: &mut Vmm, config: VirtioNetDeviceConfigUpdateInfo, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; VirtioNetDeviceMgr::update_device_ratelimiters(vm.device_manager_mut(), config) .map(|_| VmmData::Empty) @@ -594,9 +576,7 @@ impl VmmService { #[cfg(feature = "virtio-fs")] fn add_fs_device(&mut self, vmm: &mut Vmm, config: FsDeviceConfigInfo) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; let hotplug = vm.is_vm_initialized(); if !cfg!(feature = "hotplug") && hotplug { return Err(VmmActionError::FsDevice( @@ -619,9 +599,7 @@ impl VmmService { vmm: &mut Vmm, config: FsMountConfigInfo, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if !vm.is_vm_initialized() { return Err(VmmActionError::FsDevice(FsDeviceError::MicroVMNotRunning)); @@ -638,9 +616,7 @@ impl VmmService { vmm: &mut Vmm, config: FsDeviceConfigUpdateInfo, ) -> VmmRequestResult { - let vm = vmm - .get_vm_by_id_mut("") - .ok_or(VmmActionError::InvalidVMID)?; + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if !vm.is_vm_initialized() { return Err(VmmActionError::FsDevice(FsDeviceError::MicroVMNotRunning)); diff --git a/src/dragonball/src/event_manager.rs b/src/dragonball/src/event_manager.rs index 9a2ad9d1c701..f07b786506bc 100644 --- a/src/dragonball/src/event_manager.rs +++ b/src/dragonball/src/event_manager.rs @@ -135,7 +135,7 @@ impl MutEventSubscriber for VmmEpollHandler { self.vmm_event_count.fetch_add(1, Ordering::AcqRel); } EPOLL_EVENT_EXIT => { - let vm = vmm.get_vm_by_id("").unwrap(); + let vm = vmm.get_vm().unwrap(); match vm.get_reset_eventfd() { Some(ev) => { if let Err(e) = ev.read() { diff --git a/src/dragonball/src/vmm.rs b/src/dragonball/src/vmm.rs index 362415dbd9dd..a25543e342e4 100644 --- a/src/dragonball/src/vmm.rs +++ b/src/dragonball/src/vmm.rs @@ -85,12 +85,12 @@ impl Vmm { } /// Get a reference to a virtual machine managed by the VMM. - pub fn get_vm_by_id(&self, _id: &str) -> Option<&Vm> { + pub fn get_vm(&self) -> Option<&Vm> { Some(&self.vm) } /// Get a mutable reference to a virtual machine managed by the VMM. - pub fn get_vm_by_id_mut(&mut self, _id: &str) -> Option<&mut Vm> { + pub fn get_vm_mut(&mut self) -> Option<&mut Vm> { Some(&mut self.vm) } @@ -156,7 +156,7 @@ impl Vmm { /// Waits for all vCPUs to exit and terminates the Dragonball process. fn stop(&mut self, exit_code: i32) -> i32 { info!("Vmm is stopping."); - if let Some(vm) = self.get_vm_by_id_mut("") { + if let Some(vm) = self.get_vm_mut() { if vm.is_vm_initialized() { if let Err(e) = vm.remove_devices() { warn!("failed to remove devices: {:?}", e); From 6a1fe85f1092ee4e8580bffeefd54f2f65e88e7c Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Wed, 6 Jul 2022 01:01:05 +0800 Subject: [PATCH 0110/1953] vfio: add vfio as TODO We add vfio as TODO in this commit and create a github issue for this. Signed-off-by: Chao Wu --- src/dragonball/src/device_manager/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index af19fdeac35b..b8e1ea9969be 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -631,6 +631,7 @@ impl DeviceManager { /// Start all registered devices when booting the associated virtual machine. pub fn start_devices(&mut self) -> std::result::Result<(), StartMicroVmError> { + // TODO: add vfio support here. issue #4589. Ok(()) } From 5d3b53ee7beb3107277df849db9598347ed737ea Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Wed, 6 Jul 2022 01:01:39 +0800 Subject: [PATCH 0111/1953] downtime: add downtime support add downtime support in `resume_all_vcpus_with_downtime` Signed-off-by: Chao Wu --- src/dragonball/src/api/v1/instance_info.rs | 4 ++++ src/dragonball/src/vm/mod.rs | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/dragonball/src/api/v1/instance_info.rs b/src/dragonball/src/api/v1/instance_info.rs index d457b6124ba3..ae159aa61405 100644 --- a/src/dragonball/src/api/v1/instance_info.rs +++ b/src/dragonball/src/api/v1/instance_info.rs @@ -54,6 +54,8 @@ pub struct InstanceInfo { pub async_state: AsyncState, /// List of tids of vcpu threads (vcpu index, tid) pub tids: Vec<(u8, u32)>, + /// Last instance downtime + pub last_instance_downtime: u64, } impl InstanceInfo { @@ -66,6 +68,7 @@ impl InstanceInfo { pid: std::process::id(), async_state: AsyncState::Uninitialized, tids: Vec::new(), + last_instance_downtime: 0, } } } @@ -79,6 +82,7 @@ impl Default for InstanceInfo { pid: std::process::id(), async_state: AsyncState::Uninitialized, tids: Vec::new(), + last_instance_downtime: 0, } } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 7afdefedcc43..e372af51e9f7 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -420,6 +420,18 @@ impl Vm { pub fn resume_all_vcpus_with_downtime(&mut self) -> std::result::Result<(), VcpuManagerError> { self.vcpu_manager()?.resume_all_vcpus()?; + if self.start_instance_downtime != 0 { + let now = TimestampUs::default(); + let downtime = now.time_us - self.start_instance_downtime; + info!(self.logger, "VM: instance downtime: {} us", downtime); + self.start_instance_downtime = 0; + if let Ok(mut info) = self.shared_info.write() { + info.last_instance_downtime = downtime; + } else { + error!(self.logger, "Failed to update live upgrade downtime, couldn't be written due to poisoned lock"); + } + } + Ok(()) } From e14e98bbeb6a2d0c141591275e55813c8d8a38e4 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Wed, 6 Jul 2022 11:20:50 +0800 Subject: [PATCH 0112/1953] cpu_topo: add handle_cpu_topology function add handle_cpu_topology funciton to make it easier to understand the set_vm_configuration function. Signed-off-by: Chao Wu --- src/dragonball/src/api/v1/vmm_action.rs | 56 ++++++++++++++----------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index ae9db4fb4c2e..4510b7e03d8a 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -16,6 +16,9 @@ use crate::event_manager::EventManager; use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; +use self::VmConfigError::*; +use self::VmmActionError::MachineConfig; + #[cfg(feature = "virtio-blk")] pub use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, @@ -335,9 +338,6 @@ impl VmmService { vmm: &mut Vmm, machine_config: VmConfigInfo, ) -> VmmRequestResult { - use self::VmConfigError::*; - use self::VmmActionError::MachineConfig; - let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(MachineConfig(UpdateNotAllowedPostBoot)); @@ -356,27 +356,7 @@ impl VmmService { if config.cpu_topology != machine_config.cpu_topology { let cpu_topology = &machine_config.cpu_topology; - // Check if dies_per_socket, cores_per_die, threads_per_core and socket number is valid - if cpu_topology.threads_per_core < 1 || cpu_topology.threads_per_core > 2 { - return Err(MachineConfig(InvalidThreadsPerCore( - cpu_topology.threads_per_core, - ))); - } - let vcpu_count_from_topo = cpu_topology - .sockets - .checked_mul(cpu_topology.dies_per_socket) - .ok_or(MachineConfig(VcpuCountExceedsMaximum))? - .checked_mul(cpu_topology.cores_per_die) - .ok_or(MachineConfig(VcpuCountExceedsMaximum))? - .checked_mul(cpu_topology.threads_per_core) - .ok_or(MachineConfig(VcpuCountExceedsMaximum))?; - if vcpu_count_from_topo > MAX_SUPPORTED_VCPUS { - return Err(MachineConfig(VcpuCountExceedsMaximum)); - } - if vcpu_count_from_topo < config.vcpu_count { - return Err(MachineConfig(InvalidCpuTopology(vcpu_count_from_topo))); - } - config.cpu_topology = cpu_topology.clone(); + config.cpu_topology = handle_cpu_topology(cpu_topology, config.vcpu_count)?.clone(); } else { // the same default let mut default_cpu_topology = CpuTopology { @@ -627,3 +607,31 @@ impl VmmService { .map_err(VmmActionError::FsDevice) } } + +fn handle_cpu_topology( + cpu_topology: &CpuTopology, + vcpu_count: u8, +) -> std::result::Result<&CpuTopology, VmmActionError> { + // Check if dies_per_socket, cores_per_die, threads_per_core and socket number is valid + if cpu_topology.threads_per_core < 1 || cpu_topology.threads_per_core > 2 { + return Err(MachineConfig(InvalidThreadsPerCore( + cpu_topology.threads_per_core, + ))); + } + let vcpu_count_from_topo = cpu_topology + .sockets + .checked_mul(cpu_topology.dies_per_socket) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))? + .checked_mul(cpu_topology.cores_per_die) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))? + .checked_mul(cpu_topology.threads_per_core) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))?; + if vcpu_count_from_topo > MAX_SUPPORTED_VCPUS { + return Err(MachineConfig(VcpuCountExceedsMaximum)); + } + if vcpu_count_from_topo < vcpu_count { + return Err(MachineConfig(InvalidCpuTopology(vcpu_count_from_topo))); + } + + Ok(cpu_topology) +} From 47a4142e0d88eaf888269ae98b396911a44a23a2 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Wed, 6 Jul 2022 11:38:12 +0800 Subject: [PATCH 0113/1953] fs: change vhostuser and virtio into const change fs mode vhostuser and virtio into const. Signed-off-by: Chao Wu --- src/dragonball/src/device_manager/fs_dev_mgr.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/dragonball/src/device_manager/fs_dev_mgr.rs b/src/dragonball/src/device_manager/fs_dev_mgr.rs index d699dcbde573..088dc980f189 100644 --- a/src/dragonball/src/device_manager/fs_dev_mgr.rs +++ b/src/dragonball/src/device_manager/fs_dev_mgr.rs @@ -27,6 +27,10 @@ const USE_SHARED_IRQ: bool = true; const USE_GENERIC_IRQ: bool = true; // Default cache size is 2 Gi since this is a typical VM memory size. const DEFAULT_CACHE_SIZE: u64 = 2 * 1024 * 1024 * 1024; +// We have 2 supported fs device mode, vhostuser and virtio +const VHOSTUSER_FS_MODE: &str = "vhostuser"; +// We have 2 supported fs device mode, vhostuser and virtio +const VIRTIO_FS_MODE: &str = "virtio"; /// Errors associated with `FsDeviceConfig`. #[derive(Debug, thiserror::Error)] @@ -152,7 +156,7 @@ impl std::default::Default for FsDeviceConfigInfo { impl FsDeviceConfigInfo { /// The default mode is set to 'virtio' for 'virtio-fs' device. pub fn default_fs_mode() -> String { - String::from("virtio") + String::from(VIRTIO_FS_MODE) } /// The default cache policy @@ -227,7 +231,7 @@ impl ConfigItem for FsDeviceConfigInfo { fn check_conflicts(&self, other: &Self) -> Result<(), FsDeviceError> { if self.tag == other.tag { Err(FsDeviceError::FsDeviceTagAlreadyExists(self.tag.clone())) - } else if self.mode.as_str() == "vhostuser" && self.sock_path == other.sock_path { + } else if self.mode.as_str() == VHOSTUSER_FS_MODE && self.sock_path == other.sock_path { Err(FsDeviceError::FsDevicePathAlreadyExists( self.sock_path.clone(), )) @@ -354,8 +358,8 @@ impl FsDeviceMgr { ctx: &mut DeviceOpContext, epoll_mgr: EpollManager, ) -> std::result::Result { - match config.mode.as_str() { - "virtio" => Self::attach_virtio_fs_devices(config, ctx, epoll_mgr), + match &config.mode as &str { + VIRTIO_FS_MODE => Self::attach_virtio_fs_devices(config, ctx, epoll_mgr), _ => Err(FsDeviceError::CreateFsDevice(virtio::Error::InvalidInput)), } } From 9cee52153b263d9a7267f9a6df1fc493b676b941 Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Wed, 6 Jul 2022 11:58:21 +0800 Subject: [PATCH 0114/1953] fmt: do cargo fmt and add a dependency for blk_dev fmt: do cargo fmt and add a dependency for blk_dev Signed-off-by: Chao Wu --- src/dragonball/src/device_manager/blk_dev_mgr.rs | 1 + src/dragonball/src/vcpu/aarch64.rs | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 0d35c4f95f03..9b3ee9041748 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -21,6 +21,7 @@ use serde_derive::{Deserialize, Serialize}; use crate::address_space_manager::GuestAddressSpaceImpl; use crate::config_manager::{ConfigItem, DeviceConfigInfo, RateLimiterConfigInfo}; +use crate::device_manager::blk_dev_mgr::BlockDeviceError::InvalidDeviceId; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; use crate::get_bucket_update; use crate::vm::KernelConfigInfo; diff --git a/src/dragonball/src/vcpu/aarch64.rs b/src/dragonball/src/vcpu/aarch64.rs index 759774a62090..054a1f65d4ba 100644 --- a/src/dragonball/src/vcpu/aarch64.rs +++ b/src/dragonball/src/vcpu/aarch64.rs @@ -6,20 +6,20 @@ // Use of this source code is governed by a BSD-style license that can be // found in the THIRD-PARTY file. +use std::ops::Deref; use std::sync::mpsc::{channel, Sender}; use std::sync::Arc; -use std::ops::Deref; use crate::IoManagerCached; -use dbs_utils::time::TimestampUs; use dbs_arch::regs; use dbs_boot::get_fdt_addr; +use dbs_utils::time::TimestampUs; use kvm_ioctls::{VcpuFd, VmFd}; use vm_memory::{Address, GuestAddress, GuestAddressSpace}; use vmm_sys_util::eventfd::EventFd; use crate::address_space_manager::GuestAddressSpaceImpl; -use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuStateEvent, VcpuError}; +use crate::vcpu::vcpu_impl::{Result, Vcpu, VcpuError, VcpuStateEvent}; use crate::vcpu::VcpuConfig; #[allow(unused)] @@ -111,8 +111,7 @@ impl Vcpu { .map_err(VcpuError::REGSConfiguration)?; } - self.mpidr = - regs::read_mpidr(&self.fd).map_err(VcpuError::REGSConfiguration)?; + self.mpidr = regs::read_mpidr(&self.fd).map_err(VcpuError::REGSConfiguration)?; Ok(()) } From 7a4183980e25f02daf92847736eb5654ac442755 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Tue, 28 Jun 2022 11:53:29 +0800 Subject: [PATCH 0115/1953] dragonball: add device info support for aarch64 Implement generate_virtio_device_info() and get_virtio_mmio_device_info() functions su support the mmio_device_info member, which is used by FDT. Fixes: #4544 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/device_manager/mod.rs | 55 ++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index b8e1ea9969be..0ce6af60622d 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -5,6 +5,7 @@ use std::io; use std::sync::{Arc, Mutex, MutexGuard}; +use std::collections::HashMap; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; @@ -321,6 +322,33 @@ impl DeviceOpContext { Ok(()) } + + #[cfg(target_arch = "aarch64")] + fn generate_virtio_device_info(&self) -> Result> { + let mut dev_info = HashMap::new(); + #[cfg(feature = "dbs-virtio-devices")] + for (_index, device) in self.virtio_devices.iter().enumerate() { + let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_mmio_device_info(device)?; + let dev_type; + let device_id; + if let Some(mmiov2_device) = + device.as_any().downcast_ref::() + { + dev_type = mmiov2_device.get_device_type(); + device_id = None; + } else { + return Err(DeviceMgrError::InvalidOperation); + } + dev_info.insert( + ( + DeviceType::Virtio(dev_type), + format!("virtio-{}@0x{:08x?}", dev_type, mmio_base), + ), + MMIODeviceInfo::new(mmio_base, mmio_size, vec![irq], device_id), + ); + } + Ok(dev_info) + } } #[cfg(all(feature = "hotplug", not(feature = "dbs-upcall")))] @@ -626,6 +654,14 @@ impl DeviceManager { ctx.generate_kernel_boot_args(kernel_config) .map_err(StartMicroVmError::DeviceManager)?; + #[cfg(target_arch = "aarch64")] + { + let dev_info = ctx + .generate_virtio_device_info() + .map_err(StartMicrovmError::DeviceManager)?; + self.mmio_device_info.extend(dev_info); + } + Ok(()) } @@ -679,6 +715,25 @@ impl DeviceManager { pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { Some(&self.mmio_device_info) } + + #[cfg(feature = "dbs-virtio-devices")] + fn get_virtio_mmio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { + let resources = device.get_assigned_resources(); + let irq = resources + .get_legacy_irq() + .ok_or(DeviceMgrError::GetDeviceResource)?; + + if let Some(mmio_dev) = device + .as_any() + .downcast_ref::() + { + if let Resource::MmioAddressRange { base, size } = mmio_dev.get_mmio_cfg_res() { + return Ok((base, size, irq)); + } + } + + Err(DeviceMgrError::GetDeviceResource) + } } #[cfg(feature = "dbs-virtio-devices")] From f6f96b8fee9c4bbfd0a28ae826b34f8510b5e8ed Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Sun, 10 Jul 2022 17:35:30 +0800 Subject: [PATCH 0116/1953] dragonball: add legacy device support for aarch64 Implement RTC device for aarch64. Fixes: #4544 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/device_manager/legacy.rs | 83 ++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 102 +++++++++++++++++++- src/dragonball/src/error.rs | 6 +- src/dragonball/src/vm/aarch64.rs | 30 +++--- 4 files changed, 201 insertions(+), 20 deletions(-) diff --git a/src/dragonball/src/device_manager/legacy.rs b/src/dragonball/src/device_manager/legacy.rs index 9dbb300d4b5f..b3e33a7f223a 100644 --- a/src/dragonball/src/device_manager/legacy.rs +++ b/src/dragonball/src/device_manager/legacy.rs @@ -13,6 +13,8 @@ use std::sync::{Arc, Mutex}; use dbs_device::device_manager::Error as IoManagerError; use dbs_legacy_devices::SerialDevice; +#[cfg(target_arch = "aarch64")] +use dbs_legacy_devices::RTCDevice; use vmm_sys_util::eventfd::EventFd; // The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself. @@ -42,6 +44,10 @@ pub enum Error { pub struct LegacyDeviceManager { #[cfg(target_arch = "x86_64")] i8042_reset_eventfd: EventFd, + #[cfg(target_arch = "aarch64")] + pub(crate) _rtc_device: Arc>, + #[cfg(target_arch = "aarch64")] + _rtc_eventfd: EventFd, pub(crate) com1_device: Arc>, _com1_eventfd: EventFd, pub(crate) com2_device: Arc>, @@ -140,6 +146,83 @@ pub(crate) mod x86_64 { } } +#[cfg(target_arch = "aarch64")] +pub(crate) mod aarch64 { + use super::*; + use dbs_device::device_manager::{IoManager}; + use dbs_device::resources::DeviceResources; + use std::collections::HashMap; + use kvm_ioctls::VmFd; + + type Result = ::std::result::Result; + + impl LegacyDeviceManager { + pub fn create_manager( + bus: &mut IoManager, + vm_fd: Option>, + resources: &HashMap, + ) -> Result { + let (com1_device, com1_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com1").unwrap())?; + let (com2_device, com2_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com2").unwrap())?; + let (rtc_device, rtc_eventfd) = + Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get("rtc").unwrap())?; + + Ok(LegacyDeviceManager { + _rtc_device: rtc_device, + _rtc_eventfd: rtc_eventfd, + com1_device, + _com1_eventfd: com1_eventfd, + com2_device, + _com2_eventfd: com2_eventfd, + }) + } + + fn create_com_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + resources: &DeviceResources, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(SerialDevice::new( + eventfd.try_clone().map_err(Error::EventFd)? + ))); + + bus.register_device_io(device.clone(), resources.get_all_resources()) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + let irq = resources.get_legacy_irq().unwrap(); + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + + fn create_rtc_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + resources: &DeviceResources, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(RTCDevice::new())); + + bus.register_device_io(device.clone(), resources.get_all_resources()) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + let irq = resources.get_legacy_irq().unwrap(); + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + } +} + #[cfg(test)] mod tests { #[cfg(target_arch = "x86_64")] diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 0ce6af60622d..1e52600e5da4 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -13,6 +13,8 @@ use dbs_address_space::AddressSpace; use dbs_arch::{DeviceType, MMIODeviceInfo}; use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext}; use dbs_device::resources::Resource; +#[cfg(target_arch = "aarch64")] +use dbs_device::resources::DeviceResources; use dbs_device::DeviceIo; use dbs_interrupt::KvmIrqManager; use dbs_legacy_devices::ConsoleHandler; @@ -532,11 +534,31 @@ impl DeviceManager { &mut self, ctx: &mut DeviceOpContext, ) -> std::result::Result<(), StartMicroVmError> { - #[cfg(target_arch = "x86_64")] + #[cfg( + any(target_arch = "x86_64", + all(target_arch = "aarch64", feature = "dbs-virtio-devices") + ) + )] { let mut tx = ctx.io_context.begin_tx(); - let legacy_manager = - LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + let legacy_manager; + + #[cfg(target_arch = "x86_64")] + { + let legacy_manager = + LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + } + + #[cfg(target_arch = "aarch64")] + #[cfg(feature = "dbs-virtio-devices")] + { + let resources = self.get_legacy_resources()?; + legacy_manager = LegacyDeviceManager::create_manager( + &mut tx.io_manager, + Some(self.vm_fd.clone()), + &resources, + ); + } match legacy_manager { Ok(v) => { @@ -658,7 +680,7 @@ impl DeviceManager { { let dev_info = ctx .generate_virtio_device_info() - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; self.mmio_device_info.extend(dev_info); } @@ -716,6 +738,78 @@ impl DeviceManager { Some(&self.mmio_device_info) } + #[cfg(feature = "dbs-virtio-devices")] + fn get_legacy_resources( + &mut self, + ) -> std::result::Result, StartMicroVmError> { + let mut resources = HashMap::new(); + let legacy_devices = vec![ + (DeviceType::Serial, String::from("com1")), + (DeviceType::Serial, String::from("com2")), + (DeviceType::RTC, String::from("rtc")), + ]; + + for (device_type, device_id) in legacy_devices { + let res = self.allocate_mmio_device_resource()?; + self.add_mmio_device_info(&res, device_type, device_id.clone(), None); + resources.insert(device_id.clone(), res); + } + + Ok(resources) + } + + fn mmio_device_info_to_resources( + &self, + key: &(DeviceType, String), + ) -> std::result::Result { + self.mmio_device_info + .get(key) + .map(|info| { + let mut resources = DeviceResources::new(); + resources.append(Resource::LegacyIrq(info.irqs[0])); + resources.append(Resource::MmioAddressRange { + base: info.base, + size: info.size, + }); + resources + }) + .ok_or(StartMicroVmError::DeviceManager( + DeviceMgrError::GetDeviceResource, + )) + } + + #[cfg(feature = "dbs-virtio-devices")] + fn allocate_mmio_device_resource( + &self, + ) -> std::result::Result { + let mut requests = Vec::new(); + requests.push(ResourceConstraint::MmioAddress { + range: None, + align: MMIO_DEFAULT_CFG_SIZE, + size: MMIO_DEFAULT_CFG_SIZE, + }); + requests.push(ResourceConstraint::LegacyIrq { irq: None }); + + self.res_manager + .allocate_device_resources(&requests, false) + .map_err(StartMicroVmError::AllocateResource) + } + + fn add_mmio_device_info( + &mut self, + resource: &DeviceResources, + device_type: DeviceType, + device_id: String, + msi_device_id: Option, + ) { + let (base, size) = resource.get_mmio_address_ranges()[0]; + let irq = resource.get_legacy_irq().unwrap(); + self.mmio_device_info.insert( + (device_type, device_id), + MMIODeviceInfo::new(base, size, vec![irq], msi_device_id), + ); + } + #[cfg(feature = "dbs-virtio-devices")] fn get_virtio_mmio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { let resources = device.get_assigned_resources(); diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index b4ee40119456..454ad45b59f0 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,7 +12,7 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; -use crate::{address_space_manager, device_manager, vcpu, vm}; +use crate::{address_space_manager, device_manager, vcpu, vm, resource_manager}; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; @@ -73,6 +73,10 @@ pub enum Error { /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] pub enum StartMicroVmError { + /// Failed to allocate resources. + #[error("cannot allocate resources")] + AllocateResource(#[source] resource_manager::ResourceError), + /// Cannot read from an Event file descriptor. #[error("failure while reading from EventFd file descriptor")] EventFd, diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 25450f380d74..ffecaf356feb 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -21,7 +21,7 @@ use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; -use crate::error::{Error, StartMicrovmError}; +use crate::error::{Error, StartMicroVmError}; use crate::event_manager::EventManager; /// Configures the system and should be called once per vm before starting vcpu threads. @@ -63,12 +63,12 @@ impl Vm { } /// Creates the irq chip in-kernel device model. - pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> { let vcpu_count = self.vm_config.vcpu_count; self.irqchip_handle = Some( dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupGIC(e)))?, ); Ok(()) @@ -88,16 +88,16 @@ impl Vm { epoll_mgr: EpollManager, vm_as: GuestAddressSpaceImpl, request_ts: TimestampUs, - ) -> Result<(), StartMicrovmError> { + ) -> Result<(), StartMicroVmError> { let reset_eventfd = - EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?; + EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicroVmError::EventFd)?; self.reset_eventfd = Some( reset_eventfd .try_clone() - .map_err(|_| StartMicrovmError::EventFd)?, + .map_err(|_| StartMicroVmError::EventFd)?, ); self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .set_reset_event_fd(reset_eventfd); // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before @@ -106,9 +106,9 @@ impl Vm { // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; self.setup_interrupt_controller()?; self.init_devices(epoll_mgr)?; @@ -124,8 +124,8 @@ impl Vm { vm_memory: &GuestMemoryImpl, cmdline: &Cmdline, initrd: Option, - ) -> std::result::Result<(), StartMicrovmError> { - let vcpu_manager = self.vcpu_manager().map_err(StartMicrovmError::Vcpu)?; + ) -> std::result::Result<(), StartMicroVmError> { + let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; let vcpu_mpidr = vcpu_manager .vcpus() .into_iter() @@ -141,17 +141,17 @@ impl Vm { self.get_irqchip(), &initrd, ) - .map_err(StartMicrovmError::ConfigureSystem) + .map_err(StartMicroVmError::ConfigureSystem) } pub(crate) fn register_events( &mut self, event_mgr: &mut EventManager, - ) -> std::result::Result<(), StartMicrovmError> { - let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?; + ) -> std::result::Result<(), StartMicroVmError> { + let reset_evt = self.get_reset_eventfd().ok_or(StartMicroVmError::EventFd)?; event_mgr .register_exit_eventfd(reset_evt) - .map_err(|_| StartMicrovmError::RegisterEvent)?; + .map_err(|_| StartMicroVmError::RegisterEvent)?; Ok(()) } From 939959e72600d49e450532e4a8567594df91d7a3 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 6 Jun 2022 20:26:45 +0800 Subject: [PATCH 0117/1953] docs: add Dragonball to hypervisors Fixes:#4193 Signed-off-by: Zhongtao Hu --- docs/hypervisors.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/hypervisors.md b/docs/hypervisors.md index 02dd49aa126b..e380450b203d 100644 --- a/docs/hypervisors.md +++ b/docs/hypervisors.md @@ -33,6 +33,7 @@ are available, their default values and how each setting can be used. [Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-clh.toml` | [Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-fc.toml` | [QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` | +[`Dragonball`] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-dragonball.toml` | ## Determine currently configured hypervisor @@ -52,6 +53,7 @@ the hypervisors: [Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | | [Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | | [QEMU] | Lots of features | Lots | | good | good | Good option for most users | | All users | +[`Dragonball`] | Built-in VMM, low CPU and memory overhead| Minimal | | excellent | excellent | Optimized for most container workloads | `out-of-the-box` Kata Containers experience | For further details, see the [Virtualization in Kata Containers](design/virtualization.md) document and the official documentation for each hypervisor. @@ -60,3 +62,4 @@ For further details, see the [Virtualization in Kata Containers](design/virtuali [Firecracker]: https://github.com/firecracker-microvm/firecracker [KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine [QEMU]: http://www.qemu-project.org +[`Dragonball`]: https://github.com/openanolis/dragonball-sandbox From 458f6f42f69a99cefd4c53ab0d279a5e56dec84c Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 11 Jul 2022 17:46:10 +0800 Subject: [PATCH 0118/1953] dragonball: use const string for legacy device type As string "com1", "com2" and "rtc" are used in two files (device_manager/mod.rs and device_manager/legacy.rs), we use public const variables COM1, COM2 and RTC to replace them respectively. Fixes: #4544 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/device_manager/legacy.rs | 14 +++++++++++--- src/dragonball/src/device_manager/mod.rs | 8 ++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/dragonball/src/device_manager/legacy.rs b/src/dragonball/src/device_manager/legacy.rs index b3e33a7f223a..59f4206ab9e8 100644 --- a/src/dragonball/src/device_manager/legacy.rs +++ b/src/dragonball/src/device_manager/legacy.rs @@ -156,18 +156,26 @@ pub(crate) mod aarch64 { type Result = ::std::result::Result; + /// LegacyDeviceType: com1 + pub const COM1: &str = "com1"; + /// LegacyDeviceType: com2 + pub const COM2: &str = "com2"; + /// LegacyDeviceType: rtc + pub const RTC: &str = "rtc"; + impl LegacyDeviceManager { + /// Create a LegacyDeviceManager instance handling legacy devices. pub fn create_manager( bus: &mut IoManager, vm_fd: Option>, resources: &HashMap, ) -> Result { let (com1_device, com1_eventfd) = - Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com1").unwrap())?; + Self::create_com_device(bus, vm_fd.as_ref(), resources.get(COM1).unwrap())?; let (com2_device, com2_eventfd) = - Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com2").unwrap())?; + Self::create_com_device(bus, vm_fd.as_ref(), resources.get(COM2).unwrap())?; let (rtc_device, rtc_eventfd) = - Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get("rtc").unwrap())?; + Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get(RTC).unwrap())?; Ok(LegacyDeviceManager { _rtc_device: rtc_device, diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 1e52600e5da4..d9d03947d105 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -54,7 +54,7 @@ pub mod console_manager; pub use self::console_manager::ConsoleManager; mod legacy; -pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; +pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager, aarch64::{COM1, COM2, RTC}}; #[cfg(feature = "virtio-vsock")] /// Device manager for user-space vsock devices. @@ -744,9 +744,9 @@ impl DeviceManager { ) -> std::result::Result, StartMicroVmError> { let mut resources = HashMap::new(); let legacy_devices = vec![ - (DeviceType::Serial, String::from("com1")), - (DeviceType::Serial, String::from("com2")), - (DeviceType::RTC, String::from("rtc")), + (DeviceType::Serial, String::from(COM1)), + (DeviceType::Serial, String::from(COM2)), + (DeviceType::RTC, String::from(RTC)), ]; for (device_type, device_id) in legacy_devices { From d2584991eb9d133312d6dd86fee56d553a8bd398 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 11 Jul 2022 17:55:04 +0800 Subject: [PATCH 0119/1953] dragonball: fix dependency unused warning Fix the warning "unused import: `dbs_arch::gic::Error as GICError`" and "unused import: `dbs_arch::gic::GICDevice`" in file src/vm/mod.rs when compiling. Fixes: #4544 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/vm/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index e372af51e9f7..e8c6d6446ef5 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -63,7 +63,7 @@ pub enum VmError { /// Cannot setup GIC #[cfg(target_arch = "aarch64")] #[error("failed to configure GIC")] - SetupGIC(dbs_arch::gic::Error), + SetupGIC(GICError), } /// Configuration information for user defined NUMA nodes. @@ -187,7 +187,7 @@ pub struct Vm { // Arm specific fields. // On aarch64 we need to keep around the fd obtained by creating the VGIC device. #[cfg(target_arch = "aarch64")] - irqchip_handle: Option>, + irqchip_handle: Option>, #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] upcall_client: Option>>, From 9ae2a45b386164d8de005ed5f8fa2494050f167f Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Tue, 12 Jul 2022 10:03:24 -0700 Subject: [PATCH 0120/1953] cgroups: remove unnecessary get_paths() Change get_mounts to get paths from a borrowed argument rather than calling get_paths a second time. Fixes #3768 Signed-off-by: Derek Lee --- src/agent/rustjail/src/cgroups/fs/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 84aa9bc50c50..70168d2f0b50 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -911,9 +911,8 @@ pub fn get_paths() -> Result> { Ok(m) } -pub fn get_mounts() -> Result> { +pub fn get_mounts(paths: &HashMap) -> Result> { let mut m = HashMap::new(); - let paths = get_paths()?; for l in fs::read_to_string(MOUNTS)?.lines() { let p: Vec<&str> = l.splitn(2, " - ").collect(); @@ -951,7 +950,7 @@ impl Manager { let mut m = HashMap::new(); let paths = get_paths()?; - let mounts = get_mounts()?; + let mounts = get_mounts(&paths)?; for key in paths.keys() { let mnt = mounts.get(key); From 3f6123b4ddae3a2b50165152392e714da329b549 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Thu, 9 Jun 2022 08:03:04 +0800 Subject: [PATCH 0121/1953] libs: update configuration and annotations 1. support annotation for runtime.name, hypervisor_name, agent_name. 2. fix parse memory from annotation Signed-off-by: Zhongtao Hu --- src/libs/Cargo.lock | 19 ++- src/libs/kata-types/Cargo.toml | 1 + src/libs/kata-types/src/annotations/mod.rs | 115 +++++++++++------- src/libs/kata-types/src/config/runtime.rs | 4 + src/libs/kata-types/tests/test_config.rs | 68 ++++++----- .../tests/texture/configuration-anno-0.toml | 2 + .../tests/texture/configuration-anno-1.toml | 3 +- src/libs/logging/Cargo.toml | 2 +- 8 files changed, 128 insertions(+), 86 deletions(-) diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index a6f89948584b..e5d9cb160e12 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -46,6 +46,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "byte-unit" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8" + [[package]] name = "byteorder" version = "1.4.3" @@ -388,6 +394,7 @@ dependencies = [ name = "kata-types" version = "0.1.0" dependencies = [ + "byte-unit", "glob", "lazy_static", "num_cpus", @@ -670,9 +677,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" dependencies = [ "serde", "serde_derive", @@ -680,18 +687,18 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c" +checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" dependencies = [ "protobuf", ] [[package]] name = "protobuf-codegen-pure" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440" +checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" dependencies = [ "protobuf", "protobuf-codegen", diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index 786615e55311..ce7dcaf069a8 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -11,6 +11,7 @@ license = "Apache-2.0" edition = "2018" [dependencies] +byte-unit = "3.1.4" glob = "0.3.0" lazy_static = "1.4.0" num_cpus = "1.13.1" diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index d06a67d4305b..0a517e22160a 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -14,6 +14,7 @@ use serde::Deserialize; use crate::config::hypervisor::get_hypervisor_plugin; use crate::config::TomlConfig; +use crate::sl; /// CRI-containerd specific annotations. pub mod cri_containerd; @@ -274,6 +275,13 @@ pub const KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P: &str = "io.katacontainers.config.hy // Runtime related annotations /// Prefix for Runtime configurations. pub const KATA_ANNO_CFG_RUNTIME_PREFIX: &str = "io.katacontainers.config.runtime."; +/// runtime name +pub const KATA_ANNO_CFG_RUNTIME_NAME: &str = "io.katacontainers.config.runtime.name"; +/// hypervisor name +pub const KATA_ANNO_CFG_RUNTIME_HYPERVISOR: &str = + "io.katacontainers.config.runtime.hypervisor_name"; +/// agent name +pub const KATA_ANNO_CFG_RUNTIME_AGENT: &str = "io.katacontainers.config.runtime.agent_name"; /// A sandbox annotation that determines if seccomp should be applied inside guest. pub const KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP: &str = "io.katacontainers.config.runtime.disable_guest_seccomp"; @@ -396,30 +404,24 @@ impl Annotation { impl Annotation { /// update config info by annotation - pub fn update_config_by_annotation( - &self, - config: &mut TomlConfig, - hypervisor_name: &str, - agent_name: &str, - ) -> Result<()> { - if config.hypervisor.get_mut(hypervisor_name).is_none() { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("hypervisor {} not found", hypervisor_name), - )); + pub fn update_config_by_annotation(&self, config: &mut TomlConfig) -> Result<()> { + if let Some(hv) = self.annotations.get(KATA_ANNO_CFG_RUNTIME_HYPERVISOR) { + if config.hypervisor.get(hv).is_some() { + config.runtime.hypervisor_name = hv.to_string(); + } } - - if config.agent.get_mut(agent_name).is_none() { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("agent {} not found", agent_name), - )); + if let Some(ag) = self.annotations.get(KATA_ANNO_CFG_RUNTIME_AGENT) { + if config.agent.get(ag).is_some() { + config.runtime.agent_name = ag.to_string(); + } } + let hypervisor_name = &config.runtime.hypervisor_name; + let agent_name = &config.runtime.agent_name; + let bool_err = io::Error::new(io::ErrorKind::InvalidData, "parse bool error".to_string()); let u32_err = io::Error::new(io::ErrorKind::InvalidData, "parse u32 error".to_string()); let u64_err = io::Error::new(io::ErrorKind::InvalidData, "parse u64 error".to_string()); let i32_err = io::Error::new(io::ErrorKind::InvalidData, "parse i32 error".to_string()); - let mut hv = config.hypervisor.get_mut(hypervisor_name).unwrap(); let mut ag = config.agent.get_mut(agent_name).unwrap(); for (key, value) in &self.annotations { @@ -632,32 +634,40 @@ impl Annotation { hv.machine_info.entropy_source = value.to_string(); } // Hypervisor Memory related annotations - KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => match self.get_value::(key) { - Ok(r) => { - let mem = r.unwrap_or_default(); - if mem - < get_hypervisor_plugin(hypervisor_name) - .unwrap() - .get_min_memory() - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "Memory specified in annotation {} is less than minimum required {}", - mem, - get_hypervisor_plugin(hypervisor_name) - .unwrap() - .get_min_memory() - ), - )); - } else { - hv.memory_info.default_memory = mem; + KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => { + match byte_unit::Byte::from_str(value) { + Ok(mem_bytes) => { + let memory_size = mem_bytes + .get_adjusted_unit(byte_unit::ByteUnit::MiB) + .get_value() + as u32; + info!(sl!(), "get mem {} from annotations: {}", memory_size, value); + if memory_size + < get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_min_memory() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "memory specified in annotation {} is less than minimum limitation {}", + memory_size, + get_hypervisor_plugin(hypervisor_name) + .unwrap() + .get_min_memory() + ), + )); + } + hv.memory_info.default_memory = memory_size; + } + Err(error) => { + error!( + sl!(), + "failed to parse byte from string {} error {:?}", value, error + ); } } - Err(_e) => { - return Err(u32_err); - } - }, + } KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS => match self.get_value::(key) { Ok(v) => { hv.memory_info.memory_slots = v.unwrap_or_default(); @@ -829,7 +839,21 @@ impl Annotation { return Err(u32_err); } }, - //update runtume config + //update runtime config + KATA_ANNO_CFG_RUNTIME_NAME => { + let runtime = vec!["virt-container", "linux-container", "wasm-container"]; + if runtime.contains(&value.as_str()) { + config.runtime.name = value.to_string(); + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "runtime specified in annotation {} is not in {:?}", + &value, &runtime + ), + )); + } + } KATA_ANNO_CFG_DISABLE_GUEST_SECCOMP => match self.get_value::(key) { Ok(r) => { config.runtime.disable_guest_seccomp = r.unwrap_or_default(); @@ -876,10 +900,7 @@ impl Annotation { config.runtime.vfio_mode = value.to_string(); } _ => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("Annotation {} not enabled", key), - )); + warn!(sl!(), "Annotation {} not enabled", key); } } } diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index a39b71220591..ce8e9efa5902 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -21,6 +21,10 @@ pub struct Runtime { #[serde(default)] pub hypervisor_name: String, + /// Agent name + #[serde(default)] + pub agent_name: String, + /// If enabled, the runtime will log additional debug messages to the system log. #[serde(default, rename = "enable_debug")] pub debug: bool, diff --git a/src/libs/kata-types/tests/test_config.rs b/src/libs/kata-types/tests/test_config.rs index ad668920498e..b7d5f953b191 100644 --- a/src/libs/kata-types/tests/test_config.rs +++ b/src/libs/kata-types/tests/test_config.rs @@ -18,6 +18,7 @@ mod tests { KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES, + KATA_ANNO_CFG_RUNTIME_NAME, }; use kata_types::config::KataConfig; use kata_types::config::{QemuConfig, TomlConfig}; @@ -118,7 +119,7 @@ mod tests { ); anno_hash.insert( KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY.to_string(), - "100".to_string(), + "100MiB".to_string(), ); anno_hash.insert( KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS.to_string(), @@ -169,9 +170,7 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_ok()); + assert!(anno.update_config_by_annotation(&mut config).is_ok()); KataConfig::set_active_config(Some(config), "qemu", "agnet0"); if let Some(ag) = KataConfig::get_default_config().get_agent() { assert_eq!( @@ -292,9 +291,10 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_ok()); + if let Some(hv) = KataConfig::get_default_config().get_hypervisor() { + assert_eq!(hv.blockdev_info.block_device_driver, "virtio-blk"); + } } #[test] @@ -315,9 +315,10 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_ok()); + if let Some(hv) = KataConfig::get_default_config().get_hypervisor() { + assert!(hv.memory_info.enable_guest_swap) + } } #[test] @@ -341,9 +342,7 @@ mod tests { let path = Path::new(path).join("tests/texture/configuration-anno-0.toml"); let content = fs::read_to_string(&path).unwrap(); let mut config = TomlConfig::load(&content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } #[test] @@ -366,9 +365,7 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(&content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } #[test] @@ -388,9 +385,7 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } #[test] @@ -411,9 +406,7 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } #[test] @@ -434,9 +427,7 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } #[test] @@ -457,9 +448,7 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } #[test] @@ -480,8 +469,25 @@ mod tests { let anno = Annotation::new(anno_hash); let mut config = TomlConfig::load(content).unwrap(); - assert!(anno - .update_config_by_annotation(&mut config, "qemu", "agent0") - .is_err()); + assert!(anno.update_config_by_annotation(&mut config).is_err()); + } + + #[test] + fn test_fail_to_change_runtime_name() { + let content = include_str!("texture/configuration-anno-0.toml"); + + let qemu = QemuConfig::new(); + qemu.register(); + + let config = TomlConfig::load(content).unwrap(); + KataConfig::set_active_config(Some(config), "qemu", "agent0"); + let mut anno_hash = HashMap::new(); + anno_hash.insert( + KATA_ANNO_CFG_RUNTIME_NAME.to_string(), + "other-container".to_string(), + ); + let anno = Annotation::new(anno_hash); + let mut config = TomlConfig::load(content).unwrap(); + assert!(anno.update_config_by_annotation(&mut config).is_err()); } } diff --git a/src/libs/kata-types/tests/texture/configuration-anno-0.toml b/src/libs/kata-types/tests/texture/configuration-anno-0.toml index 30b199260832..807de57b69ae 100644 --- a/src/libs/kata-types/tests/texture/configuration-anno-0.toml +++ b/src/libs/kata-types/tests/texture/configuration-anno-0.toml @@ -84,5 +84,7 @@ sandbox_bind_mounts=["/proc/self"] vfio_mode="vfio" experimental=["a", "b"] enable_pprof = true +hypervisor_name = "qemu" +agent_name = "agent0" diff --git a/src/libs/kata-types/tests/texture/configuration-anno-1.toml b/src/libs/kata-types/tests/texture/configuration-anno-1.toml index 9a411bdff5af..66e11dbe5c17 100644 --- a/src/libs/kata-types/tests/texture/configuration-anno-1.toml +++ b/src/libs/kata-types/tests/texture/configuration-anno-1.toml @@ -83,5 +83,6 @@ sandbox_bind_mounts=["/proc/self"] vfio_mode="vfio" experimental=["a", "b"] enable_pprof = true - +hypervisor_name = "qemu" +agent_name = "agent0" diff --git a/src/libs/logging/Cargo.toml b/src/libs/logging/Cargo.toml index 36685c15a3ed..3457072bc690 100644 --- a/src/libs/logging/Cargo.toml +++ b/src/libs/logging/Cargo.toml @@ -12,7 +12,7 @@ serde_json = "1.0.73" # - Dynamic keys required to allow HashMap keys to be slog::Serialized. # - The 'max_*' features allow changing the log level at runtime # (by stopping the compiler from removing log calls). -slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] } +slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] } slog-json = "2.4.0" slog-async = "2.7.0" slog-scope = "4.4.0" From 3d6156f6ec71de5539bc442d4a19f3263ec76c1f Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Fri, 3 Dec 2021 18:53:48 +0800 Subject: [PATCH 0122/1953] runtime-rs: support dragonball and runtime-binary Fixes: #3785 Signed-off-by: Quanwei Zhou Signed-off-by: Zhongtao Hu --- src/agent/Cargo.lock | 7 + src/dragonball/src/device_manager/mod.rs | 2 +- src/dragonball/src/vm/mod.rs | 2 + .../kata-types/src/config/hypervisor/mod.rs | 4 + src/runtime-rs/Cargo.lock | 1271 +++++++++++++++-- src/runtime-rs/Cargo.toml | 12 +- src/runtime-rs/Makefile | 12 +- src/runtime-rs/crates/agent/Cargo.toml | 2 +- src/runtime-rs/crates/hypervisor/Cargo.toml | 9 + .../crates/hypervisor/src/dragonball/inner.rs | 309 ++++ .../hypervisor/src/dragonball/inner_device.rs | 316 ++++ .../src/dragonball/inner_hypervisor.rs | 137 ++ .../crates/hypervisor/src/dragonball/mod.rs | 130 ++ .../hypervisor/src/dragonball/vmm_instance.rs | 335 +++++ .../crates/hypervisor/src/kernel_param.rs | 177 +++ src/runtime-rs/crates/hypervisor/src/lib.rs | 17 +- src/runtime-rs/crates/hypervisor/src/utils.rs | 27 + .../network_info/network_info_from_link.rs | 55 +- .../src/network/network_model/route_model.rs | 4 +- .../src/network/network_with_netns.rs | 2 + .../resource/src/network/utils/address.rs | 28 +- .../crates/resource/src/rootfs/mod.rs | 6 +- .../resource/src/share_fs/share_virtio_fs.rs | 2 +- .../src/share_fs/share_virtio_fs_inline.rs | 10 +- .../share_fs/share_virtio_fs_standalone.rs | 1 + .../crates/resource/src/share_fs/utils.rs | 4 +- .../crates/runtimes/common/Cargo.toml | 2 +- .../runtimes/common/src/runtime_handler.rs | 9 +- .../runtimes/linux_container/Cargo.toml | 1 + .../runtimes/linux_container/src/lib.rs | 2 + src/runtime-rs/crates/runtimes/src/manager.rs | 7 +- .../src/container_manager/container.rs | 11 +- .../src/container_manager/container_inner.rs | 24 +- .../src/container_manager/manager.rs | 3 +- .../src/container_manager/process.rs | 37 +- .../crates/runtimes/virt_container/src/lib.rs | 40 +- .../runtimes/virt_container/src/sandbox.rs | 28 +- .../crates/runtimes/wasm_container/Cargo.toml | 1 + .../crates/runtimes/wasm_container/src/lib.rs | 3 +- src/runtime-rs/crates/service/Cargo.toml | 2 +- .../crates/service/src/task_service.rs | 5 +- src/runtime-rs/crates/shim/Cargo.toml | 4 +- src/runtime-rs/crates/shim/src/panic_hook.rs | 18 +- src/tools/agent-ctl/Cargo.lock | 13 - src/tools/runk/Cargo.lock | 101 +- src/tools/trace-forwarder/Cargo.lock | 4 +- 46 files changed, 2836 insertions(+), 360 deletions(-) create mode 100644 src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/kernel_param.rs create mode 100644 src/runtime-rs/crates/hypervisor/src/utils.rs diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index b927b597db52..fb10ad35b19d 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -98,6 +98,12 @@ version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +[[package]] +name = "byte-unit" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8" + [[package]] name = "byteorder" version = "1.4.3" @@ -683,6 +689,7 @@ dependencies = [ name = "kata-types" version = "0.1.0" dependencies = [ + "byte-unit", "glob", "lazy_static", "num_cpus", diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index d9d03947d105..c37191ab026c 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -365,7 +365,7 @@ impl DeviceOpContext { pub(crate) fn remove_hotplug_mmio_device( &self, - _dev: &Arc, + _dev: &Arc, _callback: Option<()>, ) -> Result<()> { Err(DeviceMgrError::InvalidOperation) diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index e8c6d6446ef5..f5f62a040775 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -728,6 +728,7 @@ impl Vm { #[cfg(feature = "hotplug")] impl Vm { + #[cfg(feature = "dbs-upcall")] /// initialize upcall client for guest os #[cfg(feature = "dbs-upcall")] fn new_upcall(&mut self) -> std::result::Result<(), StartMicroVmError> { @@ -769,6 +770,7 @@ impl Vm { } } + #[cfg(feature = "dbs-upcall")] /// Get upcall client. #[cfg(feature = "dbs-upcall")] pub fn upcall_client(&self) -> &Option>> { diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 34a8e4c19fcd..01ac15f66ee5 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -620,6 +620,10 @@ pub struct NetworkInfo { /// Default 0-sized value means unlimited rate. #[serde(default)] pub tx_rate_limiter_max_rate: u64, + + /// network queues + #[serde(default)] + pub network_queues: u32, } impl NetworkInfo { diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 8a59c5df06ef..47651d8c8985 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -2,6 +2,27 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "actix-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-rt" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000" +dependencies = [ + "actix-macros", + "futures-core", + "tokio", +] + [[package]] name = "addr2line" version = "0.17.0" @@ -50,9 +71,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + +[[package]] +name = "arc-swap" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" [[package]] name = "arc-swap" @@ -60,11 +87,23 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + [[package]] name = "async-trait" -version = "0.1.53" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ "proc-macro2", "quote", @@ -85,9 +124,9 @@ checksum = "cc17ab023b4091c10ff099f9deebaeeb59b5189df07e554c4fef042b70745d68" [[package]] name = "backtrace" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" dependencies = [ "addr2line", "cc", @@ -104,6 +143,37 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "blake3" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", + "digest 0.10.3", +] + +[[package]] +name = "blobfs" +version = "0.1.0" +source = "git+https://github.com/dragonflyoss/image-service.git?rev=316380792092f73c99f832c4cb44ef4319d6f76b#316380792092f73c99f832c4cb44ef4319d6f76b" +dependencies = [ + "fuse-backend-rs", + "libc", + "log", + "nydus-error", + "rafs", + "serde", + "serde_json", + "serde_with", + "storage", + "vm-memory", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -113,6 +183,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + +[[package]] +name = "byte-unit" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8" + [[package]] name = "byteorder" version = "1.4.3" @@ -135,6 +226,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "caps" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61bf7211aad104ce2769ec05efcdfabf85ee84ac92461d142f22cf8badd0e54c" +dependencies = [ + "errno", + "libc", + "thiserror", +] + [[package]] name = "cc" version = "1.0.73" @@ -177,7 +279,7 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time 0.1.44", + "time 0.1.43", "winapi", ] @@ -210,6 +312,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "containerd-shim-protos" version = "0.2.0" @@ -221,12 +329,30 @@ dependencies = [ "ttrpc", ] +[[package]] +name = "core-foundation-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" +dependencies = [ + "libc", +] + [[package]] name = "cpuid-bool" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.4" @@ -247,6 +373,205 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "5.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.12.1", + "lock_api", + "parking_lot_core 0.9.3", +] + +[[package]] +name = "dbs-address-space" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9acd47f8b1ad8a6a62450d2d83ced5452dbf9549e2b98709d945554b22a45ed7" +dependencies = [ + "arc-swap 1.5.0", + "libc", + "nix 0.23.1", + "thiserror", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "dbs-allocator" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92a384ac9bd9c22c486c7a66e68cdc6cd504da7793b69bd891f3d85070c999b6" +dependencies = [ + "thiserror", +] + +[[package]] +name = "dbs-arch" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d235999408e59e60d18461debfb31d504813cfa5e497ff9d45c1c319980cf74a" +dependencies = [ + "kvm-bindings", + "kvm-ioctls", + "libc", + "memoffset", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "dbs-boot" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37036c65dc89724ff5628cee6c48ebe75027f989398317b2a5155924ba9c2bf7" +dependencies = [ + "dbs-arch", + "kvm-bindings", + "kvm-ioctls", + "lazy_static", + "libc", + "thiserror", + "vm-fdt", + "vm-memory", +] + +[[package]] +name = "dbs-device" +version = "0.1.0" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +dependencies = [ + "thiserror", +] + +[[package]] +name = "dbs-interrupt" +version = "0.1.0" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +dependencies = [ + "dbs-device", + "kvm-bindings", + "kvm-ioctls", + "libc", + "vmm-sys-util", +] + +[[package]] +name = "dbs-legacy-devices" +version = "0.1.0" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +dependencies = [ + "dbs-device", + "dbs-utils", + "log", + "serde", + "vm-superio", + "vmm-sys-util", +] + +[[package]] +name = "dbs-uhttp" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b773f7f1b9088438e9746890c7c0836b133b07935812867a33e06e81c92c0cdc" +dependencies = [ + "libc", + "mio", +] + +[[package]] +name = "dbs-utils" +version = "0.1.0" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +dependencies = [ + "anyhow", + "event-manager", + "libc", + "log", + "serde", + "thiserror", + "timerfd", + "vmm-sys-util", +] + +[[package]] +name = "dbs-virtio-devices" +version = "0.1.0" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +dependencies = [ + "blobfs", + "byteorder", + "caps", + "dbs-device", + "dbs-interrupt", + "dbs-utils", + "epoll", + "fuse-backend-rs", + "io-uring", + "kvm-bindings", + "kvm-ioctls", + "libc", + "log", + "nix 0.23.1", + "rafs", + "rlimit", + "serde", + "serde_json", + "thiserror", + "threadpool", + "virtio-bindings", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "derive-new" version = "0.5.9" @@ -267,6 +592,61 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", + "subtle", +] + +[[package]] +name = "diskarbitration-sys" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f82432ae94d42f160b6e17389d6e1c1eee29827b99ad32d35a0a96bb98bedb5" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "dragonball" +version = "0.1.0" +dependencies = [ + "arc-swap 1.5.0", + "bytes 1.1.0", + "dbs-address-space", + "dbs-allocator", + "dbs-arch", + "dbs-boot", + "dbs-device", + "dbs-interrupt", + "dbs-legacy-devices", + "dbs-utils", + "dbs-virtio-devices", + "kvm-bindings", + "kvm-ioctls", + "lazy_static", + "libc", + "linux-loader", + "log", + "nix 0.23.1", + "seccompiler", + "serde", + "serde_derive", + "serde_json", + "slog", + "slog-scope", + "thiserror", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "either" version = "1.6.1" @@ -275,24 +655,65 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "enum-iterator" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +checksum = "2953d1df47ac0eb70086ccabf0275aa8da8591a28bd358ee2b52bd9f9e3ff9e9" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "epoll" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "event-manager" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "377fa591135fbe23396a18e2655a6d5481bf7c5823cdfa3cc81b01a229cbe640" +dependencies = [ + "libc", + "vmm-sys-util", +] + [[package]] name = "fail" version = "0.5.0" @@ -319,6 +740,22 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -335,6 +772,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "fuse-backend-rs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a96ec48cd39ee2504eaa4a31b88262b7d13151a4da0b53af8fd212c7c9ffa5d" +dependencies = [ + "arc-swap 1.5.0", + "bitflags", + "caps", + "core-foundation-sys", + "diskarbitration-sys", + "lazy_static", + "libc", + "log", + "mio", + "nix 0.23.1", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + [[package]] name = "futures" version = "0.1.31" @@ -412,6 +870,12 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.21" @@ -459,7 +923,7 @@ checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -482,9 +946,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" [[package]] name = "git2" -version = "0.13.25" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" dependencies = [ "bitflags", "libc", @@ -508,12 +972,35 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "governor" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19775995ee20209163239355bc3ad2f33f83da35d9ef72dea26e5af753552c87" +dependencies = [ + "dashmap", + "futures 0.3.21", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot 0.12.1", + "quanta", + "rand 0.8.5", + "smallvec", +] + [[package]] name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +[[package]] +name = "hashbrown" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" + [[package]] name = "heck" version = "0.3.3" @@ -538,21 +1025,41 @@ dependencies = [ "libc", ] +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "hypervisor" version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "dbs-utils", + "dragonball", + "go-flag", + "kata-sys-util", "kata-types", "libc", "logging", + "nix 0.16.1", + "seccompiler", "serde_json", "slog", "slog-scope", "thiserror", + "tokio", + "vmm-sys-util", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -566,12 +1073,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -583,6 +1090,22 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "io-lifetimes" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504" + +[[package]] +name = "io-uring" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d75829ed9377bab6c90039fe47b9d84caceb4b5063266142e21bcce6550cda8" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "iovec" version = "0.1.4" @@ -603,9 +1126,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "jobserver" @@ -616,6 +1139,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kata-sys-util" version = "0.1.0" @@ -643,6 +1175,7 @@ dependencies = [ name = "kata-types" version = "0.1.0" dependencies = [ + "byte-unit", "glob", "lazy_static", "num_cpus", @@ -653,7 +1186,27 @@ dependencies = [ "slog", "slog-scope", "thiserror", - "toml 0.5.8", + "toml 0.5.9", +] + +[[package]] +name = "kvm-bindings" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78c049190826fff959994b7c1d8a2930d0a348f1b8f3aa4f9bb34cd5d7f2952" +dependencies = [ + "vmm-sys-util", +] + +[[package]] +name = "kvm-ioctls" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97422ba48d7ffb66fd4d18130f72ab66f9bbbf791fb7a87b9291cdcfec437593" +dependencies = [ + "kvm-bindings", + "libc", + "vmm-sys-util", ] [[package]] @@ -664,15 +1217,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.13.4+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" dependencies = [ "cc", "libc", @@ -682,9 +1235,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", "libc", @@ -692,6 +1245,21 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-loader" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5e77493808403a6bd56a301a64ea6b9342e36ea845044bf0dfdf56fe52fa08" +dependencies = [ + "vm-memory", +] + +[[package]] +name = "linux-raw-sys" +version = "0.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" + [[package]] name = "linux_container" version = "0.1.0" @@ -699,6 +1267,7 @@ dependencies = [ "anyhow", "async-trait", "common", + "kata-types", "tokio", ] @@ -714,9 +1283,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", ] @@ -732,6 +1301,25 @@ dependencies = [ "slog-scope", ] +[[package]] +name = "lz4-sys" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matches" version = "0.1.9" @@ -740,9 +1328,9 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -755,35 +1343,23 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", - "autocfg", ] [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -806,9 +1382,9 @@ dependencies = [ [[package]] name = "netlink-packet-route" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733ea73609acfd7fa7ddadfb7bf709b0471668c456ad9513685af543a06342b2" +checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab" dependencies = [ "anyhow", "bitflags", @@ -832,23 +1408,24 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.9.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8785b8141e8432aa45fceb922a7e876d7da3fad37fa7e7ec702ace3aa0826b" +checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" dependencies = [ "bytes 1.1.0", "futures 0.3.21", "log", "netlink-packet-core", "netlink-sys", + "thiserror", "tokio", ] [[package]] name = "netlink-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c9f9547a08241bee7b6558b9b98e1f290d187de8b7cfca2bbb4937bcaa8f8" +checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027" dependencies = [ "bytes 1.1.0", "futures 0.3.21", @@ -872,9 +1449,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.22.3" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" dependencies = [ "bitflags", "cc", @@ -885,31 +1462,32 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.1" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9" dependencies = [ "bitflags", - "cc", "cfg-if 1.0.0", "libc", - "memoffset", ] [[package]] -name = "ntapi" -version = "0.3.7" +name = "no-std-compat" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -917,9 +1495,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -936,18 +1514,50 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + +[[package]] +name = "nydus-error" +version = "0.2.0" +source = "git+https://github.com/dragonflyoss/image-service.git?rev=316380792092f73c99f832c4cb44ef4319d6f76b#316380792092f73c99f832c4cb44ef4319d6f76b" +dependencies = [ + "backtrace", + "httpdate", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "nydus-utils" +version = "0.1.0" +source = "git+https://github.com/dragonflyoss/image-service.git?rev=316380792092f73c99f832c4cb44ef4319d6f76b#316380792092f73c99f832c4cb44ef4319d6f76b" dependencies = [ + "blake3", + "flate2", + "fuse-backend-rs", + "lazy_static", "libc", + "log", + "lz4-sys", + "nydus-error", + "serde", + "serde_json", + "sha2", + "zstd", ] [[package]] name = "object" -version = "0.27.1" +version = "0.28.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" dependencies = [ "memchr", ] @@ -964,9 +1574,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "opaque-debug" @@ -976,13 +1586,23 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.3", ] [[package]] @@ -999,6 +1619,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "paste" version = "1.0.7" @@ -1023,9 +1656,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1071,11 +1704,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -1169,6 +1802,22 @@ dependencies = [ "ttrpc-codegen", ] +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.2+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.18" @@ -1178,6 +1827,34 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rafs" +version = "0.1.0" +source = "git+https://github.com/dragonflyoss/image-service.git?rev=316380792092f73c99f832c4cb44ef4319d6f76b#316380792092f73c99f832c4cb44ef4319d6f76b" +dependencies = [ + "anyhow", + "arc-swap 0.4.8", + "bitflags", + "blake3", + "flate2", + "fuse-backend-rs", + "futures 0.3.21", + "lazy_static", + "libc", + "log", + "lz4-sys", + "nix 0.23.1", + "nydus-error", + "nydus-utils", + "serde", + "serde_json", + "serde_with", + "sha2", + "spmc", + "storage", + "vm-memory", +] + [[package]] name = "rand" version = "0.3.23" @@ -1287,6 +1964,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "738bc47119e3eeccc7e94c4a506901aea5e7b4944ecd0829cbebf4af04ceda12" +dependencies = [ + "bitflags", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1307,9 +1993,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -1318,9 +2004,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "remove_dir_all" @@ -1335,6 +2021,7 @@ dependencies = [ name = "resource" version = "0.1.0" dependencies = [ + "actix-rt", "agent", "anyhow", "async-trait", @@ -1360,17 +2047,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "rlimit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "347703a5ae47adf1e693144157be231dde38c72bd485925cae7407ad3e52480b" +dependencies = [ + "libc", +] + [[package]] name = "rtnetlink" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f54290e54521dac3de4149d83ddf9f62a359b3cc93bcb494a794a41e6f4744b" +checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0" dependencies = [ "futures 0.3.21", "log", "netlink-packet-route", "netlink-proto", - "nix 0.22.3", + "nix 0.24.1", "thiserror", "tokio", ] @@ -1408,6 +2104,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.34.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c267b8394eb529872c3cf92e181c378b41fea36e68130357b52493701d2e" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "winapi", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -1416,9 +2126,9 @@ checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "scopeguard" @@ -1426,26 +2136,35 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "seccompiler" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d1292a1131b22ccea49f30bd106f1238b5ddeec1a98d39268dcc31d540e68" +dependencies = [ + "libc", +] + [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -1454,15 +2173,37 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serial_test" version = "0.5.1" @@ -1470,7 +2211,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d" dependencies = [ "lazy_static", - "parking_lot", + "parking_lot 0.11.2", "serial_test_derive", ] @@ -1507,10 +2248,10 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpuid-bool", - "digest", + "digest 0.9.0", "opaque-debug", ] @@ -1597,7 +2338,7 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" dependencies = [ - "arc-swap", + "arc-swap 1.5.0", "lazy_static", "slog", ] @@ -1629,6 +2370,46 @@ dependencies = [ "winapi", ] +[[package]] +name = "spmc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02a8428da277a8e3a15271d79943e80ccc2ef254e78813a166a08d65e4c3ece5" + +[[package]] +name = "storage" +version = "0.5.0" +source = "git+https://github.com/dragonflyoss/image-service.git?rev=316380792092f73c99f832c4cb44ef4319d6f76b#316380792092f73c99f832c4cb44ef4319d6f76b" +dependencies = [ + "anyhow", + "arc-swap 0.4.8", + "bitflags", + "dbs-uhttp", + "fuse-backend-rs", + "futures 0.3.21", + "governor", + "lazy_static", + "libc", + "log", + "nix 0.23.1", + "nydus-error", + "nydus-utils", + "serde", + "serde_json", + "serde_with", + "sha2", + "spmc", + "tokio", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strum" version = "0.24.0" @@ -1653,23 +2434,29 @@ dependencies = [ [[package]] name = "subprocess" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" dependencies = [ "libc", "winapi", ] +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" -version = "1.0.91" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -1701,18 +2488,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -1728,14 +2515,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" -version = "0.1.44" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] @@ -1750,11 +2545,20 @@ dependencies = [ "num_threads", ] +[[package]] +name = "timerfd" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f85a7c965b8e7136952f59f2a359694c78f105b2d2ff99cf6c2c404bf7e33f" +dependencies = [ + "rustix", +] + [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -1767,9 +2571,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "95eec79ea28c00a365f539f1961e9278fbcaf81c0ff6aaf0e93c181352446948" dependencies = [ "bytes 1.1.0", "libc", @@ -1777,6 +2581,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", @@ -1786,9 +2591,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ "proc-macro2", "quote", @@ -1819,18 +2624,18 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] [[package]] name = "ttrpc" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7d6c992964a013c17814c08d31708d577b0aae44ebadb58755659dd824c2d1" +checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8" dependencies = [ "async-trait", "byteorder", @@ -1880,9 +2685,15 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" [[package]] name = "unicode-normalization" @@ -1899,12 +2710,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" - [[package]] name = "unix_socket2" version = "0.5.4" @@ -1995,6 +2800,56 @@ dependencies = [ "url", ] +[[package]] +name = "virtio-bindings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff512178285488516ed85f15b5d0113a7cdb89e9e8a760b269ae4f02b84bd6b" + +[[package]] +name = "virtio-queue" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90da9e627f6aaf667cc7b6548a28be332d3e1f058f4ceeb46ab6bcee5c4b74d" +dependencies = [ + "log", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "vm-fdt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43fb5a6bd1a7d423ad72802801036719b7546cf847a103f8fe4575f5b0d45a6" + +[[package]] +name = "vm-memory" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339d4349c126fdcd87e034631d7274370cf19eb0e87b33166bcd956589fc72c5" +dependencies = [ + "arc-swap 1.5.0", + "libc", + "winapi", +] + +[[package]] +name = "vm-superio" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4b5231d334edbc03b22704caa1a022e4c07491d6df736593f26094df8b04a51" + +[[package]] +name = "vmm-sys-util" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733537bded03aaa93543f785ae997727b30d1d9f4a03b7861d23290474242e11" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "void" version = "1.0.2" @@ -2019,9 +2874,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasi" @@ -2029,6 +2884,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + [[package]] name = "wasm_container" version = "0.1.0" @@ -2036,9 +2945,20 @@ dependencies = [ "anyhow", "async-trait", "common", + "kata-types", "tokio", ] +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.2.5" @@ -2071,3 +2991,80 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] + +[[patch.unused]] +name = "dbs-upcall" +version = "0.1.0" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index dcd34e64bd7b..70b384e43185 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -1,8 +1,12 @@ [workspace] members = [ "crates/shim", - # TODO: current only for check, delete after use the agent crate - "crates/agent", - # TODO: current only for check, delete after use the resource crate - "crates/resource", ] + +[patch.'crates-io'] +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index e923ad8c71a2..462f1f188ff6 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -122,6 +122,7 @@ MONITOR_OUTPUT = $(CURDIR)/$(MONITOR) MONITOR_DIR = $(CLI_DIR)/kata-monitor SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$') VERSION := ${shell cat ./VERSION} + # List of configuration files to build and install CONFIGS = CONFIG_PATHS = @@ -279,7 +280,7 @@ TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) ##VAR DESTDIR= is a directory prepended to each installed target file DESTDIR := ##VAR BINDIR= is a directory for installing executable programs -BINDIR := /usr/bin +BINDIR := /usr/local/bin GENERATED_CODE = crates/shim/src/config.rs @@ -330,7 +331,9 @@ endef .DEFAULT_GOAL := default ##TARGET default: build code -default: $(TARGET) show-header +default: runtime show-header + +runtime: $(TARGET) $(TARGET): $(GENERATED_CODE) $(TARGET_PATH) @@ -418,6 +421,11 @@ codecov: check_tarpaulin codecov-html: check_tarpaulin cargo tarpaulin $(TARPAULIN_ARGS) -o Html +install: install-runtime install-configs + +install-runtime: runtime + install -D $(TARGET_PATH) $(BINDIR) + install-configs: $(CONFIGS) $(foreach f,$(CONFIGS),$(call INSTALL_CONFIG,$f,$(dir $(CONFIG_PATH)))) \ sudo ln -sf $(DEFAULT_HYPERVISOR_CONFIG) $(DESTDIR)/$(CONFIG_PATH) diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml index f9350bd0659e..c5febe43d782 100644 --- a/src/runtime-rs/crates/agent/Cargo.toml +++ b/src/runtime-rs/crates/agent/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "^1.0", features = ["derive"] } serde_json = ">=1.0.9" slog = "2.5.2" slog-scope = "4.4.0" -ttrpc = { version = "0.6.0" } +ttrpc = { version = "0.6.1" } tokio = { version = "1.8.0", features = ["fs", "rt"] } url = "2.2.2" diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index 4520409d2a2e..3201b4ee26d7 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -9,13 +9,22 @@ edition = "2018" [dependencies] anyhow = "^1.0" async-trait = "0.1.48" +dbs-utils = "0.1.0" +go-flag = "0.1.0" libc = ">=0.2.39" +nix = "0.16.1" +seccompiler = "0.2.0" serde_json = ">=1.0.9" slog = "2.5.2" slog-scope = "4.4.0" thiserror = "1.0" +tokio = { version = "1.8.0", features = ["sync"] } +vmm-sys-util = "0.9.0" +kata-sys-util = { path = "../../../libs/kata-sys-util" } kata-types = { path = "../../../libs/kata-types" } logging = { path = "../../../libs/logging" } +dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs"] } + [features] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs new file mode 100644 index 000000000000..aef8d3352d06 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -0,0 +1,309 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{collections::HashSet, fs::create_dir_all, path::PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use dragonball::{ + api::v1::{BlockDeviceConfigInfo, BootSourceConfig}, + vm::VmConfigInfo, +}; +use kata_sys_util::mount; +use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; + +use super::{vmm_instance::VmmInstance, RUN_PATH_PREFIX}; +use crate::{device::Device, kernel_param::KernelParams, VmmState, VM_ROOTFS_DRIVER_BLK}; + +const DRAGONBALL_KERNEL: &str = "vmlinux"; +const DRAGONBALL_ROOT_FS: &str = "rootfs"; + +unsafe impl Send for DragonballInner {} +unsafe impl Sync for DragonballInner {} +pub struct DragonballInner { + /// sandbox id + pub(crate) id: String, + + /// vm path + pub(crate) vm_path: String, + + /// jailed flag + pub(crate) jailed: bool, + + /// chroot base for the jailer + pub(crate) jailer_root: String, + + /// netns + pub(crate) netns: Option, + + /// hypervisor config + pub(crate) config: HypervisorConfig, + + /// vmm state + pub(crate) state: VmmState, + + /// vmm instance + pub(crate) vmm_instance: VmmInstance, + + /// hypervisor run dir + pub(crate) run_dir: String, + + /// pending device + pub(crate) pending_devices: Vec, + + /// cached block device + pub(crate) cached_block_devices: HashSet, +} + +impl DragonballInner { + pub fn new() -> DragonballInner { + DragonballInner { + id: "".to_string(), + vm_path: "".to_string(), + jailer_root: "".to_string(), + netns: None, + config: Default::default(), + pending_devices: vec![], + state: VmmState::NotReady, + jailed: false, + vmm_instance: VmmInstance::new(""), + run_dir: "".to_string(), + cached_block_devices: Default::default(), + } + } + + pub(crate) async fn cold_start_vm(&mut self, timeout: i32) -> Result<()> { + info!(sl!(), "start sandbox cold"); + + self.set_vm_base_config().context("set vm base config")?; + + // get rootfs driver + let rootfs_driver = self.config.blockdev_info.block_device_driver.clone(); + + // get kernel params + let mut kernel_params = KernelParams::new(self.config.debug_info.enable_debug); + kernel_params.append(&mut KernelParams::new_rootfs_kernel_params(&rootfs_driver)); + kernel_params.append(&mut KernelParams::from_string( + &self.config.boot_info.kernel_params, + )); + + // set boot source + let kernel_path = self.config.boot_info.kernel.clone(); + self.set_boot_source( + &kernel_path, + &kernel_params + .to_string() + .context("kernel params to string")?, + ) + .context("set_boot_source")?; + + // get vm rootfs + let image = { + let initrd_path = self.config.boot_info.initrd.clone(); + let image_path = self.config.boot_info.image.clone(); + if !initrd_path.is_empty() { + Ok(initrd_path) + } else if !image_path.is_empty() { + Ok(image_path) + } else { + Err(anyhow!("failed to get image")) + } + } + .context("get image")?; + self.set_vm_rootfs(&image, &rootfs_driver) + .context("set vm rootfs")?; + + // add pending devices + while let Some(dev) = self.pending_devices.pop() { + self.add_device(dev).await.context("add_device")?; + } + + // start vmm and wait ready + self.start_vmm_instance().context("start vmm instance")?; + self.wait_vmm_ready(timeout).context("wait vmm")?; + + Ok(()) + } + + pub(crate) fn run_vmm_server(&mut self) -> Result<()> { + if !self.config.jailer_path.is_empty() { + self.jailed = true; + } + + // create jailer root + create_dir_all(self.jailer_root.as_str()) + .map_err(|e| anyhow!("Failed to create dir {} err : {:?}", self.jailer_root, e))?; + + // create run dir + self.run_dir = [RUN_PATH_PREFIX, self.id.as_str()].join("/"); + create_dir_all(self.run_dir.as_str()) + .with_context(|| format!("failed to create dir {}", self.run_dir.as_str()))?; + + // run vmm server + self.vmm_instance + .run_vmm_server(&self.id, self.netns.clone()) + .context("run vmm server")?; + self.state = VmmState::VmmServerReady; + + Ok(()) + } + + pub(crate) fn cleanup_resource(&self) { + if self.jailed { + self.umount_jail_resource(DRAGONBALL_KERNEL).ok(); + self.umount_jail_resource(DRAGONBALL_ROOT_FS).ok(); + for id in &self.cached_block_devices { + self.umount_jail_resource(id.as_str()).ok(); + } + } + + std::fs::remove_dir_all(&self.vm_path) + .map_err(|err| { + error!(sl!(), "failed to remove dir all for {}", &self.vm_path); + err + }) + .ok(); + } + + fn set_vm_base_config(&mut self) -> Result<()> { + let serial_path = [&self.run_dir, "console.sock"].join("/"); + let vm_config = VmConfigInfo { + serial_path: Some(serial_path), + mem_size_mib: self.config.memory_info.default_memory as usize, + vcpu_count: self.config.cpu_info.default_vcpus as u8, + ..Default::default() + }; + info!(sl!(), "vm config: {:?}", vm_config); + + self.vmm_instance + .set_vm_configuration(vm_config) + .context("set vm configuration") + } + + pub(crate) fn umount_jail_resource(&self, jailed_path: &str) -> Result<()> { + let path = [self.jailer_root.as_str(), jailed_path].join("/"); + nix::mount::umount2(path.as_str(), nix::mount::MntFlags::MNT_DETACH) + .with_context(|| format!("umount path {}", &path)) + } + + pub(crate) fn get_resource(&self, src: &str, dst: &str) -> Result { + if self.jailed { + self.jail_resource(src, dst) + } else { + Ok(src.to_string()) + } + } + + fn jail_resource(&self, src: &str, dst: &str) -> Result { + info!(sl!(), "jail resource: src {} dst {}", src, dst); + if src.is_empty() || dst.is_empty() { + return Err(anyhow!("invalid param src {} dst {}", src, dst)); + } + + let jailed_location = [self.jailer_root.as_str(), dst].join("/"); + mount::bind_mount_unchecked(src, jailed_location.as_str(), false).context("bind_mount")?; + + let mut abs_path = String::from("/"); + abs_path.push_str(dst); + Ok(abs_path) + } + + fn set_boot_source(&mut self, kernel_path: &str, kernel_params: &str) -> Result<()> { + info!( + sl!(), + "kernel path {} kernel params {}", kernel_path, kernel_params + ); + + let mut boot_cfg = BootSourceConfig { + kernel_path: self + .get_resource(kernel_path, DRAGONBALL_KERNEL) + .context("get resource")?, + ..Default::default() + }; + + if !kernel_params.is_empty() { + boot_cfg.boot_args = Some(kernel_params.to_string()); + } + + self.vmm_instance + .put_boot_source(boot_cfg) + .context("put boot source") + } + + fn set_vm_rootfs(&mut self, path: &str, driver: &str) -> Result<()> { + info!(sl!(), "set vm rootfs {} {}", path, driver); + let jail_drive = self + .get_resource(path, DRAGONBALL_ROOT_FS) + .context("get resource")?; + + if driver == VM_ROOTFS_DRIVER_BLK { + let blk_cfg = BlockDeviceConfigInfo { + path_on_host: PathBuf::from(jail_drive), + drive_id: DRAGONBALL_ROOT_FS.to_string(), + is_root_device: false, + // Add it as a regular block device + // This allows us to use a partitioned root block device + // is_read_only + is_read_only: true, + is_direct: false, + ..Default::default() + }; + + self.vmm_instance + .insert_block_device(blk_cfg) + .context("inert block device") + } else { + Err(anyhow!( + "Unknown vm_rootfs driver {} path {:?}", + driver, + path + )) + } + } + + fn start_vmm_instance(&mut self) -> Result<()> { + info!(sl!(), "Starting VM"); + self.vmm_instance + .instance_start() + .context("Failed to start vmm")?; + self.state = VmmState::VmRunning; + Ok(()) + } + + // wait_vmm_ready will wait for timeout seconds for the VMM to be up and running. + // This does not mean that the VM is up and running. It only indicates that the VMM is up and + // running and able to handle commands to setup and launch a VM + fn wait_vmm_ready(&mut self, timeout: i32) -> Result<()> { + if timeout < 0 { + return Err(anyhow!("Invalid param timeout {}", timeout)); + } + + let time_start = std::time::Instant::now(); + loop { + match self.vmm_instance.is_running() { + Ok(_) => return Ok(()), + Err(err) => { + let time_now = std::time::Instant::now(); + if time_now.duration_since(time_start).as_millis() > timeout as u128 { + return Err(anyhow!( + "waiting vmm ready timeout {} err: {:?}", + timeout, + err + )); + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + } + } + + pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) { + self.config = config; + } + + pub fn hypervisor_config(&self) -> HypervisorConfig { + self.config.clone() + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs new file mode 100644 index 000000000000..d47cac569857 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -0,0 +1,316 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::path::PathBuf; + +use anyhow::{anyhow, Context, Result}; +use dbs_utils::net::MacAddr; +use dragonball::api::v1::{ + BlockDeviceConfigInfo, FsDeviceConfigInfo, FsMountConfigInfo, VirtioNetDeviceConfigInfo, + VsockDeviceConfigInfo, +}; + +use super::DragonballInner; +use crate::{ + device::Device, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig, ShareFsMountType, + ShareFsOperation, VmmState, VsockConfig, +}; + +const MB_TO_B: u32 = 1024 * 1024; +const DEFAULT_VIRTIO_FS_NUM_QUEUES: i32 = 1; +const DEFAULT_VIRTIO_FS_QUEUE_SIZE: i32 = 1024; + +const VIRTIO_FS: &str = "virtio-fs"; +const INLINE_VIRTIO_FS: &str = "inline-virtio-fs"; + +pub(crate) fn drive_index_to_id(index: u64) -> String { + format!("drive_{}", index) +} + +impl DragonballInner { + pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> { + if self.state == VmmState::NotReady { + info!(sl!(), "VMM not ready, queueing device {}", device); + + // add the pending device by reverse order, thus the + // start_vm would pop the devices in an right order + // to add the devices. + self.pending_devices.insert(0, device); + return Ok(()); + } + + info!(sl!(), "dragonball add device {:?}", &device); + match device { + Device::Network(config) => self.add_net_device(&config).context("add net device"), + Device::Vfio(_config) => { + todo!() + } + Device::Block(config) => self + .add_block_device( + config.path_on_host.as_str(), + config.id.as_str(), + config.is_readonly, + config.no_drop, + ) + .context("add block device"), + Device::Vsock(config) => self.add_vsock(&config).context("add vsock"), + Device::ShareFsDevice(config) => self + .add_share_fs_device(&config) + .context("add share fs device"), + Device::ShareFsMount(config) => self + .add_share_fs_mount(&config) + .context("add share fs mount"), + } + } + + pub(crate) async fn remove_device(&mut self, device: Device) -> Result<()> { + info!(sl!(), "remove device {} ", device); + + match device { + Device::Block(config) => { + let drive_id = drive_index_to_id(config.index); + self.remove_block_drive(drive_id.as_str()) + .context("remove block drive") + } + Device::Vfio(_config) => { + todo!() + } + _ => Err(anyhow!("unsupported device {:?}", device)), + } + } + + fn add_block_device( + &mut self, + path: &str, + id: &str, + read_only: bool, + no_drop: bool, + ) -> Result<()> { + let jailed_drive = self.get_resource(path, id).context("get resource")?; + self.cached_block_devices.insert(id.to_string()); + + let blk_cfg = BlockDeviceConfigInfo { + drive_id: id.to_string(), + path_on_host: PathBuf::from(jailed_drive), + is_direct: self.config.blockdev_info.block_device_cache_direct, + no_drop, + is_read_only: read_only, + ..Default::default() + }; + self.vmm_instance + .insert_block_device(blk_cfg) + .context("insert block device") + } + + fn remove_block_drive(&mut self, id: &str) -> Result<()> { + self.vmm_instance + .remove_block_device(id) + .context("remove block device")?; + + if self.cached_block_devices.contains(id) && self.jailed { + self.umount_jail_resource(id) + .context("umount jail resource")?; + self.cached_block_devices.remove(id); + } + Ok(()) + } + + fn add_net_device(&mut self, config: &NetworkConfig) -> Result<()> { + let iface_cfg = VirtioNetDeviceConfigInfo { + iface_id: config.id.clone(), + host_dev_name: config.host_dev_name.clone(), + guest_mac: match &config.guest_mac { + Some(mac) => MacAddr::from_bytes(&mac.0).ok(), + None => None, + }, + ..Default::default() + }; + + info!( + sl!(), + "add {} endpoint to {}", iface_cfg.host_dev_name, iface_cfg.iface_id + ); + + self.vmm_instance + .insert_network_device(iface_cfg) + .context("insert network device") + } + + fn add_vsock(&mut self, config: &VsockConfig) -> Result<()> { + let vsock_cfg = VsockDeviceConfigInfo { + id: String::from("root"), + guest_cid: config.guest_cid, + uds_path: Some(config.uds_path.clone()), + ..Default::default() + }; + + self.vmm_instance + .insert_vsock(vsock_cfg) + .context("insert vsock") + } + + fn parse_inline_virtiofs_args(&self, fs_cfg: &mut FsDeviceConfigInfo) -> Result<()> { + let mut debug = false; + let mut opt_list = String::new(); + + fs_cfg.mode = String::from("virtio"); + fs_cfg.cache_policy = self.config.shared_fs.virtio_fs_cache.clone(); + fs_cfg.fuse_killpriv_v2 = true; + + info!( + sl!(), + "args: {:?}", &self.config.shared_fs.virtio_fs_extra_args + ); + let args = &self.config.shared_fs.virtio_fs_extra_args; + let _ = go_flag::parse_args_with_warnings::(args, None, |flags| { + flags.add_flag("d", &mut debug); + flags.add_flag("thread-pool-size", &mut fs_cfg.thread_pool_size); + flags.add_flag("drop-sys-resource", &mut fs_cfg.drop_sys_resource); + flags.add_flag("o", &mut opt_list); + }) + .with_context(|| format!("parse args: {:?}", args))?; + + if debug { + warn!( + sl!(), + "Inline virtiofs \"-d\" option not implemented, ignore" + ); + } + + // Parse comma separated option list + if !opt_list.is_empty() { + let args: Vec<&str> = opt_list.split(',').collect(); + for arg in args { + match arg { + "no_open" => fs_cfg.no_open = true, + "open" => fs_cfg.no_open = false, + "writeback_cache" => fs_cfg.writeback_cache = true, + "no_writeback_cache" => fs_cfg.writeback_cache = false, + "writeback" => fs_cfg.writeback_cache = true, + "no_writeback" => fs_cfg.writeback_cache = false, + "xattr" => fs_cfg.xattr = true, + "no_xattr" => fs_cfg.xattr = false, + "cache_symlinks" => {} // inline virtiofs always cache symlinks + "trace" => warn!( + sl!(), + "Inline virtiofs \"-o trace\" option not supported yet, ignored." + ), + _ => warn!(sl!(), "Inline virtiofs unsupported option: {}", arg), + } + } + } + + debug!(sl!(), "Inline virtiofs config {:?}", fs_cfg); + Ok(()) + } + + fn add_share_fs_device(&self, config: &ShareFsDeviceConfig) -> Result<()> { + let mut fs_cfg = FsDeviceConfigInfo { + sock_path: config.sock_path.clone(), + tag: config.mount_tag.clone(), + num_queues: if config.queue_num > 0 { + config.queue_size as usize + } else { + DEFAULT_VIRTIO_FS_NUM_QUEUES as usize + }, + queue_size: if config.queue_size > 0 { + config.queue_size as u16 + } else { + DEFAULT_VIRTIO_FS_QUEUE_SIZE as u16 + }, + cache_size: (self.config.shared_fs.virtio_fs_cache_size as u64) + .saturating_mul(MB_TO_B as u64), + ..Default::default() + }; + self.do_add_fs_device(&config.fs_type, &mut fs_cfg) + } + + fn do_add_fs_device(&self, fs_type: &str, fs_cfg: &mut FsDeviceConfigInfo) -> Result<()> { + match fs_type { + VIRTIO_FS => { + fs_cfg.mode = String::from("vhostuser"); + } + INLINE_VIRTIO_FS => { + self.parse_inline_virtiofs_args(fs_cfg)?; + } + _ => { + return Err(anyhow!( + "hypervisor isn't configured with shared_fs supported" + )); + } + } + self.vmm_instance + .insert_fs(fs_cfg) + .map_err(|e| anyhow!("insert {} fs error. {:?}", fs_cfg.mode, e)) + } + + fn add_share_fs_mount(&mut self, config: &ShareFsMountConfig) -> Result<()> { + let ops = match config.op { + ShareFsOperation::Mount => "mount", + ShareFsOperation::Umount => "umount", + ShareFsOperation::Update => "update", + }; + + let fstype = match config.fstype { + ShareFsMountType::PASSTHROUGH => "passthroughfs", + ShareFsMountType::RAFS => "rafs", + }; + + let cfg = FsMountConfigInfo { + ops: ops.to_string(), + fstype: Some(fstype.to_string()), + source: Some(config.source.clone()), + mountpoint: config.mount_point.clone(), + config: None, + tag: config.tag.clone(), + prefetch_list_path: config.prefetch_list_path.clone(), + dax_threshold_size_kb: None, + }; + + self.vmm_instance.patch_fs(&cfg, config.op).map_err(|e| { + anyhow!( + "{:?} {} at {} error: {:?}", + config.op, + fstype, + config.mount_point.clone(), + e + ) + }) + } +} + +#[cfg(test)] +mod tests { + use dragonball::api::v1::FsDeviceConfigInfo; + + use crate::dragonball::DragonballInner; + + #[test] + fn test_parse_inline_virtiofs_args() { + let mut dragonball = DragonballInner::new(); + let mut fs_cfg = FsDeviceConfigInfo::default(); + + // no_open and writeback_cache is the default, so test open and no_writeback_cache. "-d" + // and "trace" are ignored for now, but should not return error. + dragonball.config.shared_fs.virtio_fs_extra_args = vec![ + "-o".to_string(), + "open,no_writeback_cache,xattr,trace".to_string(), + "--thread-pool-size=128".to_string(), + "--drop-sys-resource".to_string(), + "-d".to_string(), + ]; + dragonball.config.shared_fs.virtio_fs_cache = "auto".to_string(); + dragonball.parse_inline_virtiofs_args(&mut fs_cfg).unwrap(); + + assert!(!fs_cfg.no_open); + assert!(fs_cfg.xattr); + assert!(fs_cfg.fuse_killpriv_v2); + assert!(!fs_cfg.writeback_cache); + assert_eq!(fs_cfg.cache_policy, "auto".to_string()); + assert!(fs_cfg.drop_sys_resource); + assert!(fs_cfg.thread_pool_size == 128); + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs new file mode 100644 index 000000000000..2b9c3c77cec9 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs @@ -0,0 +1,137 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + collections::{HashMap, HashSet}, + iter::FromIterator, +}; + +use anyhow::{Context, Result}; + +use super::inner::DragonballInner; +use crate::{utils, VcpuThreadIds, VmmState}; + +const KATA_PATH: &str = "/run/kata"; +const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock"; + +fn get_vsock_path(root: &str) -> String { + [root, DEFAULT_HYBRID_VSOCK_NAME].join("/") +} + +impl DragonballInner { + pub(crate) async fn prepare_vm(&mut self, id: &str, netns: Option) -> Result<()> { + self.id = id.to_string(); + self.state = VmmState::NotReady; + + self.vm_path = [KATA_PATH, id].join("/"); + self.jailer_root = [self.vm_path.as_str(), "root"].join("/"); + self.netns = netns; + + // prepare vsock + let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/"); + let d = crate::device::Device::Vsock(crate::device::VsockConfig { + id: format!("vsock-{}", &self.id), + guest_cid: 3, + uds_path, + }); + + self.add_device(d).await.context("add device")?; + Ok(()) + } + + // start_vm will start the hypervisor for the given sandbox. + // In the context of dragonball, this will start the hypervisor + pub(crate) async fn start_vm(&mut self, timeout: i32) -> Result<()> { + self.run_vmm_server().context("start vmm server")?; + self.cold_start_vm(timeout).await.map_err(|error| { + error!(sl!(), "start micro vm error {:?}", error); + if let Err(err) = self.stop_vm() { + error!(sl!(), "failed to call end err : {:?}", err); + } + error + })?; + + Ok(()) + } + + pub(crate) fn stop_vm(&mut self) -> Result<()> { + info!(sl!(), "Stopping dragonball VM"); + self.vmm_instance.stop().context("stop")?; + Ok(()) + } + + pub(crate) fn pause_vm(&self) -> Result<()> { + info!(sl!(), "do pause vm"); + self.vmm_instance.pause().context("pause vm")?; + Ok(()) + } + + pub(crate) fn resume_vm(&self) -> Result<()> { + info!(sl!(), "do resume vm"); + self.vmm_instance.resume().context("resume vm")?; + Ok(()) + } + + pub(crate) async fn save_vm(&self) -> Result<()> { + todo!() + } + + pub(crate) async fn get_agent_socket(&self) -> Result { + const HYBRID_VSOCK_SCHEME: &str = "hvsock"; + Ok(format!( + "{}://{}", + HYBRID_VSOCK_SCHEME, + get_vsock_path(&self.jailer_root), + )) + } + + pub(crate) async fn disconnect(&mut self) { + self.state = VmmState::NotReady; + } + + pub(crate) async fn get_thread_ids(&self) -> Result { + let mut vcpu_thread_ids: VcpuThreadIds = VcpuThreadIds { + vcpus: HashMap::new(), + }; + + for tid in self.vmm_instance.get_vcpu_tids() { + vcpu_thread_ids.vcpus.insert(tid.0 as u32, tid.1 as u32); + } + info!(sl!(), "get thread ids {:?}", vcpu_thread_ids); + Ok(vcpu_thread_ids) + } + + pub(crate) async fn cleanup(&self) -> Result<()> { + self.cleanup_resource(); + Ok(()) + } + + pub(crate) async fn get_pids(&self) -> Result> { + let mut pids = HashSet::new(); + // get shim thread ids + pids.insert(self.vmm_instance.pid()); + + for tid in utils::get_child_threads(self.vmm_instance.pid()) { + pids.insert(tid); + } + + // remove vcpus + for tid in self.vmm_instance.get_vcpu_tids() { + pids.remove(&tid.1); + } + + info!(sl!(), "get pids {:?}", pids); + Ok(Vec::from_iter(pids.into_iter())) + } + + pub(crate) async fn check(&self) -> Result<()> { + Ok(()) + } + + pub(crate) async fn get_jailer_root(&self) -> Result { + Ok(self.jailer_root.clone()) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs new file mode 100644 index 000000000000..27adfd73b08b --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -0,0 +1,130 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +mod inner; +mod inner_device; +mod inner_hypervisor; +use inner::DragonballInner; +pub mod vmm_instance; + +pub const RUN_PATH_PREFIX: &str = "/run/kata"; + +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; +use tokio::sync::RwLock; + +use crate::{device::Device, Hypervisor, VcpuThreadIds}; + +unsafe impl Send for Dragonball {} +unsafe impl Sync for Dragonball {} +pub struct Dragonball { + inner: Arc>, +} + +impl Default for Dragonball { + fn default() -> Self { + Self::new() + } +} + +impl Dragonball { + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(DragonballInner::new())), + } + } + + pub async fn set_hypervisor_config(&mut self, config: HypervisorConfig) { + let mut inner = self.inner.write().await; + inner.set_hypervisor_config(config) + } +} + +#[async_trait] +impl Hypervisor for Dragonball { + async fn prepare_vm(&self, id: &str, netns: Option) -> Result<()> { + let mut inner = self.inner.write().await; + inner.prepare_vm(id, netns).await + } + + async fn start_vm(&self, timeout: i32) -> Result<()> { + let mut inner = self.inner.write().await; + inner.start_vm(timeout).await + } + + async fn stop_vm(&self) -> Result<()> { + let mut inner = self.inner.write().await; + inner.stop_vm() + } + + async fn pause_vm(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.pause_vm() + } + + async fn resume_vm(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.resume_vm() + } + + async fn save_vm(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.save_vm().await + } + + async fn add_device(&self, device: Device) -> Result<()> { + let mut inner = self.inner.write().await; + inner.add_device(device).await + } + + async fn remove_device(&self, device: Device) -> Result<()> { + let mut inner = self.inner.write().await; + inner.remove_device(device).await + } + + async fn get_agent_socket(&self) -> Result { + let inner = self.inner.read().await; + inner.get_agent_socket().await + } + + async fn disconnect(&self) { + let mut inner = self.inner.write().await; + inner.disconnect().await + } + + async fn hypervisor_config(&self) -> HypervisorConfig { + let inner = self.inner.read().await; + inner.hypervisor_config() + } + + async fn get_thread_ids(&self) -> Result { + let inner = self.inner.read().await; + inner.get_thread_ids().await + } + + async fn cleanup(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.cleanup().await + } + + async fn get_pids(&self) -> Result> { + let inner = self.inner.read().await; + inner.get_pids().await + } + + async fn check(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.check().await + } + + async fn get_jailer_root(&self) -> Result { + let inner = self.inner.read().await; + inner.get_jailer_root().await + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs new file mode 100644 index 000000000000..70172c73a91d --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -0,0 +1,335 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::{ + fs::{File, OpenOptions}, + os::unix::{io::IntoRawFd, prelude::AsRawFd}, + sync::{ + mpsc::{channel, Receiver, Sender}, + Arc, Mutex, RwLock, + }, + thread, +}; + +use anyhow::{anyhow, Context, Result}; +use dragonball::{ + api::v1::{ + BlockDeviceConfigInfo, BootSourceConfig, FsDeviceConfigInfo, FsMountConfigInfo, + InstanceInfo, InstanceState, VirtioNetDeviceConfigInfo, VmmAction, VmmActionError, VmmData, + VmmRequest, VmmResponse, VmmService, VsockDeviceConfigInfo, + }, + vm::VmConfigInfo, + Vmm, +}; +use nix::sched::{setns, CloneFlags}; +use seccompiler::BpfProgram; +use vmm_sys_util::eventfd::EventFd; + +use crate::ShareFsOperation; + +pub enum Request { + Sync(VmmAction), +} + +const DRAGONBALL_VERSION: &str = env!("CARGO_PKG_VERSION"); +const REQUEST_RETRY: u32 = 500; +const KVM_DEVICE: &str = "/dev/kvm"; + +pub struct VmmInstance { + /// VMM instance info directly accessible from runtime + vmm_shared_info: Arc>, + to_vmm: Option>, + from_vmm: Option>, + to_vmm_fd: EventFd, + seccomp: BpfProgram, + vmm_thread: Option>>, +} + +impl VmmInstance { + pub fn new(id: &str) -> Self { + let vmm_shared_info = Arc::new(RwLock::new(InstanceInfo::new( + String::from(id), + DRAGONBALL_VERSION.to_string(), + ))); + + let to_vmm_fd = EventFd::new(libc::EFD_NONBLOCK) + .unwrap_or_else(|_| panic!("Failed to create eventfd for vmm {}", id)); + + VmmInstance { + vmm_shared_info, + to_vmm: None, + from_vmm: None, + to_vmm_fd, + seccomp: vec![], + vmm_thread: None, + } + } + + pub fn get_shared_info(&self) -> Arc> { + self.vmm_shared_info.clone() + } + + fn set_instance_id(&mut self, id: &str) { + let share_info_lock = self.vmm_shared_info.clone(); + share_info_lock.write().unwrap().id = String::from(id); + } + + pub fn get_vcpu_tids(&self) -> Vec<(u8, u32)> { + let info = self.vmm_shared_info.clone(); + let result = info.read().unwrap().tids.clone(); + result + } + + pub fn run_vmm_server(&mut self, id: &str, netns: Option) -> Result<()> { + let kvm = OpenOptions::new().read(true).write(true).open(KVM_DEVICE)?; + + let (to_vmm, from_runtime) = channel(); + let (to_runtime, from_vmm) = channel(); + + self.set_instance_id(id); + + let vmm_service = VmmService::new(from_runtime, to_runtime); + + self.to_vmm = Some(to_vmm); + self.from_vmm = Some(from_vmm); + + let api_event_fd2 = self.to_vmm_fd.try_clone().expect("Failed to dup eventfd"); + let vmm = Vmm::new( + self.vmm_shared_info.clone(), + api_event_fd2, + self.seccomp.clone(), + self.seccomp.clone(), + Some(kvm.into_raw_fd()), + ) + .expect("Failed to start vmm"); + + self.vmm_thread = Some( + thread::Builder::new() + .name("vmm_master".to_owned()) + .spawn(move || { + || -> Result { + debug!(sl!(), "run vmm thread start"); + if let Some(netns_path) = netns { + info!(sl!(), "set netns for vmm master {}", &netns_path); + let netns_fd = File::open(&netns_path) + .with_context(|| format!("open netns path {}", &netns_path))?; + setns(netns_fd.as_raw_fd(), CloneFlags::CLONE_NEWNET) + .context("set netns ")?; + } + let exit_code = + Vmm::run_vmm_event_loop(Arc::new(Mutex::new(vmm)), vmm_service); + debug!(sl!(), "run vmm thread exited: {}", exit_code); + Ok(exit_code) + }() + .map_err(|e| { + error!(sl!(), "run vmm thread err. {:?}", e); + e + }) + }) + .expect("Failed to start vmm event loop"), + ); + + Ok(()) + } + + pub fn put_boot_source(&self, boot_source_cfg: BootSourceConfig) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::ConfigureBootSource( + boot_source_cfg, + ))) + .context("Failed to configure boot source")?; + Ok(()) + } + + pub fn instance_start(&self) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::StartMicroVm)) + .context("Failed to start MicroVm")?; + Ok(()) + } + + pub fn is_uninitialized(&self) -> bool { + let share_info = self + .vmm_shared_info + .read() + .expect("Failed to read share_info due to poisoned lock"); + matches!(share_info.state, InstanceState::Uninitialized) + } + + pub fn is_running(&self) -> Result<()> { + let share_info_lock = self.vmm_shared_info.clone(); + let share_info = share_info_lock + .read() + .expect("Failed to read share_info due to poisoned lock"); + if let InstanceState::Running = share_info.state { + return Ok(()); + } + Err(anyhow!("vmm is not running")) + } + + pub fn get_machine_info(&self) -> Result> { + if let Ok(VmmData::MachineConfiguration(vm_config)) = + self.handle_request(Request::Sync(VmmAction::GetVmConfiguration)) + { + return Ok(vm_config); + } + Err(anyhow!("Failed to get machine info")) + } + + pub fn insert_block_device(&self, device_cfg: BlockDeviceConfigInfo) -> Result<()> { + self.handle_request_with_retry(Request::Sync(VmmAction::InsertBlockDevice( + device_cfg.clone(), + ))) + .with_context(|| format!("Failed to insert block device {:?}", device_cfg))?; + Ok(()) + } + + pub fn remove_block_device(&self, id: &str) -> Result<()> { + info!(sl!(), "remove block device {}", id); + self.handle_request(Request::Sync(VmmAction::RemoveBlockDevice(id.to_string()))) + .with_context(|| format!("Failed to remove block device {:?}", id))?; + Ok(()) + } + + pub fn set_vm_configuration(&self, vm_config: VmConfigInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::SetVmConfiguration( + vm_config.clone(), + ))) + .with_context(|| format!("Failed to set vm configuration {:?}", vm_config))?; + Ok(()) + } + + pub fn insert_network_device(&self, net_cfg: VirtioNetDeviceConfigInfo) -> Result<()> { + self.handle_request_with_retry(Request::Sync(VmmAction::InsertNetworkDevice( + net_cfg.clone(), + ))) + .with_context(|| format!("Failed to insert network device {:?}", net_cfg))?; + Ok(()) + } + + pub fn insert_vsock(&self, vsock_cfg: VsockDeviceConfigInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::InsertVsockDevice( + vsock_cfg.clone(), + ))) + .with_context(|| format!("Failed to insert vsock device {:?}", vsock_cfg))?; + Ok(()) + } + + pub fn insert_fs(&self, fs_cfg: &FsDeviceConfigInfo) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::InsertFsDevice(fs_cfg.clone()))) + .with_context(|| format!("Failed to insert {} fs device {:?}", fs_cfg.mode, fs_cfg))?; + Ok(()) + } + + pub fn patch_fs(&self, cfg: &FsMountConfigInfo, op: ShareFsOperation) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::ManipulateFsBackendFs(cfg.clone()))) + .with_context(|| { + format!( + "Failed to {:?} backend {:?} at {} mount config {:?}", + op, cfg.fstype, cfg.mountpoint, cfg + ) + })?; + Ok(()) + } + + pub fn pause(&self) -> Result<()> { + todo!() + } + + pub fn resume(&self) -> Result<()> { + todo!() + } + + pub fn pid(&self) -> u32 { + std::process::id() + } + + pub fn stop(&mut self) -> Result<()> { + self.handle_request(Request::Sync(VmmAction::ShutdownMicroVm)) + .map_err(|e| { + warn!(sl!(), "Failed to shutdown MicroVM. {}", e); + e + }) + .ok(); + // vmm is not running, join thread will be hang. + if self.is_uninitialized() || self.vmm_thread.is_none() { + debug!(sl!(), "vmm-master thread is uninitialized or has exited."); + return Ok(()); + } + debug!(sl!(), "join vmm-master thread exit."); + + // vmm_thread must be exited, otherwise there will be other sync issues. + // unwrap is safe, if vmm_thread is None, impossible run to here. + self.vmm_thread.take().unwrap().join().ok(); + info!(sl!(), "vmm-master thread join succeed."); + Ok(()) + } + + fn send_request(&self, vmm_action: VmmAction) -> Result { + if let Some(ref to_vmm) = self.to_vmm { + to_vmm + .send(Box::new(vmm_action.clone())) + .with_context(|| format!("Failed to send {:?} via channel ", vmm_action))?; + } else { + return Err(anyhow!("to_vmm is None")); + } + + //notify vmm action + if let Err(e) = self.to_vmm_fd.write(1) { + return Err(anyhow!("failed to notify vmm: {}", e)); + } + + if let Some(from_vmm) = self.from_vmm.as_ref() { + match from_vmm.recv() { + Err(e) => Err(anyhow!("vmm recv err: {}", e)), + Ok(vmm_outcome) => Ok(vmm_outcome), + } + } else { + Err(anyhow!("from_vmm is None")) + } + } + + fn handle_request(&self, req: Request) -> Result { + let Request::Sync(vmm_action) = req; + match self.send_request(vmm_action) { + Ok(vmm_outcome) => match *vmm_outcome { + Ok(vmm_data) => Ok(vmm_data), + Err(vmm_action_error) => Err(anyhow!("vmm action error: {:?}", vmm_action_error)), + }, + Err(e) => Err(e), + } + } + + fn handle_request_with_retry(&self, req: Request) -> Result { + let Request::Sync(vmm_action) = req; + for count in 0..REQUEST_RETRY { + match self.send_request(vmm_action.clone()) { + Ok(vmm_outcome) => match *vmm_outcome { + Ok(vmm_data) => { + info!( + sl!(), + "success to send {:?} after retry {}", &vmm_action, count + ); + return Ok(vmm_data); + } + Err(vmm_action_error) => { + if let VmmActionError::UpcallNotReady = vmm_action_error { + std::thread::sleep(std::time::Duration::from_millis(10)); + continue; + } else { + return Err(vmm_action_error.into()); + } + } + }, + Err(err) => { + return Err(err); + } + } + } + return Err(anyhow::anyhow!( + "After {} attempts, it still doesn't work.", + REQUEST_RETRY + )); + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs new file mode 100644 index 000000000000..d8b20b59723a --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs @@ -0,0 +1,177 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Result}; + +use crate::{VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_PMEM}; + +// Port where the agent will send the logs. Logs are sent through the vsock in cases +// where the hypervisor has no console.sock, i.e dragonball +const VSOCK_LOGS_PORT: &str = "1025"; + +const KERNEL_KV_DELIMITER: &str = "="; +const KERNEL_PARAM_DELIMITER: &str = " "; + +#[derive(Debug, Clone, PartialEq)] +pub struct Param { + pub key: String, + pub value: String, +} + +impl Param { + pub fn new(key: &str, value: &str) -> Self { + Param { + key: key.to_owned(), + value: value.to_owned(), + } + } +} + +#[derive(Debug, PartialEq)] +pub(crate) struct KernelParams { + params: Vec, +} + +impl KernelParams { + pub(crate) fn new(debug: bool) -> Self { + // default kernel params + let mut params = vec![ + Param::new("reboot", "k"), + Param::new("earlyprintk", "ttyS0"), + Param::new("initcall_debug", ""), + Param::new("panic", "1"), + Param::new("systemd.unit", "kata-containers.target"), + Param::new("systemd.mask", "systemd-networkd.service"), + ]; + + if debug { + params.push(Param::new("agent.log_vport", VSOCK_LOGS_PORT)); + } + + Self { params } + } + + pub(crate) fn new_rootfs_kernel_params(rootfs_driver: &str) -> Self { + let params = match rootfs_driver { + VM_ROOTFS_DRIVER_BLK => { + vec![ + Param { + key: "root".to_string(), + value: "/dev/vda1".to_string(), + }, + Param { + key: "rootflags".to_string(), + value: "data=ordered,errors=remount-ro ro".to_string(), + }, + Param { + key: "rootfstype".to_string(), + value: "ext4".to_string(), + }, + ] + } + VM_ROOTFS_DRIVER_PMEM => { + vec![ + Param { + key: "root".to_string(), + value: "/dev/pmem0p1".to_string(), + }, + Param { + key: "rootflags".to_string(), + value: "data=ordered,errors=remount-ro,dax ro".to_string(), + }, + Param { + key: "rootfstype".to_string(), + value: "ext4".to_string(), + }, + ] + } + _ => vec![], + }; + Self { params } + } + + pub(crate) fn append(&mut self, params: &mut KernelParams) { + self.params.append(&mut params.params); + } + + pub(crate) fn from_string(params_string: &str) -> Self { + let mut params = vec![]; + + let parameters_vec: Vec<&str> = params_string.split(KERNEL_PARAM_DELIMITER).collect(); + + for param in parameters_vec.iter() { + if param.is_empty() { + continue; + } + + let ps: Vec<&str> = param.splitn::<_>(2, KERNEL_KV_DELIMITER).collect(); + + if ps.len() == 2 { + params.push(Param { + key: String::from(ps[0]), + value: String::from(ps[1]), + }); + } else { + params.push(Param { + key: String::from(ps[0]), + value: String::from(""), + }); + } + } + + Self { params } + } + + pub(crate) fn to_string(&self) -> Result { + let mut parameters: Vec = Vec::new(); + + for param in &self.params { + if param.key.is_empty() && param.value.is_empty() { + return Err(anyhow!("Empty key and value")); + } else if param.key.is_empty() { + return Err(anyhow!("Empty key")); + } else if param.value.is_empty() { + parameters.push(param.key.to_string()); + } else { + parameters.push(format!( + "{}{}{}", + param.key, KERNEL_KV_DELIMITER, param.value + )); + } + } + + Ok(parameters.join(KERNEL_PARAM_DELIMITER)) + } +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + + use super::*; + + #[test] + fn test_kernel_params() -> Result<()> { + let expect_params_string = "k1=v1 k2=v2 k3=v3".to_string(); + let expect_params = KernelParams { + params: vec![ + Param::new("k1", "v1"), + Param::new("k2", "v2"), + Param::new("k3", "v3"), + ], + }; + + // check kernel params from string + let kernel_params = KernelParams::from_string(&expect_params_string); + assert_eq!(kernel_params, expect_params); + + // check kernel params to string + let kernel_params_string = expect_params.to_string()?; + assert_eq!(kernel_params_string, expect_params_string); + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 0889f3322eb7..095ebd6629c3 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -11,6 +11,10 @@ logging::logger_with_subsystem!(sl, "hypervisor"); pub mod device; pub use device::*; +pub mod dragonball; +mod kernel_param; +pub use kernel_param::Param; +mod utils; use std::collections::HashMap; @@ -18,9 +22,20 @@ use anyhow::Result; use async_trait::async_trait; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; +// Config which driver to use as vm root dev +const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk"; +const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem"; + +#[derive(PartialEq)] +pub(crate) enum VmmState { + NotReady, + VmmServerReady, + VmRunning, +} + +// vcpu mapping from vcpu number to thread number #[derive(Debug)] pub struct VcpuThreadIds { - /// List of tids of vcpu threads (vcpu index, tid) pub vcpus: HashMap, } diff --git a/src/runtime-rs/crates/hypervisor/src/utils.rs b/src/runtime-rs/crates/hypervisor/src/utils.rs new file mode 100644 index 000000000000..8ecf98950022 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/utils.rs @@ -0,0 +1,27 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::collections::HashSet; + +pub fn get_child_threads(pid: u32) -> HashSet { + let mut result = HashSet::new(); + let path_name = format!("/proc/{}/task", pid); + let path = std::path::Path::new(path_name.as_str()); + if path.is_dir() { + if let Ok(dir) = path.read_dir() { + for entity in dir { + if let Ok(entity) = entity.as_ref() { + let file_name = entity.file_name(); + let file_name = file_name.to_str().unwrap_or_default(); + if let Ok(tid) = file_name.parse::() { + result.insert(tid); + } + } + } + } + } + result +} diff --git a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs index fa341a1ae619..a15f09796f08 100644 --- a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs +++ b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs @@ -4,7 +4,7 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::{convert::TryFrom, net::Ipv4Addr}; +use std::convert::TryFrom; use agent::{ARPNeighbor, IPAddress, IPFamily, Interface, Route}; use anyhow::{Context, Result}; @@ -16,7 +16,7 @@ use netlink_packet_route::{ use super::NetworkInfo; use crate::network::utils::{ - address::Address, + address::{parse_ip, Address}, link::{self, LinkAttrs}, }; @@ -66,10 +66,15 @@ async fn handle_addresses(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Resu .set_link_index_filter(attrs.index) .execute(); - let mut addresses = Vec::new(); - while let Some(addr_msg) = addr_msg_list.try_next().await? { - if addr_msg.header.family as i32 != libc::AF_INET { - warn!(sl!(), "unsupported ipv6 addr. {:?}", addr_msg); + let mut addresses = vec![]; + while let Some(addr_msg) = addr_msg_list + .try_next() + .await + .context("try next address msg")? + { + let family = addr_msg.header.family as i32; + if family != libc::AF_INET && family != libc::AF_INET6 { + warn!(sl!(), "unsupported ip family {}", family); continue; } let a = Address::try_from(addr_msg).context("get addr from msg")?; @@ -99,12 +104,13 @@ fn generate_neigh(name: &str, n: &NeighbourMessage) -> Result { for nla in &n.nlas { match nla { Nla::Destination(addr) => { - if addr.len() != 4 { - continue; - } - let dest = Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); + let dest = parse_ip(addr, n.header.family).context("parse ip")?; let addr = Some(IPAddress { - family: IPFamily::V4, + family: if dest.is_ipv4() { + IPFamily::V4 + } else { + IPFamily::V6 + }, address: dest.to_string(), mask: "".to_string(), }); @@ -136,7 +142,11 @@ async fn handle_neighbors( let name = &attrs.name; let mut neighs = vec![]; let mut neigh_msg_list = handle.neighbours().get().execute(); - while let Some(neigh) = neigh_msg_list.try_next().await? { + while let Some(neigh) = neigh_msg_list + .try_next() + .await + .context("try next neigh msg")? + { // get neigh filter with index if neigh.header.ifindex == attrs.index { neighs.push(generate_neigh(name, &neigh).context("generate neigh")?) @@ -170,10 +180,14 @@ fn generate_route(name: &str, route: &RouteMessage) -> Result> { })) } -async fn handle_routes(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result> { +async fn get_route_from_msg( + routes: &mut Vec, + handle: &rtnetlink::Handle, + attrs: &LinkAttrs, + ip_version: rtnetlink::IpVersion, +) -> Result<()> { let name = &attrs.name; - let mut routes = vec![]; - let mut route_msg_list = handle.route().get(rtnetlink::IpVersion::V4).execute(); + let mut route_msg_list = handle.route().get(ip_version).execute(); while let Some(route) = route_msg_list.try_next().await? { // get route filter with index if let Some(index) = route.output_interface() { @@ -184,6 +198,17 @@ async fn handle_routes(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result< } } } + Ok(()) +} + +async fn handle_routes(handle: &rtnetlink::Handle, attrs: &LinkAttrs) -> Result> { + let mut routes = vec![]; + get_route_from_msg(&mut routes, handle, attrs, rtnetlink::IpVersion::V4) + .await + .context("get ip v4 route")?; + get_route_from_msg(&mut routes, handle, attrs, rtnetlink::IpVersion::V6) + .await + .context("get ip v6 route")?; Ok(routes) } diff --git a/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs index 5955af0ff41d..cb47bdad2143 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/route_model.rs @@ -70,13 +70,13 @@ impl NetworkModel for RouteModel { // change sysctl for tap0_kata // echo 1 > /proc/sys/net/ipv4/conf/tap0_kata/accept_local let accept_local_path = format!("/proc/sys/net/ipv4/conf/{}/accept_local", &tap_name); - std::fs::write(&accept_local_path, "1".to_string()) + std::fs::write(&accept_local_path, "1") .with_context(|| format!("Failed to echo 1 > {}", &accept_local_path))?; // echo 1 > /proc/sys/net/ipv4/conf/eth0/proxy_arp // This enabled ARP reply on peer eth0 to prevent without any reply on VPC let proxy_arp_path = format!("/proc/sys/net/ipv4/conf/{}/proxy_arp", &virt_name); - std::fs::write(&proxy_arp_path, "1".to_string()) + std::fs::write(&proxy_arp_path, "1") .with_context(|| format!("Failed to echo 1 > {}", &proxy_arp_path))?; Ok(()) diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index c228a7c8a91b..c8c76b6e25f7 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -196,6 +196,8 @@ async fn create_endpoint( .context("network info from link")?, ); + info!(sl!(), "network info {:?}", network_info); + Ok((endpoint, network_info)) } diff --git a/src/runtime-rs/crates/resource/src/network/utils/address.rs b/src/runtime-rs/crates/resource/src/network/utils/address.rs index 916d011d585e..d481e39da18e 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/address.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/address.rs @@ -41,19 +41,13 @@ impl TryFrom for Address { valid_ltf: 0, }; - let mut local = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); - let mut dst = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); - for nla in nlas.into_iter() { match nla { Nla::Address(a) => { - dst = parse_ip(a, header.family)?; - } - Nla::Local(a) => { - local = parse_ip(a, header.family)?; + addr.addr = parse_ip(&a, header.family)?; } Nla::Broadcast(b) => { - addr.broadcast = parse_ip(b, header.family)?; + addr.broadcast = parse_ip(&b, header.family)?; } Nla::Label(l) => { addr.label = l; @@ -66,27 +60,11 @@ impl TryFrom for Address { } } - // IPv6 sends the local address as IFA_ADDRESS with no - // IFA_LOCAL, IPv4 sends both IFA_LOCAL and IFA_ADDRESS - // with IFA_ADDRESS being the peer address if they differ - // - // But obviously, as there are IPv6 PtP addresses, too, - // IFA_LOCAL should also be handled for IPv6. - if local.is_unspecified() { - if header.family == AF_INET as u8 && local == dst { - addr.addr = dst; - } else { - addr.addr = local; - addr.peer = dst; - } - } else { - addr.addr = dst; - } Ok(addr) } } -fn parse_ip(ip: Vec, family: u8) -> Result { +pub(crate) fn parse_ip(ip: &Vec, family: u8) -> Result { let support_len = if family as u16 == AF_INET { 4 } else { 16 }; if ip.len() != support_len { return Err(anyhow!( diff --git a/src/runtime-rs/crates/resource/src/rootfs/mod.rs b/src/runtime-rs/crates/resource/src/rootfs/mod.rs index 7ea27fe0d611..fcf796e550a0 100644 --- a/src/runtime-rs/crates/resource/src/rootfs/mod.rs +++ b/src/runtime-rs/crates/resource/src/rootfs/mod.rs @@ -58,10 +58,7 @@ impl RootFsResource { // Safe as single_layer_rootfs must have one layer let layer = &mounts_vec[0]; - let rootfs = if let Some(_dev_id) = get_block_device(&layer.source) { - // block rootfs - unimplemented!() - } else if let Some(share_fs) = share_fs { + let rootfs = if let Some(share_fs) = share_fs { // share fs rootfs let share_fs_mount = share_fs.get_share_fs_mount(); share_fs_rootfs::ShareFsRootfs::new(&share_fs_mount, cid, bundle_path, layer) @@ -102,6 +99,7 @@ fn is_single_layer_rootfs(rootfs_mounts: &[Mount]) -> bool { rootfs_mounts.len() == 1 } +#[allow(dead_code)] fn get_block_device(file_path: &str) -> Option { if file_path.is_empty() { return None; diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs index 364614dfd84e..f1a5bc5fe246 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs.rs @@ -15,7 +15,7 @@ use super::utils; pub(crate) const MOUNT_GUEST_TAG: &str = "kataShared"; pub(crate) const PASSTHROUGH_FS_DIR: &str = "passthrough"; -pub(crate) const FS_TYPE_VIRTIO_FS: &str = "virtio_fs"; +pub(crate) const FS_TYPE_VIRTIO_FS: &str = "virtiofs"; pub(crate) const KATA_VIRTIO_FS_DEV_TYPE: &str = "virtio-fs"; const VIRTIO_FS_SOCKET: &str = "virtiofsd.sock"; diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs index e903694ead6d..e3967b8ce376 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_inline.rs @@ -22,10 +22,7 @@ use super::{ }; lazy_static! { - pub(crate) static ref SHARED_DIR_VIRTIO_FS_OPTIONS: Vec:: = vec![ - String::from("default_permissions,allow_other,rootmode=040000,user_id=0,group_id=0"), - String::from("nodev"), - ]; + pub(crate) static ref SHARED_DIR_VIRTIO_FS_OPTIONS: Vec:: = vec![String::from("nodev")]; } #[derive(Debug, Clone)] @@ -70,16 +67,13 @@ impl ShareFs for ShareVirtioFsInline { // setup storage let mut storages: Vec = Vec::new(); - let mut shared_options = SHARED_DIR_VIRTIO_FS_OPTIONS.clone(); - shared_options.push(format!("tag={}", MOUNT_GUEST_TAG)); - let shared_volume: Storage = Storage { driver: String::from(KATA_VIRTIO_FS_DEV_TYPE), driver_options: Vec::new(), source: String::from(MOUNT_GUEST_TAG), fs_type: String::from(FS_TYPE_VIRTIO_FS), fs_group: None, - options: shared_options, + options: SHARED_DIR_VIRTIO_FS_OPTIONS.clone(), mount_point: String::from(KATA_GUEST_SHARE_DIR), }; diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs index b6f143dcd319..ab0ef9af4736 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs @@ -133,6 +133,7 @@ impl ShareVirtioFsStandalone { } } } + inner.pid = None; Ok(()) } diff --git a/src/runtime-rs/crates/resource/src/share_fs/utils.rs b/src/runtime-rs/crates/resource/src/share_fs/utils.rs index bd90d6bd9b51..fbdf93f78e97 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/utils.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/utils.rs @@ -53,9 +53,9 @@ pub(crate) fn get_host_rw_shared_path(id: &str) -> PathBuf { fn do_get_guest_any_path(target: &str, cid: &str, is_volume: bool, is_virtiofs: bool) -> String { let dir = PASSTHROUGH_FS_DIR; let guest_share_dir = if is_virtiofs { - Path::new("/") + Path::new("/").to_path_buf() } else { - Path::new(KATA_GUEST_SHARE_DIR) + Path::new(KATA_GUEST_SHARE_DIR).to_path_buf() }; let path = if is_volume && !is_virtiofs { diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index af2ea082b12c..f82b4473e172 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -19,7 +19,7 @@ slog-scope = "4.4.0" strum = { version = "0.24.0", features = ["derive"] } thiserror = "^1.0" tokio = { version = "1.8.0", features = ["rt-multi-thread", "process", "fs"] } -ttrpc = { version = "0.6.0" } +ttrpc = { version = "0.6.1" } agent = { path = "../../agent" } kata-sys-util = { path = "../../../../libs/kata-sys-util" } diff --git a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs index d74b83b1d096..bf137f689f63 100644 --- a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs +++ b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use anyhow::Result; use async_trait::async_trait; +use kata_types::config::TomlConfig; use tokio::sync::mpsc::Sender; use crate::{message::Message, ContainerManager, Sandbox}; @@ -31,8 +32,12 @@ pub trait RuntimeHandler: Send + Sync { where Self: Sized; - async fn new_instance(&self, sid: &str, msg_sender: Sender) - -> Result; + async fn new_instance( + &self, + sid: &str, + msg_sender: Sender, + config: &TomlConfig, + ) -> Result; fn cleanup(&self, id: &str) -> Result<()>; } diff --git a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml index 81d4e3e03a95..58e6f6012ca3 100644 --- a/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/linux_container/Cargo.toml @@ -10,3 +10,4 @@ async-trait = "0.1.48" tokio = { version = "1.8.0" } common = { path = "../common" } +kata-types = { path = "../../../../libs/kata-types" } \ No newline at end of file diff --git a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs index d50de90b17b0..4a805e3fadf3 100644 --- a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use anyhow::Result; use async_trait::async_trait; use common::{message::Message, RuntimeHandler, RuntimeInstance}; +use kata_types::config::TomlConfig; use tokio::sync::mpsc::Sender; unsafe impl Send for LinuxContainer {} @@ -32,6 +33,7 @@ impl RuntimeHandler for LinuxContainer { &self, _sid: &str, _msg_sender: Sender, + _config: &TomlConfig, ) -> Result { todo!() } diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index cfeab919be8e..131a2761250e 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -55,7 +55,7 @@ impl RuntimeHandlerManagerInner { _ => return Err(anyhow!("Unsupported runtime: {}", &config.runtime.name)), }; let runtime_instance = runtime_handler - .new_instance(&self.id, self.msg_sender.clone()) + .new_instance(&self.id, self.msg_sender.clone(), config) .await .context("new runtime instance")?; @@ -276,6 +276,9 @@ fn load_config(spec: &oci::Spec) -> Result { String::from("") }; info!(sl!(), "get config path {:?}", &config_path); - let (toml_config, _) = TomlConfig::load_from_file(&config_path).context("load toml config")?; + let (mut toml_config, _) = + TomlConfig::load_from_file(&config_path).context("load toml config")?; + annotation.update_config_by_annotation(&mut toml_config)?; + info!(sl!(), "get config content {:?}", &toml_config); Ok(toml_config) } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index cb1c7b2b1cba..d31ee4224826 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -130,8 +130,7 @@ impl Container { &config.container_id, spec.linux .as_ref() - .map(|linux| linux.resources.as_ref()) - .flatten(), + .and_then(|linux| linux.resources.as_ref()), ) .await?; @@ -299,7 +298,7 @@ impl Container { pub async fn pause(&self) -> Result<()> { let inner = self.inner.read().await; - if inner.init_process.status == ProcessStatus::Paused { + if inner.init_process.get_status().await == ProcessStatus::Paused { warn!(self.logger, "container is paused no need to pause"); return Ok(()); } @@ -312,7 +311,7 @@ impl Container { pub async fn resume(&self) -> Result<()> { let inner = self.inner.read().await; - if inner.init_process.status == ProcessStatus::Running { + if inner.init_process.get_status().await == ProcessStatus::Running { warn!(self.logger, "container is running no need to resume"); return Ok(()); } @@ -331,8 +330,8 @@ impl Container { ) -> Result<()> { let logger = logger_with_process(process); let inner = self.inner.read().await; - if inner.init_process.status != ProcessStatus::Running { - warn!(logger, "container is running no need to resume"); + if inner.init_process.get_status().await != ProcessStatus::Running { + warn!(logger, "container is not running"); return Ok(()); } self.agent diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index 2920b23ff877..f9ff08ebfaf2 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -49,8 +49,8 @@ impl ContainerInner { self.init_process.process.container_id() } - pub(crate) fn check_state(&self, states: Vec) -> Result<()> { - let state = self.init_process.status; + pub(crate) async fn check_state(&self, states: Vec) -> Result<()> { + let state = self.init_process.get_status().await; if states.contains(&state) { return Ok(()); } @@ -62,8 +62,9 @@ impl ContainerInner { )) } - pub(crate) fn set_state(&mut self, state: ProcessStatus) { - self.init_process.status = state; + pub(crate) async fn set_state(&mut self, state: ProcessStatus) { + let mut status = self.init_process.status.write().await; + *status = state; } pub(crate) async fn start_exec_process(&mut self, process: &ContainerProcess) -> Result<()> { @@ -79,9 +80,9 @@ impl ContainerInner { process: Some(exec.oci_process.clone()), }) .await - .map(|_| { - exec.process.status = ProcessStatus::Running; - }) + .context("exec process")?; + exec.process.set_status(ProcessStatus::Running).await; + Ok(()) } pub(crate) async fn win_resize_process( @@ -91,6 +92,7 @@ impl ContainerInner { width: u32, ) -> Result<()> { self.check_state(vec![ProcessStatus::Created, ProcessStatus::Running]) + .await .context("check state")?; self.agent @@ -118,6 +120,7 @@ impl ContainerInner { pub(crate) async fn start_container(&mut self, cid: &ContainerID) -> Result<()> { self.check_state(vec![ProcessStatus::Created, ProcessStatus::Stopped]) + .await .context("check state")?; self.agent @@ -127,7 +130,7 @@ impl ContainerInner { .await .context("start container")?; - self.set_state(ProcessStatus::Running); + self.set_state(ProcessStatus::Running).await; Ok(()) } @@ -179,7 +182,7 @@ impl ContainerInner { // close the exit channel to wakeup wait service // send to notify watchers who are waiting for the process exit - self.init_process.stop(); + self.init_process.stop().await; Ok(()) } @@ -192,6 +195,7 @@ impl ContainerInner { info!(logger, "begin to stop process"); // do not stop again when state stopped, may cause multi cleanup resource self.check_state(vec![ProcessStatus::Running]) + .await .context("check state")?; // if use force mode to stop container, stop always successful @@ -215,7 +219,7 @@ impl ContainerInner { .exec_processes .get_mut(&process.exec_id) .ok_or_else(|| anyhow!("failed to find exec"))?; - exec.process.stop(); + exec.process.stop().await; } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs index 155cf0a9c4d2..b4b20bbf36d1 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/manager.rs @@ -96,10 +96,11 @@ impl ContainerManager for VirtContainerManager { let c = containers .get(container_id) .ok_or_else(|| Error::ContainerNotFound(container_id.to_string()))?; + let state = c.state_process(process).await.context("state process"); c.delete_exec_process(process) .await .context("delete process")?; - c.state_process(process).await.context("state process") + return state; } } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs index 28b9b023c7fd..927e2d10c505 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/process.rs @@ -39,7 +39,7 @@ pub struct Process { pub height: u32, pub width: u32, - pub status: ProcessStatus, + pub status: Arc>, pub exit_status: Arc>, pub exit_watcher_rx: Option>, @@ -73,7 +73,7 @@ impl Process { terminal, height: 0, width: 0, - status: ProcessStatus::Created, + status: Arc::new(RwLock::new(ProcessStatus::Created)), exit_status: Arc::new(RwLock::new(ProcessExitStatus::new())), exit_watcher_rx: Some(receiver), exit_watcher_tx: Some(sender), @@ -133,8 +133,8 @@ impl Process { let logger = self.logger.new(o!("io name" => io_name)); let _ = tokio::spawn(async move { match tokio::io::copy(&mut reader, &mut writer).await { - Err(e) => warn!(logger, "io: failed to copy stdin stream {}", e), - Ok(length) => warn!(logger, "io: stop to copy stdin stream length {}", length), + Err(e) => warn!(logger, "io: failed to copy stream {}", e), + Ok(length) => warn!(logger, "io: stop to copy stream length {}", length), }; wgw.done(); @@ -147,8 +147,9 @@ impl Process { let logger = self.logger.clone(); info!(logger, "start run io wait"); let process = self.process.clone(); - let status = self.exit_status.clone(); + let exit_status = self.exit_status.clone(); let exit_notifier = self.exit_watcher_tx.take(); + let status = self.status.clone(); let _ = tokio::spawn(async move { //wait on all of the container's io stream terminated @@ -171,8 +172,13 @@ impl Process { info!(logger, "end wait process exit code {}", resp.status); - let mut locked_status = status.write().await; - locked_status.update_exit_code(resp.status); + let mut exit_status = exit_status.write().await; + exit_status.update_exit_code(resp.status); + drop(exit_status); + + let mut status = status.write().await; + *status = ProcessStatus::Stopped; + drop(status); drop(exit_notifier); info!(logger, "end io wait thread"); @@ -195,17 +201,28 @@ impl Process { stdout: self.stdout.clone(), stderr: self.stderr.clone(), terminal: self.terminal, - status: self.status, + status: self.get_status().await, exit_status: exit_status.exit_code, exited_at: exit_status.exit_time, }) } - pub fn stop(&mut self) { - self.status = ProcessStatus::Stopped; + pub async fn stop(&mut self) { + let mut status = self.status.write().await; + *status = ProcessStatus::Stopped; } pub async fn close_io(&mut self) { self.wg_stdin.wait().await; } + + pub async fn get_status(&self) -> ProcessStatus { + let status = self.status.read().await; + *status + } + + pub async fn set_status(&self, new_status: ProcessStatus) { + let mut status = self.status.write().await; + *status = new_status; + } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs index 1710a8370193..737b1a18f437 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -16,14 +16,16 @@ pub mod sandbox; use std::sync::Arc; use agent::kata::KataAgent; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use common::{message::Message, RuntimeHandler, RuntimeInstance}; -use hypervisor::Hypervisor; +use hypervisor::{dragonball::Dragonball, Hypervisor}; use kata_types::config::{hypervisor::register_hypervisor_plugin, DragonballConfig, TomlConfig}; use resource::ResourceManager; use tokio::sync::mpsc::Sender; +const HYPERVISOR_DRAGONBALL: &str = "dragonball"; + unsafe impl Send for VirtContainer {} unsafe impl Sync for VirtContainer {} pub struct VirtContainer {} @@ -49,14 +51,9 @@ impl RuntimeHandler for VirtContainer { &self, sid: &str, msg_sender: Sender, + config: &TomlConfig, ) -> Result { - let (toml_config, _) = TomlConfig::load_from_file("").context("load config")?; - - // TODO: new sandbox and container manager - // TODO: get from hypervisor - let hypervisor = new_hypervisor(&toml_config) - .await - .context("new hypervisor")?; + let hypervisor = new_hypervisor(config).await.context("new hypervisor")?; // get uds from hypervisor and get config from toml_config let agent = Arc::new(KataAgent::new(kata_types::config::Agent { @@ -77,7 +74,7 @@ impl RuntimeHandler for VirtContainer { sid, agent.clone(), hypervisor.clone(), - &toml_config, + config, )?); let pid = std::process::id(); @@ -104,7 +101,24 @@ impl RuntimeHandler for VirtContainer { } } -async fn new_hypervisor(_toml_config: &TomlConfig) -> Result> { - // TODO: implement ready hypervisor - todo!() +async fn new_hypervisor(toml_config: &TomlConfig) -> Result> { + let hypervisor_name = &toml_config.runtime.hypervisor_name; + let hypervisor_config = toml_config + .hypervisor + .get(hypervisor_name) + .ok_or_else(|| anyhow!("failed to get hypervisor for {}", &hypervisor_name)) + .context("get hypervisor")?; + + // TODO: support other hypervisor + // issue: https://github.com/kata-containers/kata-containers/issues/4634 + match hypervisor_name.as_str() { + HYPERVISOR_DRAGONBALL => { + let mut hypervisor = Dragonball::new(); + hypervisor + .set_hypervisor_config(hypervisor_config.clone()) + .await; + Ok(Arc::new(hypervisor)) + } + _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)), + } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index b98492af47c7..470663bcb5db 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -16,7 +16,10 @@ use common::{ use containerd_shim_protos::events::task::TaskOOM; use hypervisor::Hypervisor; use kata_types::config::TomlConfig; -use resource::{ResourceConfig, ResourceManager}; +use resource::{ + network::{NetworkConfig, NetworkWithNetNsConfig}, + ResourceConfig, ResourceManager, +}; use tokio::sync::{mpsc::Sender, Mutex, RwLock}; use crate::health_check::HealthCheck; @@ -68,19 +71,32 @@ impl VirtSandbox { agent, hypervisor, resource_manager, - monitor: Arc::new(HealthCheck::new(true, true)), + monitor: Arc::new(HealthCheck::new(true, false)), }) } async fn prepare_for_start_sandbox( &self, + _id: &str, netns: Option, - _config: &TomlConfig, + config: &TomlConfig, ) -> Result> { let mut resource_configs = vec![]; - if let Some(_netns_path) = netns { - // TODO: support network + if let Some(netns_path) = netns { + let network_config = ResourceConfig::Network(NetworkConfig::NetworkResourceWithNetNs( + NetworkWithNetNsConfig { + network_model: config.runtime.internetworking_model.clone(), + netns_path, + queues: self + .hypervisor + .hypervisor_config() + .await + .network_info + .network_queues as usize, + }, + )); + resource_configs.push(network_config); } let hypervisor_config = self.hypervisor.hypervisor_config().await; @@ -111,7 +127,7 @@ impl Sandbox for VirtSandbox { // generate device and setup before start vm // should after hypervisor.prepare_vm - let resources = self.prepare_for_start_sandbox(netns, config).await?; + let resources = self.prepare_for_start_sandbox(id, netns, config).await?; self.resource_manager .prepare_before_start_vm(resources) .await diff --git a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml index 9dfce237e258..b8174ee8224b 100644 --- a/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/wasm_container/Cargo.toml @@ -10,3 +10,4 @@ async-trait = "0.1.48" tokio = { version = "1.8.0" } common = { path = "../common" } +kata-types = { path = "../../../../libs/kata-types" } \ No newline at end of file diff --git a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs index c92cd965a0dc..28a81fc49df1 100644 --- a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs @@ -8,8 +8,8 @@ use std::sync::Arc; use anyhow::Result; use async_trait::async_trait; use common::{message::Message, RuntimeHandler, RuntimeInstance}; +use kata_types::config::TomlConfig; use tokio::sync::mpsc::Sender; - unsafe impl Send for WasmContainer {} unsafe impl Sync for WasmContainer {} pub struct WasmContainer {} @@ -32,6 +32,7 @@ impl RuntimeHandler for WasmContainer { &self, _sid: &str, _msg_sender: Sender, + _config: &TomlConfig, ) -> Result { todo!() } diff --git a/src/runtime-rs/crates/service/Cargo.toml b/src/runtime-rs/crates/service/Cargo.toml index b3aa85a64fff..6d7f64ff5d7b 100644 --- a/src/runtime-rs/crates/service/Cargo.toml +++ b/src/runtime-rs/crates/service/Cargo.toml @@ -10,7 +10,7 @@ async-trait = "0.1.48" slog = "2.5.2" slog-scope = "4.4.0" tokio = { version = "1.8.0", features = ["rt-multi-thread"] } -ttrpc = { version = "0.6.0" } +ttrpc = { version = "0.6.1" } common = { path = "../runtimes/common" } containerd-shim-protos = { version = "0.2.0", features = ["async"]} diff --git a/src/runtime-rs/crates/service/src/task_service.rs b/src/runtime-rs/crates/service/src/task_service.rs index 77c368d6c9bc..447207a85125 100644 --- a/src/runtime-rs/crates/service/src/task_service.rs +++ b/src/runtime-rs/crates/service/src/task_service.rs @@ -47,9 +47,8 @@ where .await .map_err(|err| ttrpc::Error::Others(format!("failed to handler message {:?}", err)))?; debug!(logger, "<==== task service {:?}", &resp); - Ok(resp - .try_into() - .map_err(|err| ttrpc::Error::Others(format!("failed to translate to shim {:?}", err)))?) + resp.try_into() + .map_err(|err| ttrpc::Error::Others(format!("failed to translate to shim {:?}", err))) } macro_rules! impl_service { diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml index c1a1d1d79d0f..87d0533e4404 100644 --- a/src/runtime-rs/crates/shim/Cargo.toml +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -22,8 +22,8 @@ log = "0.4.14" nix = "0.23.1" protobuf = "2.27.0" sha2 = "=0.9.3" -slog = {version = "2.7.0", features = ["std", "release_max_level_trace", "max_level_trace"]} -slog-async = "2.7.0" +slog = {version = "2.5.2", features = ["std", "release_max_level_trace", "max_level_trace"]} +slog-async = "2.5.2" slog-scope = "4.4.0" slog-stdlog = "4.1.0" thiserror = "1.0.30" diff --git a/src/runtime-rs/crates/shim/src/panic_hook.rs b/src/runtime-rs/crates/shim/src/panic_hook.rs index 0b0f4e1db8f8..88dbf305a66b 100644 --- a/src/runtime-rs/crates/shim/src/panic_hook.rs +++ b/src/runtime-rs/crates/shim/src/panic_hook.rs @@ -4,10 +4,12 @@ // SPDX-License-Identifier: Apache-2.0 // -use std::{boxed::Box, ops::Deref}; +use std::{boxed::Box, fs::OpenOptions, io::Write, ops::Deref}; use backtrace::Backtrace; +const KMESG_DEVICE: &str = "/dev/kmsg"; + // TODO: the Kata 1.x runtime had a SIGUSR1 handler that would log a formatted backtrace on // receiving that signal. It could be useful to re-add that feature. pub(crate) fn set_panic_hook() { @@ -36,6 +38,20 @@ pub(crate) fn set_panic_hook() { "A panic occurred at {}:{}: {}\r\n{:?}", filename, line, cause, bt_data ); + // print panic log to dmesg + // The panic log size is too large to /dev/kmsg, so write by line. + if let Ok(mut file) = OpenOptions::new().write(true).open(KMESG_DEVICE) { + file.write_all( + format!("A panic occurred at {}:{}: {}", filename, line, cause).as_bytes(), + ) + .ok(); + let lines: Vec<&str> = bt_data.split('\n').collect(); + for line in lines { + file.write_all(line.as_bytes()).ok(); + } + + file.flush().ok(); + } std::process::abort(); })); } diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index b476b4985659..a7629599e73e 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -509,19 +509,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nix" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e06129fb611568ef4e868c14b326274959aa70ff7776e9d55323531c374945" -dependencies = [ - "bitflags", - "cc", - "cfg-if 1.0.0", - "libc", - "memoffset", -] - [[package]] name = "nix" version = "0.23.1" diff --git a/src/tools/runk/Cargo.lock b/src/tools/runk/Cargo.lock index a1691a796525..415766d1bdbf 100644 --- a/src/tools/runk/Cargo.lock +++ b/src/tools/runk/Cargo.lock @@ -63,16 +63,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.1.0" @@ -127,7 +117,7 @@ checksum = "cdae996d9638ba03253ffa1c93345a585974a97abbdeab9176c77922f3efc1e8" dependencies = [ "libc", "log", - "nix 0.23.1", + "nix", "regex", ] @@ -495,15 +485,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "itertools" version = "0.10.3" @@ -540,7 +521,7 @@ dependencies = [ "derive_builder", "libc", "logging", - "nix 0.23.1", + "nix", "oci", "rustjail", "serde", @@ -632,19 +613,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nix" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" -dependencies = [ - "bitflags", - "cc", - "cfg-if 0.1.10", - "libc", - "void", -] - [[package]] name = "nix" version = "0.23.1" @@ -830,7 +798,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ - "bytes 1.1.0", + "bytes", "prost-derive", ] @@ -840,7 +808,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" dependencies = [ - "bytes 1.1.0", + "bytes", "heck 0.3.3", "itertools", "log", @@ -871,15 +839,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ - "bytes 1.1.0", + "bytes", "prost", ] [[package]] name = "protobuf" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" dependencies = [ "serde", "serde_derive", @@ -887,18 +855,18 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de113bba758ccf2c1ef816b127c958001b7831136c9bc3f8e9ec695ac4e82b0c" +checksum = "aec1632b7c8f2e620343439a7dfd1f3c47b18906c4be58982079911482b5d707" dependencies = [ "protobuf", ] [[package]] name = "protobuf-codegen-pure" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1a4febc73bf0cada1d77c459a0c8e5973179f1cfd5b0f1ab789d45b17b6440" +checksum = "9f8122fdb18e55190c796b088a16bdb70cd7acdcd48f7a8b796b58c62e532cc6" dependencies = [ "protobuf", "protobuf-codegen", @@ -908,7 +876,7 @@ dependencies = [ name = "protocols" version = "0.1.0" dependencies = [ - "async-trait", + "oci", "protobuf", "ttrpc", "ttrpc-codegen", @@ -978,7 +946,7 @@ dependencies = [ "libcontainer", "liboci-cli", "logging", - "nix 0.23.1", + "nix", "oci", "rustjail", "serde", @@ -1004,7 +972,7 @@ dependencies = [ "inotify", "lazy_static", "libc", - "nix 0.23.1", + "nix", "oci", "path-absolutize", "protobuf", @@ -1269,7 +1237,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" dependencies = [ - "bytes 1.1.0", + "bytes", "libc", "memchr", "mio", @@ -1294,36 +1262,19 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-vsock" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6" -dependencies = [ - "bytes 0.4.12", - "futures", - "libc", - "tokio", - "vsock", -] - [[package]] name = "ttrpc" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "004604e91de38bc16cb9c7898187343075388ea414ad24896a21fc4e91a7c861" +checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8" dependencies = [ - "async-trait", "byteorder", - "futures", "libc", "log", - "nix 0.16.1", + "nix", "protobuf", "protobuf-codegen-pure", "thiserror", - "tokio", - "tokio-vsock", ] [[package]] @@ -1387,22 +1338,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - -[[package]] -name = "vsock" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" -dependencies = [ - "libc", - "nix 0.23.1", -] - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" diff --git a/src/tools/trace-forwarder/Cargo.lock b/src/tools/trace-forwarder/Cargo.lock index ab87c9db7112..5357d1ec9490 100644 --- a/src/tools/trace-forwarder/Cargo.lock +++ b/src/tools/trace-forwarder/Cargo.lock @@ -528,9 +528,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.14.0" +version = "2.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" [[package]] name = "quote" From 1befbe6738cbae85b37da2a30178ecf2c47bebfa Mon Sep 17 00:00:00 2001 From: Chao Wu Date: Thu, 30 Jun 2022 21:16:47 +0800 Subject: [PATCH 0123/1953] runtime-rs: Cargo lock for fix version problem Cargo lock for fix version problem Signed-off-by: Chao Wu --- src/runtime-rs/Cargo.lock | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 47651d8c8985..606170aa3705 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -420,14 +420,13 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.3.4" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" +checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c" dependencies = [ "cfg-if 1.0.0", - "hashbrown 0.12.1", - "lock_api", - "parking_lot_core 0.9.3", + "num_cpus", + "parking_lot 0.12.1", ] [[package]] @@ -946,9 +945,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" [[package]] name = "git2" -version = "0.14.4" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" +checksum = "3826a6e0e2215d7a41c2bfc7c9244123969273f3476b939a226aac0ab56e9e3c" dependencies = [ "bitflags", "libc", @@ -995,12 +994,6 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -[[package]] -name = "hashbrown" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" - [[package]] name = "heck" version = "0.3.3" @@ -1078,7 +1071,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg", - "hashbrown 0.11.2", + "hashbrown", ] [[package]] @@ -1223,9 +1216,9 @@ checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" -version = "0.13.4+1.4.2" +version = "0.13.2+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" +checksum = "3a42de9a51a5c12e00fc0e4ca6bc2ea43582fc6418488e8f615e905d886f258b" dependencies = [ "cc", "libc", From 274598ae56af309dbfe44feda714cc8b9448eb01 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Wed, 6 Jul 2022 17:07:22 +0800 Subject: [PATCH 0124/1953] kata-runtime: add dragonball config check support. add dragonball config check support. Signed-off-by: wllenyj --- .../cmd/kata-runtime/kata-check_amd64.go | 2 ++ src/runtime/pkg/katautils/config.go | 28 +++++++++++++++++++ src/runtime/virtcontainers/hypervisor.go | 6 ++++ .../virtcontainers/hypervisor_linux.go | 2 ++ 4 files changed, 38 insertions(+) diff --git a/src/runtime/cmd/kata-runtime/kata-check_amd64.go b/src/runtime/cmd/kata-runtime/kata-check_amd64.go index 46b3a2916561..09f5bfe1793f 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_amd64.go +++ b/src/runtime/cmd/kata-runtime/kata-check_amd64.go @@ -107,6 +107,8 @@ func setCPUtype(hypervisorType vc.HypervisorType) error { fallthrough case "clh": fallthrough + case "dragonball": + fallthrough case "qemu": archRequiredCPUFlags = map[string]string{ cpuFlagVMX: "Virtualization support", diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 0903c8ea9ed0..fe93a84128fe 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -51,6 +51,7 @@ const ( clhHypervisorTableType = "clh" qemuHypervisorTableType = "qemu" acrnHypervisorTableType = "acrn" + dragonballHypervisorTableType = "dragonball" // the maximum amount of PCI bridges that can be cold plugged in a VM maxPCIBridges uint32 = 5 @@ -989,6 +990,30 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { }, nil } +func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err + } + image, err := h.image() + if err != nil { + return vc.HypervisorConfig{}, err + } + kernelParams := h.kernelParams() + + return vc.HypervisorConfig{ + KernelPath: kernel, + ImagePath: image, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + EntropySource: h.GetEntropySource(), + Debug: h.Debug, + }, nil +} + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { if f.TemplatePath == "" { f.TemplatePath = defaultTemplatePath @@ -1022,6 +1047,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi case clhHypervisorTableType: config.HypervisorType = vc.ClhHypervisor hConfig, err = newClhHypervisorConfig(hypervisor) + case dragonballHypervisorTableType: + config.HypervisorType = vc.DragonballHypervisor + hConfig, err = newDragonballHypervisorConfig(hypervisor) } if err != nil { diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 48eda0977881..b82700a0e1e0 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -46,6 +46,9 @@ const ( // ClhHypervisor is the ICH hypervisor. ClhHypervisor HypervisorType = "clh" + // DragonballHypervisor is the Dragonball hypervisor. + DragonballHypervisor HypervisorType = "dragonball" + // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" @@ -169,6 +172,9 @@ func (hType *HypervisorType) Set(value string) error { case "clh": *hType = ClhHypervisor return nil + case "dragonball": + *hType = DragonballHypervisor + return nil case "mock": *hType = MockHypervisor return nil diff --git a/src/runtime/virtcontainers/hypervisor_linux.go b/src/runtime/virtcontainers/hypervisor_linux.go index 1fd0375b81da..3d81c1ada019 100644 --- a/src/runtime/virtcontainers/hypervisor_linux.go +++ b/src/runtime/virtcontainers/hypervisor_linux.go @@ -37,6 +37,8 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) { return &Acrn{}, nil case ClhHypervisor: return &cloudHypervisor{}, nil + case DragonballHypervisor: + return &mockHypervisor{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: From 3c989521b1bbd7315f47c6729ed0c893d7eb3778 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Mon, 11 Jul 2022 07:57:40 +0800 Subject: [PATCH 0125/1953] dragonball: update for review update for review Fixes: #3785 Signed-off-by: Quanwei Zhou --- src/dragonball/src/api/v1/boot_source.rs | 2 +- src/dragonball/src/api/v1/vmm_action.rs | 5 ++- .../src/device_manager/blk_dev_mgr.rs | 7 ++-- src/dragonball/src/device_manager/legacy.rs | 8 ++--- src/dragonball/src/device_manager/mod.rs | 35 ++++++++++--------- src/dragonball/src/error.rs | 2 +- .../resource/src/network/utils/address.rs | 2 +- 7 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/dragonball/src/api/v1/boot_source.rs b/src/dragonball/src/api/v1/boot_source.rs index 0094c8d7c7b0..8ff7e030dc46 100644 --- a/src/dragonball/src/api/v1/boot_source.rs +++ b/src/dragonball/src/api/v1/boot_source.rs @@ -5,7 +5,7 @@ use serde_derive::{Deserialize, Serialize}; /// Default guest kernel command line: -/// - `reboot=k` shut down the guest on reboot, instead of well... rebooting; +/// - `reboot=k` shutdown the guest on reboot, instead of well... rebooting; /// - `panic=1` on panic, reboot after 1 second; /// - `pci=off` do not scan for PCI devices (ser boot time); /// - `nomodules` disable loadable kernel module support; diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 4510b7e03d8a..06004f3f0caf 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -332,7 +332,7 @@ impl VmmService { Ok(VmmData::Empty) } - /// Set virtual machine configuration configurations. + /// Set virtual machine configuration. pub fn set_vm_configuration( &mut self, vmm: &mut Vmm, @@ -498,8 +498,7 @@ impl VmmService { } #[cfg(feature = "virtio-blk")] - // Only call this function as part of the API. - // If the drive_id does not exist, a new Block Device Config is added to the list. + // Remove the device fn remove_block_device( &mut self, vmm: &mut Vmm, diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index 9b3ee9041748..e4688b4f6fab 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -372,8 +372,7 @@ impl BlockDeviceMgr { for info in mgr.info_list.iter() { info.config.check_conflicts(&config)?; } - let config2 = config.clone(); - let index = mgr.create(config2)?; + let index = mgr.create(config.clone())?; if !ctx.is_hotplug { return Ok(()); } @@ -497,8 +496,6 @@ impl BlockDeviceMgr { ) -> std::result::Result>, virtio::Error> { let epoll_mgr = ctx.epoll_mgr.clone().ok_or(virtio::Error::InvalidInput)?; - // Safe to unwrap() because we have verified it when parsing device type. - //let path = cfg.path_on_host.to_str().unwrap(); let mut block_files: Vec> = vec![]; match cfg.device_type { @@ -762,7 +759,7 @@ impl BlockDeviceMgr { } impl Default for BlockDeviceMgr { - /// Constructor for the BlockDeviceConfigs. It initializes an empty LinkedList. + /// Constructor for the BlockDeviceMgr. It initializes an empty LinkedList. fn default() -> BlockDeviceMgr { BlockDeviceMgr { info_list: VecDeque::::new(), diff --git a/src/dragonball/src/device_manager/legacy.rs b/src/dragonball/src/device_manager/legacy.rs index 59f4206ab9e8..50a47cab7310 100644 --- a/src/dragonball/src/device_manager/legacy.rs +++ b/src/dragonball/src/device_manager/legacy.rs @@ -12,9 +12,9 @@ use std::io; use std::sync::{Arc, Mutex}; use dbs_device::device_manager::Error as IoManagerError; -use dbs_legacy_devices::SerialDevice; #[cfg(target_arch = "aarch64")] use dbs_legacy_devices::RTCDevice; +use dbs_legacy_devices::SerialDevice; use vmm_sys_util::eventfd::EventFd; // The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself. @@ -149,10 +149,10 @@ pub(crate) mod x86_64 { #[cfg(target_arch = "aarch64")] pub(crate) mod aarch64 { use super::*; - use dbs_device::device_manager::{IoManager}; + use dbs_device::device_manager::IoManager; use dbs_device::resources::DeviceResources; - use std::collections::HashMap; use kvm_ioctls::VmFd; + use std::collections::HashMap; type Result = ::std::result::Result; @@ -194,7 +194,7 @@ pub(crate) mod aarch64 { ) -> Result<(Arc>, EventFd)> { let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; let device = Arc::new(Mutex::new(SerialDevice::new( - eventfd.try_clone().map_err(Error::EventFd)? + eventfd.try_clone().map_err(Error::EventFd)?, ))); bus.register_device_io(device.clone(), resources.get_all_resources()) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index c37191ab026c..43c237d4ce72 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -3,18 +3,20 @@ //! Device manager to manage IO devices for a virtual machine. +#[cfg(target_arch = "aarch64")] +use std::collections::HashMap; + use std::io; use std::sync::{Arc, Mutex, MutexGuard}; -use std::collections::HashMap; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; #[cfg(target_arch = "aarch64")] use dbs_arch::{DeviceType, MMIODeviceInfo}; use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext}; -use dbs_device::resources::Resource; #[cfg(target_arch = "aarch64")] use dbs_device::resources::DeviceResources; +use dbs_device::resources::Resource; use dbs_device::DeviceIo; use dbs_interrupt::KvmIrqManager; use dbs_legacy_devices::ConsoleHandler; @@ -54,7 +56,10 @@ pub mod console_manager; pub use self::console_manager::ConsoleManager; mod legacy; -pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager, aarch64::{COM1, COM2, RTC}}; +pub use self::legacy::{Error as LegacyDeviceError, LegacyDeviceManager}; + +#[cfg(target_arch = "aarch64")] +pub use self::legacy::aarch64::{COM1, COM2, RTC}; #[cfg(feature = "virtio-vsock")] /// Device manager for user-space vsock devices. @@ -333,9 +338,7 @@ impl DeviceOpContext { let (mmio_base, mmio_size, irq) = DeviceManager::get_virtio_mmio_device_info(device)?; let dev_type; let device_id; - if let Some(mmiov2_device) = - device.as_any().downcast_ref::() - { + if let Some(mmiov2_device) = device.as_any().downcast_ref::() { dev_type = mmiov2_device.get_device_type(); device_id = None; } else { @@ -534,19 +537,20 @@ impl DeviceManager { &mut self, ctx: &mut DeviceOpContext, ) -> std::result::Result<(), StartMicroVmError> { - #[cfg( - any(target_arch = "x86_64", - all(target_arch = "aarch64", feature = "dbs-virtio-devices") - ) - )] + #[cfg(any( + target_arch = "x86_64", + all(target_arch = "aarch64", feature = "dbs-virtio-devices") + ))] { let mut tx = ctx.io_context.begin_tx(); let legacy_manager; #[cfg(target_arch = "x86_64")] { - let legacy_manager = - LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + legacy_manager = LegacyDeviceManager::create_manager( + &mut tx.io_manager, + Some(self.vm_fd.clone()), + ); } #[cfg(target_arch = "aarch64")] @@ -817,10 +821,7 @@ impl DeviceManager { .get_legacy_irq() .ok_or(DeviceMgrError::GetDeviceResource)?; - if let Some(mmio_dev) = device - .as_any() - .downcast_ref::() - { + if let Some(mmio_dev) = device.as_any().downcast_ref::() { if let Resource::MmioAddressRange { base, size } = mmio_dev.get_mmio_cfg_res() { return Ok((base, size, irq)); } diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 454ad45b59f0..35cf639bb27d 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,7 +12,7 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; -use crate::{address_space_manager, device_manager, vcpu, vm, resource_manager}; +use crate::{address_space_manager, device_manager, resource_manager, vcpu, vm}; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; diff --git a/src/runtime-rs/crates/resource/src/network/utils/address.rs b/src/runtime-rs/crates/resource/src/network/utils/address.rs index d481e39da18e..0484c9f364d0 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/address.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/address.rs @@ -64,7 +64,7 @@ impl TryFrom for Address { } } -pub(crate) fn parse_ip(ip: &Vec, family: u8) -> Result { +pub(crate) fn parse_ip(ip: &[u8], family: u8) -> Result { let support_len = if family as u16 == AF_INET { 4 } else { 16 }; if ip.len() != support_len { return Err(anyhow!( From d93e4b939d0ab4ab512eb05d0ea5389c909a34ab Mon Sep 17 00:00:00 2001 From: Fupan Li Date: Thu, 14 Jul 2022 16:39:49 +0800 Subject: [PATCH 0126/1953] container: kill all of the processes in this container When a container terminated, we should make sure there's no processes left after destroying the container. Before this commit, kata-agent depended on the kernel's pidns to destroy all of the process in a container after the 1 process exit in a container. This is true for those container using a separated pidns, but for the case of shared pidns within the sandbox, the container exit wouldn't trigger the pidns terminated, and there would be some daemon process left in this container, this wasn't expected. Fixes: #4663 Signed-off-by: Fupan Li --- src/agent/rustjail/src/container.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index f95aaffd8138..2c360cf164ef 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -1092,6 +1092,16 @@ impl BaseContainer for LinuxContainer { fs::remove_dir_all(&self.root)?; if let Some(cgm) = self.cgroup_manager.as_mut() { + // Kill all of the processes created in this container to prevent + // the leak of some daemon process when this container shared pidns + // with the sandbox. + let pids = cgm.get_pids().context("get cgroup pids")?; + for i in pids { + if let Err(e) = signal::kill(Pid::from_raw(i), Signal::SIGKILL) { + warn!(self.logger, "kill the process {} error: {:?}", i, e); + } + } + cgm.destroy().context("destroy cgroups")?; } Ok(()) From f690b0aad0ef5900d9b9648a5532b8b7ce64c518 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 14 Jul 2022 19:21:47 -0700 Subject: [PATCH 0127/1953] qemu: Add liburing to qemu build io_uring is a Linux API for asynchronous I/O introduced in qemu 5.0. It is designed to better performance than older aio API. We could leverage this in order to get better storage performance. We should be adding liburing-dev to qemu build to leverage this feature. However liburing-dev package is not available in ubuntu 20.04, it is avaiable in 22.04. Upgrading the ubuntu version in the dockerfile to 22.04 is causing issues in the static qemu build related to libpmem. So instead we are building liburing from source until those build issues are solved. Fixes: #4645 Signed-off-by: Archana Shinde --- tools/packaging/static-build/qemu/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/qemu/Dockerfile b/tools/packaging/static-build/qemu/Dockerfile index 61cc6ce9516c..68088402175a 100644 --- a/tools/packaging/static-build/qemu/Dockerfile +++ b/tools/packaging/static-build/qemu/Dockerfile @@ -11,6 +11,7 @@ WORKDIR /root/qemu # This is required to keep build dependencies with security fixes. ARG CACHE_TIMEOUT RUN echo "$CACHE_TIMEOUT" +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get upgrade -y && \ apt-get --no-install-recommends install -y \ @@ -59,6 +60,12 @@ ARG BUILD_SUFFIX ARG QEMU_DESTDIR ARG QEMU_TARBALL +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN git clone https://github.com/axboe/liburing/ ~/liburing && \ + cd ~/liburing && \ + git checkout tags/liburing-2.1 && \ + make && make install && ldconfig + COPY scripts/configure-hypervisor.sh /root/configure-hypervisor.sh COPY qemu /root/kata_qemu COPY scripts/apply_patches.sh /root/apply_patches.sh @@ -66,7 +73,6 @@ COPY scripts/patch_qemu.sh /root/patch_qemu.sh COPY static-build/scripts/qemu-build-post.sh /root/static-build/scripts/qemu-build-post.sh COPY static-build/qemu.blacklist /root/static-build/qemu.blacklist -SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN git clone --depth=1 "${QEMU_REPO}" qemu && \ cd qemu && \ git fetch --depth=1 origin "${QEMU_VERSION}" && git checkout FETCH_HEAD && \ From 996a6b80bcaeeaef58ae737aa9c921161d7a81ea Mon Sep 17 00:00:00 2001 From: liubin Date: Fri, 15 Jul 2022 16:50:29 +0800 Subject: [PATCH 0128/1953] kata-sys-util: upgrade nix version New nix is supporting UMOUNT_NOFOLLOW, upgrade nix version to use this flag instead of the self-defined flag. Fixes: #4670 Signed-off-by: liubin --- src/agent/Cargo.lock | 16 ++++++++++++++-- src/agent/Cargo.toml | 2 +- src/agent/src/console.rs | 4 ++-- src/agent/src/main.rs | 4 ++-- src/libs/kata-sys-util/Cargo.toml | 2 +- src/libs/kata-sys-util/src/mount.rs | 5 +---- src/runtime-rs/Cargo.lock | 11 ++++++----- src/runtime-rs/crates/hypervisor/Cargo.toml | 2 +- src/runtime-rs/crates/resource/Cargo.toml | 2 +- .../src/share_fs/share_virtio_fs_standalone.rs | 2 +- src/runtime-rs/crates/runtimes/common/Cargo.toml | 2 +- src/runtime-rs/crates/shim/Cargo.toml | 2 +- 12 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index fb10ad35b19d..ee32dc420b66 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -631,7 +631,7 @@ dependencies = [ "logging", "netlink-packet-utils", "netlink-sys", - "nix 0.23.1", + "nix 0.24.1", "oci", "opentelemetry", "procfs", @@ -674,7 +674,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix 0.23.1", + "nix 0.24.1", "oci", "once_cell", "rand 0.7.3", @@ -918,6 +918,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "libc", + "memoffset", +] + [[package]] name = "ntapi" version = "0.3.7" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 49c7e32a36b3..1d20766b2eec 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -12,7 +12,7 @@ lazy_static = "1.3.0" ttrpc = { version = "0.6.0", features = ["async"], default-features = false } protobuf = "2.27.0" libc = "0.2.58" -nix = "0.23.0" +nix = "0.24.1" capctl = "0.2.0" serde_json = "1.0.39" scan_fmt = "0.2.3" diff --git a/src/agent/src/console.rs b/src/agent/src/console.rs index c705af1b7173..8f1ae5ff32af 100644 --- a/src/agent/src/console.rs +++ b/src/agent/src/console.rs @@ -9,7 +9,7 @@ use anyhow::{anyhow, Result}; use nix::fcntl::{self, FcntlArg, FdFlag, OFlag}; use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use nix::pty::{openpty, OpenptyResult}; -use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType}; +use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr}; use nix::sys::stat::Mode; use nix::sys::wait; use nix::unistd::{self, close, dup2, fork, setsid, ForkResult, Pid}; @@ -67,7 +67,7 @@ pub async fn debug_console_handler( SockFlag::SOCK_CLOEXEC, None, )?; - let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port); + let addr = VsockAddr::new(libc::VMADDR_CID_ANY, port); socket::bind(listenfd, &addr)?; socket::listen(listenfd, 1)?; diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 6a023e093e70..ecf7a7b9df08 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -22,7 +22,7 @@ extern crate slog; use anyhow::{anyhow, Context, Result}; use clap::{AppSettings, Parser}; use nix::fcntl::OFlag; -use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType}; +use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr}; use nix::unistd::{self, dup, Pid}; use std::env; use std::ffi::OsStr; @@ -132,7 +132,7 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver>(mountpoint: P, lazy_umount: bool) -> Result<() // Counterpart of nix::umount2, with support of `UMOUNT_FOLLOW`. fn umount2>(path: P, lazy_umount: bool) -> std::io::Result<()> { let path_ptr = path.as_ref().as_os_str().as_bytes().as_ptr() as *const c_char; - let mut flags = UMOUNT_NOFOLLOW; + let mut flags = MntFlags::UMOUNT_NOFOLLOW.bits(); if lazy_umount { flags |= MntFlags::MNT_DETACH.bits(); } diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 606170aa3705..8a0b9c0b8587 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -294,7 +294,7 @@ dependencies = [ "kata-sys-util", "kata-types", "lazy_static", - "nix 0.23.1", + "nix 0.24.1", "oci", "protobuf", "serde_json", @@ -1037,7 +1037,7 @@ dependencies = [ "kata-types", "libc", "logging", - "nix 0.16.1", + "nix 0.24.1", "seccompiler", "serde_json", "slog", @@ -1153,7 +1153,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix 0.23.1", + "nix 0.24.1", "oci", "once_cell", "rand 0.7.3", @@ -1462,6 +1462,7 @@ dependencies = [ "bitflags", "cfg-if 1.0.0", "libc", + "memoffset", ] [[package]] @@ -2029,7 +2030,7 @@ dependencies = [ "logging", "netlink-packet-route", "netlink-sys", - "nix 0.16.1", + "nix 0.24.1", "oci", "rand 0.7.3", "rtnetlink", @@ -2261,7 +2262,7 @@ dependencies = [ "libc", "log", "logging", - "nix 0.23.1", + "nix 0.24.1", "oci", "protobuf", "rand 0.8.5", diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index 3201b4ee26d7..0de423b6bf78 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -12,7 +12,7 @@ async-trait = "0.1.48" dbs-utils = "0.1.0" go-flag = "0.1.0" libc = ">=0.2.39" -nix = "0.16.1" +nix = "0.24.1" seccompiler = "0.2.0" serde_json = ">=1.0.9" slog = "2.5.2" diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index e5fe51bb2e5a..463c5d50716c 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -14,7 +14,7 @@ lazy_static = "1.4.0" libc = ">=0.2.39" netlink-sys = "0.8.3" netlink-packet-route = "0.12.0" -nix = "0.16.0" +nix = "0.24.1" rand = "^0.7.2" rtnetlink = "0.10.0" scopeguard = "1.0.0" diff --git a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs index ab0ef9af4736..9c798d74672b 100644 --- a/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs +++ b/src/runtime-rs/crates/resource/src/share_fs/share_virtio_fs_standalone.rs @@ -128,7 +128,7 @@ impl ShareVirtioFsStandalone { info!(sl!(), "shutdown virtiofsd pid {}", pid); let pid = ::nix::unistd::Pid::from_raw(pid as i32); if let Err(err) = ::nix::sys::signal::kill(pid, nix::sys::signal::SIGKILL) { - if err != ::nix::Error::Sys(nix::errno::Errno::ESRCH) { + if err != ::nix::Error::ESRCH { return Err(anyhow!("failed to kill virtiofsd pid {} {}", pid, err)); } } diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index f82b4473e172..56cec862c2fb 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -11,7 +11,7 @@ anyhow = "^1.0" async-trait = "0.1.48" containerd-shim-protos = { version = "0.2.0", features = ["async"]} lazy_static = "1.4.0" -nix = "0.23.1" +nix = "0.24.1" protobuf = "2.27.0" serde_json = "1.0.39" slog = "2.5.2" diff --git a/src/runtime-rs/crates/shim/Cargo.toml b/src/runtime-rs/crates/shim/Cargo.toml index 87d0533e4404..ecc0e4751df5 100644 --- a/src/runtime-rs/crates/shim/Cargo.toml +++ b/src/runtime-rs/crates/shim/Cargo.toml @@ -19,7 +19,7 @@ containerd-shim-protos = { version = "0.2.0", features = ["async"]} go-flag = "0.1.0" libc = "0.2.108" log = "0.4.14" -nix = "0.23.1" +nix = "0.24.1" protobuf = "2.27.0" sha2 = "=0.9.3" slog = {version = "2.5.2", features = ["std", "release_max_level_trace", "max_level_trace"]} From 2b01e9ba40efd8b7f699ff03221aac56b0dfecf3 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 18 Jul 2022 09:31:33 +0800 Subject: [PATCH 0129/1953] dragonball: fix warning Add map_err for vcpu_manager.set_reset_event_fd() function. Fixes: #4676 Signed-off-by: xuejun-xj --- src/dragonball/src/vm/aarch64.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index ffecaf356feb..7e249f501650 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -98,7 +98,8 @@ impl Vm { ); self.vcpu_manager() .map_err(StartMicroVmError::Vcpu)? - .set_reset_event_fd(reset_eventfd); + .set_reset_event_fd(reset_eventfd) + .map_err(StartMicroVmError::Vcpu)?; // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP From d8920b00cd026b17a855922c77a6208f6cbd65b7 Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Wed, 13 Jul 2022 15:30:03 +0800 Subject: [PATCH 0130/1953] runtime-rs: support functionalities of ipvlan endpoint Add support for ipvlan endpoint Fixes: #4655 Signed-off-by: Ji-Xinyou --- .../src/network/endpoint/ipvlan_endpoint.rs | 90 +++++++++++++++++++ .../resource/src/network/endpoint/mod.rs | 2 + .../resource/src/network/network_model/mod.rs | 4 +- .../resource/src/network/network_pair.rs | 1 + .../src/network/network_with_netns.rs | 8 +- 5 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs new file mode 100644 index 000000000000..5f3100278677 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/ipvlan_endpoint.rs @@ -0,0 +1,90 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{self, Error}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use super::Endpoint; +use crate::network::network_model::TC_FILTER_NET_MODEL_STR; +use crate::network::{utils, NetworkPair}; +use hypervisor::{device::NetworkConfig, Device, Hypervisor}; + +// IPVlanEndpoint is the endpoint bridged to VM +#[derive(Debug)] +pub struct IPVlanEndpoint { + pub(crate) net_pair: NetworkPair, +} + +impl IPVlanEndpoint { + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + idx: u32, + queues: usize, + ) -> Result { + // tc filter network model is the only one works for ipvlan + let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues) + .await + .context("error creating new NetworkPair")?; + Ok(IPVlanEndpoint { net_pair }) + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + Ok(NetworkConfig { + id: self.net_pair.virt_iface.name.clone(), + host_dev_name: iface.name.clone(), + guest_mac: Some(guest_mac), + }) + } +} + +#[async_trait] +impl Endpoint for IPVlanEndpoint { + async fn name(&self) -> String { + self.net_pair.virt_iface.name.clone() + } + + async fn hardware_addr(&self) -> String { + self.net_pair.tap.tap_iface.hard_addr.clone() + } + + async fn attach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .add_network_model() + .await + .context("error adding network model")?; + let config = self.get_network_config().context("get network config")?; + h.add_device(Device::Network(config)) + .await + .context("error adding device by hypervisor")?; + + Ok(()) + } + + async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .del_network_model() + .await + .context("error deleting network model")?; + let config = self + .get_network_config() + .context("error getting network config")?; + h.remove_device(Device::Network(config)) + .await + .context("error removing device by hypervisor")?; + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs index caaa69dca611..c55182f54ff7 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs @@ -8,6 +8,8 @@ mod physical_endpoint; pub use physical_endpoint::PhysicalEndpoint; mod veth_endpoint; pub use veth_endpoint::VethEndpoint; +mod ipvlan_endpoint; +pub use ipvlan_endpoint::IPVlanEndpoint; use anyhow::Result; use async_trait::async_trait; diff --git a/src/runtime-rs/crates/resource/src/network/network_model/mod.rs b/src/runtime-rs/crates/resource/src/network/network_model/mod.rs index 848e032c0a1a..16457f1e4f9c 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/mod.rs @@ -15,8 +15,8 @@ use async_trait::async_trait; use super::NetworkPair; -const TC_FILTER_NET_MODEL_STR: &str = "tcfilter"; -const ROUTE_NET_MODEL_STR: &str = "route"; +pub(crate) const TC_FILTER_NET_MODEL_STR: &str = "tcfilter"; +pub(crate) const ROUTE_NET_MODEL_STR: &str = "route"; pub enum NetworkModelType { NoneModel, diff --git a/src/runtime-rs/crates/resource/src/network/network_pair.rs b/src/runtime-rs/crates/resource/src/network/network_pair.rs index 71e212907eb6..c96898619b39 100644 --- a/src/runtime-rs/crates/resource/src/network/network_pair.rs +++ b/src/runtime-rs/crates/resource/src/network/network_pair.rs @@ -54,6 +54,7 @@ impl NetworkPair { let tap_link = create_link(handle, &tap_iface_name, queues) .await .context("create link")?; + let virt_link = get_link_by_name(handle, virt_iface_name.clone().as_str()) .await .context("get link by name")?; diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index c8c76b6e25f7..1d2a1e55f960 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -17,7 +17,7 @@ use scopeguard::defer; use tokio::sync::RwLock; use super::{ - endpoint::{Endpoint, PhysicalEndpoint, VethEndpoint}, + endpoint::{Endpoint, IPVlanEndpoint, PhysicalEndpoint, VethEndpoint}, network_entity::NetworkEntity, network_info::network_info_from_link::NetworkInfoFromLink, utils::{link, netns}, @@ -186,6 +186,12 @@ async fn create_endpoint( .context("veth endpoint")?; Arc::new(ret) } + "ipvlan" => { + let ret = IPVlanEndpoint::new(handle, &attrs.name, idx, config.queues) + .await + .context("ipvlan endpoint")?; + Arc::new(ret) + } _ => return Err(anyhow!("unsupported link type: {}", link_type)), } }; From 19eca71cd92516578c4f1ca51004295814676fa1 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 18 Jul 2022 13:24:22 +0800 Subject: [PATCH 0131/1953] runtime-rs: remove the value of hypervisor path in DB config As a built in VMM, Path, jailer path, ctlpath are not needed for Dragonball. So we don't generate those value in Makefile. Fixes: #4677 Signed-off-by: Zhongtao Hu --- src/runtime-rs/Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 462f1f188ff6..32c2214a04a6 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -33,7 +33,6 @@ ifeq ($(PREFIX),) endif PREFIXDEPS := $(PREFIX) -DBBINDIR := $(PREFIXDEPS)/bin LIBEXECDIR := $(PREFIXDEPS)/libexec SHAREDIR := $(PREFIX)/share DEFAULTSDIR := $(SHAREDIR)/defaults @@ -49,9 +48,7 @@ DEFAULT_HYPERVISOR ?= $(HYPERVISOR_DB) HYPERVISORS := $(HYPERVISOR_DB) -DBPATH := $(DBBINDIR)/$(QEMUCMD) -DBVALIDHYPERVISORPATHS := [\"$(DBPATH)\"] - +DBVALIDHYPERVISORPATHS := [] PKGDATADIR := $(PREFIXDEPS)/share/$(PROJECT_DIR) KERNELDIR := $(PKGDATADIR) IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME) From 545ae3f0ee93389616837000b5179b245d02afa3 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 18 Jul 2022 09:34:06 +0800 Subject: [PATCH 0132/1953] runtime-rs: fix warning Module anyhow::anyhow is only used on x86_64 architecture in crates/hypervisor/src/device/vfio.rs file. Fixes: #4676 Signed-off-by: xuejun-xj --- src/runtime-rs/crates/hypervisor/src/device/vfio.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs b/src/runtime-rs/crates/hypervisor/src/device/vfio.rs index 5e62d4549c9a..fcbaeb19fe99 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/vfio.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/vfio.rs @@ -6,7 +6,9 @@ use std::{fs, path::Path, process::Command}; -use anyhow::{anyhow, Context, Result}; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +use anyhow::anyhow; +use anyhow::{Context, Result}; fn override_driver(bdf: &str, driver: &str) -> Result<()> { let driver_override = format!("/sys/bus/pci/devices/{}/driver_override", bdf); From f4c3adf5963f70143c93748a4c0c86f9e8590ff1 Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 18 Jul 2022 09:37:06 +0800 Subject: [PATCH 0133/1953] runtime-rs: Add compile option file Add file aarch64-options.mk for compiling on aarch64 architectures. Fixes: #4676 Signed-off-by: xuejun-xj --- src/runtime-rs/arch/aarch64-options.mk | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/runtime-rs/arch/aarch64-options.mk diff --git a/src/runtime-rs/arch/aarch64-options.mk b/src/runtime-rs/arch/aarch64-options.mk new file mode 100644 index 000000000000..2e9e5759b73a --- /dev/null +++ b/src/runtime-rs/arch/aarch64-options.mk @@ -0,0 +1,15 @@ +# Copyright (c) 2019-2022 Alibaba Cloud +# Copyright (c) 2019-2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +MACHINETYPE := +KERNELPARAMS := +MACHINEACCELERATORS := +CPUFEATURES := pmu=off + +QEMUCMD := qemu-system-aarch64 + +# dragonball binary name +DBCMD := dragonball From 99654ce69492bf9147c188c90fcda1f62f808cbf Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Mon, 18 Jul 2022 09:41:53 +0800 Subject: [PATCH 0134/1953] runtime-rs: update dbs-xxx dependencies Update dbs-xxx commit ID for aarch64 in runtime-rs/Cargo.toml file to add dependencies for aarch64. Fixes: #4676 Signed-off-by: xuejun-xj --- src/runtime-rs/Cargo.lock | 18 ++++++++---------- src/runtime-rs/Cargo.toml | 14 ++++++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 606170aa3705..f977ef4e186c 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -455,8 +455,7 @@ dependencies = [ [[package]] name = "dbs-arch" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d235999408e59e60d18461debfb31d504813cfa5e497ff9d45c1c319980cf74a" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -469,8 +468,7 @@ dependencies = [ [[package]] name = "dbs-boot" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37036c65dc89724ff5628cee6c48ebe75027f989398317b2a5155924ba9c2bf7" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "dbs-arch", "kvm-bindings", @@ -485,7 +483,7 @@ dependencies = [ [[package]] name = "dbs-device" version = "0.1.0" -source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "thiserror", ] @@ -493,7 +491,7 @@ dependencies = [ [[package]] name = "dbs-interrupt" version = "0.1.0" -source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "dbs-device", "kvm-bindings", @@ -505,7 +503,7 @@ dependencies = [ [[package]] name = "dbs-legacy-devices" version = "0.1.0" -source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "dbs-device", "dbs-utils", @@ -528,7 +526,7 @@ dependencies = [ [[package]] name = "dbs-utils" version = "0.1.0" -source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "anyhow", "event-manager", @@ -543,7 +541,7 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" version = "0.1.0" -source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" dependencies = [ "blobfs", "byteorder", @@ -3060,4 +3058,4 @@ dependencies = [ [[patch.unused]] name = "dbs-upcall" version = "0.1.0" -source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=84eee5737cc7d85f9921c94a93e6b9dc4ae24a39#84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" +source = "git+https://github.com/openanolis/dragonball-sandbox.git?rev=7a8e832b53d66994d6a16f0513d69f540583dcd0#7a8e832b53d66994d6a16f0513d69f540583dcd0" diff --git a/src/runtime-rs/Cargo.toml b/src/runtime-rs/Cargo.toml index 70b384e43185..470b29a64d23 100644 --- a/src/runtime-rs/Cargo.toml +++ b/src/runtime-rs/Cargo.toml @@ -4,9 +4,11 @@ members = [ ] [patch.'crates-io'] -dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } -dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "84eee5737cc7d85f9921c94a93e6b9dc4ae24a39" } +dbs-device = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-utils = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-interrupt = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-legacy-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-virtio-devices = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-boot = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-arch = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } +dbs-upcall = { git = "https://github.com/openanolis/dragonball-sandbox.git", rev = "7a8e832b53d66994d6a16f0513d69f540583dcd0" } From 62182db6451764820aa06e5529da2634b2d59748 Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Fri, 15 Jul 2022 16:41:50 +0800 Subject: [PATCH 0135/1953] runtime-rs: add unit test for ipvlan endpoint Add unit test to check the integrity of IPVlanEndpoint::new(...) Fixes: #4655 Signed-off-by: Ji-Xinyou --- .../src/network/endpoint/endpoints_test.rs | 126 ++++++++++++++++++ .../resource/src/network/endpoint/mod.rs | 1 + src/runtime/virtcontainers/ipvlan_endpoint.go | 1 - 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs b/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs new file mode 100644 index 000000000000..aeb7e0e3e94d --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs @@ -0,0 +1,126 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +#[cfg(test)] +mod tests { + use anyhow::Context; + use scopeguard::defer; + + use std::sync::Arc; + + use crate::network::{ + endpoint::IPVlanEndpoint, + network_model::{ + tc_filter_model::{fetch_index, TcFilterModel}, + NetworkModelType, + }, + network_pair::{NetworkInterface, NetworkPair, TapInterface}, + }; + + // this unit test tests the integrity of IPVlanEndpoint::new() + // by comparing the manual constructed object with object constructed by new() + #[actix_rt::test] + async fn test_ipvlan_construction() { + let idx = 8192; + let mac_addr = String::from("02:00:CA:FE:00:04"); + let manual_virt_iface_name = format!("eth{}", idx); + let tap_iface_name = format!("tap{}_kata", idx); // create by kata + + if let Ok((conn, handle, _)) = + rtnetlink::new_connection().context("failed to create new netlink connection") + { + let thread_handler = tokio::spawn(conn); + defer!({ + thread_handler.abort(); + }); + + // since IPVlanEndpoint::new() needs an EXISTING virt_iface (which is created + // by containerd normally), we have to manually create a virt_iface. + if let Ok(()) = handle + .link() + .add() + .veth("foo".to_string(), manual_virt_iface_name.clone()) + .execute() + .await + .context("failed to create virt_iface") + { + if let Ok(mut result) = IPVlanEndpoint::new(&handle, "", idx, 5) + .await + .context("failed to create new IPVlan Endpoint") + { + let manual = IPVlanEndpoint { + net_pair: NetworkPair { + tap: TapInterface { + id: String::from("uniqueTestID_kata"), + name: format!("br{}_kata", idx), + tap_iface: NetworkInterface { + name: tap_iface_name.clone(), + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: manual_virt_iface_name.clone(), + hard_addr: mac_addr.clone(), + ..Default::default() + }, + model: Arc::new(TcFilterModel::new().unwrap()), // impossible to panic + network_qos: false, + }, + }; + + result.net_pair.tap.id = String::from("uniqueTestID_kata"); + result.net_pair.tap.tap_iface.hard_addr = String::from(""); + result.net_pair.virt_iface.hard_addr = mac_addr.clone(); + + // check the integrity by compare all variables + assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id); + assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name); + assert_eq!( + manual.net_pair.tap.tap_iface.name, + result.net_pair.tap.tap_iface.name + ); + assert_eq!( + manual.net_pair.tap.tap_iface.hard_addr, + result.net_pair.tap.tap_iface.hard_addr + ); + assert_eq!( + manual.net_pair.tap.tap_iface.addrs, + result.net_pair.tap.tap_iface.addrs + ); + assert_eq!( + manual.net_pair.virt_iface.name, + result.net_pair.virt_iface.name + ); + assert_eq!( + manual.net_pair.virt_iface.hard_addr, + result.net_pair.virt_iface.hard_addr + ); + assert_eq!( + manual.net_pair.virt_iface.addrs, + result.net_pair.virt_iface.addrs + ); + // using match branch to avoid deriving PartialEq trait + match manual.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} // ok + _ => unreachable!(), + } + match result.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} + _ => unreachable!(), + } + assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); + } + if let Ok(link_index) = fetch_index(&handle, manual_virt_iface_name.as_str()).await + { + assert!(handle.link().del(link_index).execute().await.is_ok()) + } + if let Ok(link_index) = fetch_index(&handle, tap_iface_name.as_str()).await { + assert!(handle.link().del(link_index).execute().await.is_ok()) + } + } + } + } +} diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs index c55182f54ff7..b8e4557b53d1 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs @@ -10,6 +10,7 @@ mod veth_endpoint; pub use veth_endpoint::VethEndpoint; mod ipvlan_endpoint; pub use ipvlan_endpoint::IPVlanEndpoint; +mod endpoints_test; use anyhow::Result; use async_trait::async_trait; diff --git a/src/runtime/virtcontainers/ipvlan_endpoint.go b/src/runtime/virtcontainers/ipvlan_endpoint.go index a257e6ecb4bd..53a57e740d7d 100644 --- a/src/runtime/virtcontainers/ipvlan_endpoint.go +++ b/src/runtime/virtcontainers/ipvlan_endpoint.go @@ -96,7 +96,6 @@ func (endpoint *IPVlanEndpoint) NetworkPair() *NetworkInterfacePair { // Attach for ipvlan endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. -// tap interface of the network pair to the Hypervisor. func (endpoint *IPVlanEndpoint) Attach(ctx context.Context, s *Sandbox) error { span, ctx := ipvlanTrace(ctx, "Attach", endpoint) defer span.End() From cebbebbe8a0ec886d70e81db19caa996a7c2e8fd Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Mon, 18 Jul 2022 21:44:18 +0800 Subject: [PATCH 0136/1953] runtime-rs: fix ctr exit failed During use, there will be cases where the container is in the stop state and get another stop. In this case, the second stop needs to be ignored. Fixes: #4683 Signed-off-by: Quanwei Zhou --- .../virt_container/src/container_manager/container_inner.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs index f9ff08ebfaf2..6cfaef7ff3b6 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container_inner.rs @@ -193,7 +193,13 @@ impl ContainerInner { ) -> Result<()> { let logger = logger_with_process(process); info!(logger, "begin to stop process"); + // do not stop again when state stopped, may cause multi cleanup resource + let state = self.init_process.get_status().await; + if state == ProcessStatus::Stopped { + return Ok(()); + } + self.check_state(vec![ProcessStatus::Running]) .await .context("check state")?; From e9988f0c6817fcf4120044343892a7b74dee3e8f Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Tue, 19 Jul 2022 08:19:24 +0800 Subject: [PATCH 0137/1953] runtime-rs: fix sandbox_cgroup_only=false panic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When run with configuration `sandbox_cgroup_only=false`, we will call `gen_overhead_path()` as the overhead path. The `cgroup-rs` will push the path with the subsystem prefix by `PathBuf::push()`. When the path has prefix “/” it will act as root path, such as ``` let mut path = PathBuf::from("/tmp"); path.push("/etc"); assert_eq!(path, PathBuf::from("/etc")); ``` So we shoud not set overhead path with prefix "/". Fixes: #4687 Signed-off-by: Quanwei Zhou --- src/runtime-rs/crates/resource/src/cgroups/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/resource/src/cgroups/utils.rs b/src/runtime-rs/crates/resource/src/cgroups/utils.rs index a81316358cb3..7a2d630982c0 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/utils.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/utils.rs @@ -12,5 +12,5 @@ // /sys/fs/cgroup/memory/kata_overhead/$CGPATH where $CGPATH is // defined by the orchestrator. pub(crate) fn gen_overhead_path(path: &str) -> String { - format!("/kata_overhead/{}", path.trim_start_matches('/')) + format!("kata_overhead/{}", path.trim_start_matches('/')) } From 4331ef80d0b7e3f9d5e4d324a40e9678f941953b Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Wed, 13 Jul 2022 14:36:09 +0800 Subject: [PATCH 0138/1953] Runtime-rs: add installation guide for rust-runtime add installation guide for rust-runtime Fixes:#4661 Signed-off-by: Zhongtao Hu --- docs/install/README.md | 3 + ...ers-3.0-rust-runtime-installation-guide.md | 69 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 docs/install/kata-containers-3.0-rust-runtime-installation-guide.md diff --git a/docs/install/README.md b/docs/install/README.md index 9ad55f0f2174..0ed42d87f5fc 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -79,3 +79,6 @@ versions. This is not recommended for normal users. * [upgrading document](../Upgrading.md) * [developer guide](../Developer-Guide.md) * [runtime documentation](../../src/runtime/README.md) + +## Kata Containers 3.0 rust runtime installation +* [installation guide](../install/kata-containers-3.0-rust-runtime-installation-guide.md) diff --git a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md new file mode 100644 index 000000000000..8e50901ba9ff --- /dev/null +++ b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md @@ -0,0 +1,69 @@ +# Kata Containers 3.0 rust runtime installation +The following is an overview of the different installation methods available. + +## Prerequisites + +Kata Containers 3.0 rust runtime requires nested virtualization or bare metal. Check +[hardware requirements](/src/runtime/README.md#hardware-requirements) to see if your system is capable of running Kata +Containers. + +### Platform support + +Kata Containers 3.0 rust runtime currently runs on 64-bit systems supporting the following +architectures: + +> **Notes:** +> For other architectures, see https://github.com/kata-containers/kata-containers/issues/4320 + +| Architecture | Virtualization technology | +|-|-| +| `x86_64`| [Intel](https://www.intel.com) VT-x | +| `aarch64` ("`arm64`")| [ARM](https://www.arm.com) Hyp | + +## Packaged installation methods + +| Installation method | Description | Automatic updates | Use case | Availability +|------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- | +| [Using kata-deploy](#kata-deploy-installation) | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!** | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | No | +| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | No | +| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | No | +| [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | No | +| [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | No | +| [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | Yes | + +### Kata Deploy Installation +`ToDo` +### Official packages +`ToDo` +### Snap Installation +`ToDo` +### Automatic Installation +`ToDo` +### Manual Installation +`ToDo` + +## Build from source installation + +### Install Kata 3.0 Rust Runtime Shim + +``` +$ git clone https://github.com/kata-containers/kata-containers.git +$ cd kata-containers/src/runtime-rs +$ make && make install +``` +After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`, the binary file `containerd-shim-kata-v2` will be installed under `/user/local/bin` . + +### Build Kata Containers Kernel +Follow the [Kernel installation guide](/tools/packaging/kernel/README.md). + +### Build Kata Rootfs +Follow the [Rootfs installation guide](../../tools/osbuilder/rootfs-builder/README.md). + +### Build Kata Image +Follow the [Image installation guide](../../tools/osbuilder/image-builder/README.md). + +### Install Containerd + +Follow the [Containerd installation guide](container-manager/containerd/containerd-install.md). + + From 54f53d57ef78371ca1c040afa6169a04c10f6c3a Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Tue, 19 Jul 2022 13:11:47 +0800 Subject: [PATCH 0139/1953] runtime-rs: support disable_guest_seccomp support disable_guest_seccomp Fixes: #4691 Signed-off-by: Quanwei Zhou --- src/runtime-rs/crates/resource/src/manager.rs | 7 +++- .../crates/resource/src/manager_inner.rs | 11 ++++-- .../runtimes/common/src/runtime_handler.rs | 2 +- .../crates/runtimes/common/src/sandbox.rs | 4 +-- .../runtimes/linux_container/src/lib.rs | 2 +- src/runtime-rs/crates/runtimes/src/manager.rs | 6 ++-- .../src/container_manager/container.rs | 35 +++++++++++++++++-- .../crates/runtimes/virt_container/src/lib.rs | 4 +-- .../runtimes/virt_container/src/sandbox.rs | 7 ++-- .../crates/runtimes/wasm_container/src/lib.rs | 2 +- 10 files changed, 59 insertions(+), 21 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/manager.rs b/src/runtime-rs/crates/resource/src/manager.rs index f5db93a175cd..94cf3138f062 100644 --- a/src/runtime-rs/crates/resource/src/manager.rs +++ b/src/runtime-rs/crates/resource/src/manager.rs @@ -25,7 +25,7 @@ impl ResourceManager { sid: &str, agent: Arc, hypervisor: Arc, - toml_config: &TomlConfig, + toml_config: Arc, ) -> Result { Ok(Self { inner: Arc::new(RwLock::new(ResourceManagerInner::new( @@ -37,6 +37,11 @@ impl ResourceManager { }) } + pub async fn config(&self) -> Arc { + let inner = self.inner.read().await; + inner.config() + } + pub async fn prepare_before_start_vm(&self, device_configs: Vec) -> Result<()> { let mut inner = self.inner.write().await; inner.prepare_before_start_vm(device_configs).await diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index c254352429c3..fb90a25a60ac 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -24,6 +24,7 @@ use crate::{ pub(crate) struct ResourceManagerInner { sid: String, + toml_config: Arc, agent: Arc, hypervisor: Arc, network: Option>, @@ -39,20 +40,26 @@ impl ResourceManagerInner { sid: &str, agent: Arc, hypervisor: Arc, - toml_config: &TomlConfig, + toml_config: Arc, ) -> Result { + let cgroups_resource = CgroupsResource::new(sid, &toml_config)?; Ok(Self { sid: sid.to_string(), + toml_config, agent, hypervisor, network: None, share_fs: None, rootfs_resource: RootFsResource::new(), volume_resource: VolumeResource::new(), - cgroups_resource: CgroupsResource::new(sid, toml_config)?, + cgroups_resource, }) } + pub fn config(&self) -> Arc { + self.toml_config.clone() + } + pub async fn prepare_before_start_vm( &mut self, device_configs: Vec, diff --git a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs index bf137f689f63..c12df38b1213 100644 --- a/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs +++ b/src/runtime-rs/crates/runtimes/common/src/runtime_handler.rs @@ -36,7 +36,7 @@ pub trait RuntimeHandler: Send + Sync { &self, sid: &str, msg_sender: Sender, - config: &TomlConfig, + config: Arc, ) -> Result; fn cleanup(&self, id: &str) -> Result<()>; diff --git a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs index 1b175204c5bb..fbb5db53bc8a 100644 --- a/src/runtime-rs/crates/runtimes/common/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/common/src/sandbox.rs @@ -7,11 +7,9 @@ use anyhow::Result; use async_trait::async_trait; -use kata_types::config::TomlConfig; - #[async_trait] pub trait Sandbox: Send + Sync { - async fn start(&self, netns: Option, config: &TomlConfig) -> Result<()>; + async fn start(&self, netns: Option) -> Result<()>; async fn stop(&self) -> Result<()>; async fn cleanup(&self, container_id: &str) -> Result<()>; async fn shutdown(&self) -> Result<()>; diff --git a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs index 4a805e3fadf3..582b4e961f0f 100644 --- a/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/linux_container/src/lib.rs @@ -33,7 +33,7 @@ impl RuntimeHandler for LinuxContainer { &self, _sid: &str, _msg_sender: Sender, - _config: &TomlConfig, + _config: Arc, ) -> Result { todo!() } diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 131a2761250e..10a4a427bb97 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -40,7 +40,7 @@ impl RuntimeHandlerManagerInner { async fn init_runtime_handler( &mut self, netns: Option, - config: &TomlConfig, + config: Arc, ) -> Result<()> { info!(sl!(), "new runtime handler {}", &config.runtime.name); let runtime_handler = match config.runtime.name.as_str() { @@ -62,7 +62,7 @@ impl RuntimeHandlerManagerInner { // start sandbox runtime_instance .sandbox - .start(netns, config) + .start(netns) .await .context("start sandbox")?; self.runtime_instance = Some(Arc::new(runtime_instance)); @@ -100,7 +100,7 @@ impl RuntimeHandlerManagerInner { }; let config = load_config(spec).context("load config")?; - self.init_runtime_handler(netns, &config) + self.init_runtime_handler(netns, Arc::new(config)) .await .context("init runtime handler")?; diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index d31ee4224826..2e50b55e924d 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -78,8 +78,9 @@ impl Container { pub async fn create(&self, mut spec: oci::Spec) -> Result<()> { // process oci spec let mut inner = self.inner.write().await; + let toml_config = self.resource_manager.config().await; let config = &self.config; - amend_spec(&mut spec).context("load spec")?; + amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp).context("load spec")?; // handler rootfs let rootfs = self @@ -372,12 +373,14 @@ impl Container { } } -fn amend_spec(spec: &mut oci::Spec) -> Result<()> { +fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> { // hook should be done on host spec.hooks = None; if let Some(linux) = spec.linux.as_mut() { - linux.seccomp = None; + if disable_guest_seccomp { + linux.seccomp = None; + } if let Some(resource) = linux.resources.as_mut() { resource.devices = Vec::new(); @@ -400,3 +403,29 @@ fn amend_spec(spec: &mut oci::Spec) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::amend_spec; + + #[test] + fn test_amend_spec_disable_guest_seccomp() { + let mut spec = oci::Spec { + linux: Some(oci::Linux { + seccomp: Some(oci::LinuxSeccomp::default()), + ..Default::default() + }), + ..Default::default() + }; + + assert!(spec.linux.as_ref().unwrap().seccomp.is_some()); + + // disable_guest_seccomp = false + amend_spec(&mut spec, false).unwrap(); + assert!(spec.linux.as_ref().unwrap().seccomp.is_some()); + + // disable_guest_seccomp = true + amend_spec(&mut spec, true).unwrap(); + assert!(spec.linux.as_ref().unwrap().seccomp.is_none()); + } +} diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs index 737b1a18f437..8869c03c076a 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -51,9 +51,9 @@ impl RuntimeHandler for VirtContainer { &self, sid: &str, msg_sender: Sender, - config: &TomlConfig, + config: Arc, ) -> Result { - let hypervisor = new_hypervisor(config).await.context("new hypervisor")?; + let hypervisor = new_hypervisor(&config).await.context("new hypervisor")?; // get uds from hypervisor and get config from toml_config let agent = Arc::new(KataAgent::new(kata_types::config::Agent { diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 470663bcb5db..6bcf3d2dfc2f 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -15,7 +15,6 @@ use common::{ }; use containerd_shim_protos::events::task::TaskOOM; use hypervisor::Hypervisor; -use kata_types::config::TomlConfig; use resource::{ network::{NetworkConfig, NetworkWithNetNsConfig}, ResourceConfig, ResourceManager, @@ -79,10 +78,10 @@ impl VirtSandbox { &self, _id: &str, netns: Option, - config: &TomlConfig, ) -> Result> { let mut resource_configs = vec![]; + let config = self.resource_manager.config().await; if let Some(netns_path) = netns { let network_config = ResourceConfig::Network(NetworkConfig::NetworkResourceWithNetNs( NetworkWithNetNsConfig { @@ -109,7 +108,7 @@ impl VirtSandbox { #[async_trait] impl Sandbox for VirtSandbox { - async fn start(&self, netns: Option, config: &TomlConfig) -> Result<()> { + async fn start(&self, netns: Option) -> Result<()> { let id = &self.sid; // if sandbox running, return @@ -127,7 +126,7 @@ impl Sandbox for VirtSandbox { // generate device and setup before start vm // should after hypervisor.prepare_vm - let resources = self.prepare_for_start_sandbox(id, netns, config).await?; + let resources = self.prepare_for_start_sandbox(id, netns).await?; self.resource_manager .prepare_before_start_vm(resources) .await diff --git a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs index 28a81fc49df1..c6872746707a 100644 --- a/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/wasm_container/src/lib.rs @@ -32,7 +32,7 @@ impl RuntimeHandler for WasmContainer { &self, _sid: &str, _msg_sender: Sender, - _config: &TomlConfig, + _config: Arc, ) -> Result { todo!() } From 402bfa0ce35d57c578171ac16f1f8d29a8dc237f Mon Sep 17 00:00:00 2001 From: liubin Date: Wed, 20 Jul 2022 12:11:59 +0800 Subject: [PATCH 0140/1953] nydus: upgrade nydus/nydus-snapshotter version Upgrade nydus/nydus-snapshotter to the latest version. Fixes: #4694 Signed-off-by: liubin --- versions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yaml b/versions.yaml index b8ffaca1291d..f1bb26448fc2 100644 --- a/versions.yaml +++ b/versions.yaml @@ -241,12 +241,12 @@ externals: nydus: description: "Nydus image acceleration service" url: "https://github.com/dragonflyoss/image-service" - version: "v1.1.2" + version: "v2.1.0-alpha.4" nydus-snapshotter: description: "Snapshotter for Nydus image acceleration service" url: "https://github.com/containerd/nydus-snapshotter" - version: "v0.1.0" + version: "v0.2.3" virtiofsd: description: "vhost-user virtio-fs device backend written in Rust" From eec9ac81efc5482f50155d2a10f31d9de6632c96 Mon Sep 17 00:00:00 2001 From: liubin Date: Wed, 20 Jul 2022 14:26:06 +0800 Subject: [PATCH 0141/1953] rustjail: check result to let it return early. check the result to let it return early if there are some errors Fixes: #4708 Signed-off-by: liubin --- src/agent/rustjail/src/mount.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 74742c0ffe47..c36f84071487 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -786,12 +786,25 @@ fn mount_from( "create dir {}: {}", dir.to_str().unwrap(), e.to_string() - ) - }); + ); + e + })?; // make sure file exists so we can bind over it if !src.is_dir() { - let _ = OpenOptions::new().create(true).write(true).open(&dest); + let _ = OpenOptions::new() + .create(true) + .write(true) + .open(&dest) + .map_err(|e| { + log_child!( + cfd_log, + "open/create dest error. {}: {:?}", + dest.as_str(), + e + ); + e + })?; } src.to_str().unwrap().to_string() } else { @@ -804,8 +817,10 @@ fn mount_from( } }; - let _ = stat::stat(dest.as_str()) - .map_err(|e| log_child!(cfd_log, "dest stat error. {}: {:?}", dest.as_str(), e)); + let _ = stat::stat(dest.as_str()).map_err(|e| { + log_child!(cfd_log, "dest stat error. {}: {:?}", dest.as_str(), e); + e + })?; mount( Some(src.as_str()), From 0d7cb7eb1640b92827342e7a149e8208180c5618 Mon Sep 17 00:00:00 2001 From: liubin Date: Thu, 21 Jul 2022 14:53:01 +0800 Subject: [PATCH 0142/1953] agent: delete agent-type property in announce Since there is only one type of agent now, the agent-type is not needed anymore. Signed-off-by: liubin --- src/agent/src/main.rs | 4 ---- src/agent/src/netlink.rs | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 6a023e093e70..2ebd7a3fbdac 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -110,10 +110,6 @@ enum SubCommand { fn announce(logger: &Logger, config: &AgentConfig) { info!(logger, "announce"; "agent-commit" => version::VERSION_COMMIT, - - // Avoid any possibility of confusion with the old agent - "agent-type" => "rust", - "agent-version" => version::AGENT_VERSION, "api-version" => version::API_VERSION, "config" => format!("{:?}", config), diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index 1de4ef6920d6..c6fc9c2079c7 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -64,7 +64,7 @@ impl Handle { pub async fn update_interface(&mut self, iface: &Interface) -> Result<()> { // The reliable way to find link is using hardware address // as filter. However, hardware filter might not be supported - // by netlink, we may have to dump link list and the find the + // by netlink, we may have to dump link list and then find the // target link. filter using name or family is supported, but // we cannot use that to find target link. // let's try if hardware address filter works. -_- @@ -178,7 +178,7 @@ impl Handle { .with_context(|| format!("Failed to parse MAC address: {}", addr))?; // Hardware filter might not be supported by netlink, - // we may have to dump link list and the find the target link. + // we may have to dump link list and then find the target link. stream .try_filter(|f| { let result = f.nlas.iter().any(|n| match n { From 43045be8d192fbd5a09fc7f92e41386160eb6fab Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Thu, 21 Jul 2022 16:27:26 +0800 Subject: [PATCH 0143/1953] runtime-rs: handle default_vcpus greator than default_maxvcpu when the default_vcpus is greater than the default_maxvcpus, the default vcpu number should be set equal to the default_maxvcpus. Fixes: #4712 Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/config/hypervisor/dragonball.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libs/kata-types/src/config/hypervisor/dragonball.rs b/src/libs/kata-types/src/config/hypervisor/dragonball.rs index 7a8c4c894304..bb72944a2e68 100644 --- a/src/libs/kata-types/src/config/hypervisor/dragonball.rs +++ b/src/libs/kata-types/src/config/hypervisor/dragonball.rs @@ -65,6 +65,10 @@ impl ConfigPlugin for DragonballConfig { db.cpu_info.default_maxvcpus = default::MAX_DRAGONBALL_VCPUS; } + if db.cpu_info.default_vcpus as u32 > db.cpu_info.default_maxvcpus { + db.cpu_info.default_vcpus = db.cpu_info.default_maxvcpus as i32; + } + if db.machine_info.entropy_source.is_empty() { db.machine_info.entropy_source = default::DEFAULT_DRAGONBALL_ENTROPY_SOURCE.to_string(); From 912641509eba7022dd311e34ab08b980c67d2254 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Thu, 21 Jul 2022 19:37:15 +0800 Subject: [PATCH 0144/1953] agent: fix fd-double-close problem in ut test_do_write_stream The fd will closed on struct Process's dropping, so don't close it again manually. Fixes: #4598 Signed-off-by: Tim Zhang --- src/agent/src/rpc.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 3004be5c0e0c..bef693fe23c7 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -2262,6 +2262,7 @@ mod tests { if d.has_fd { Some(wfd) } else { + unistd::close(wfd).unwrap(); None } }; @@ -2296,13 +2297,14 @@ mod tests { if !d.break_pipe { unistd::close(rfd).unwrap(); } - unistd::close(wfd).unwrap(); + // XXX: Do not close wfd. + // the fd will be closed on Process's dropping. + // unistd::close(wfd).unwrap(); let msg = format!("{}, result: {:?}", msg, result); assert_result!(d.result, result, msg); } } - #[tokio::test] async fn test_update_container_namespaces() { #[derive(Debug)] From 896478c92b2081d15b2fd0a6abc3b31142d4a5d1 Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Thu, 21 Jul 2022 10:12:14 +0800 Subject: [PATCH 0145/1953] runtime-rs: add functionalities support for macvlan and vlan endpoints Add macvlan and vlan support to runtime-rs code and corresponding unit tests. Fixes: #4701 Signed-off-by: Ji-Xinyou --- src/libs/Cargo.lock | 20 +- .../src/network/endpoint/endpoints_test.rs | 277 ++++++++++++++++-- .../src/network/endpoint/macvlan_endpoint.rs | 84 ++++++ .../resource/src/network/endpoint/mod.rs | 4 + .../src/network/endpoint/vlan_endpoint.rs | 88 ++++++ .../src/network/network_with_netns.rs | 22 +- .../src/network/utils/link/manager.rs | 28 ++ 7 files changed, 501 insertions(+), 22 deletions(-) create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs create mode 100644 src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index e5d9cb160e12..b82c108c4478 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -94,7 +94,7 @@ checksum = "1b827f9d9f6c2fff719d25f5d44cbc8d2ef6df1ef00d055c5c14d5dc25529579" dependencies = [ "libc", "log", - "nix", + "nix 0.23.1", "regex", ] @@ -376,7 +376,7 @@ dependencies = [ "kata-types", "lazy_static", "libc", - "nix", + "nix 0.24.2", "num_cpus", "oci", "once_cell", @@ -508,6 +508,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "memoffset", +] + [[package]] name = "ntapi" version = "0.3.7" @@ -1108,7 +1120,7 @@ dependencies = [ "futures", "libc", "log", - "nix", + "nix 0.23.1", "protobuf", "protobuf-codegen-pure", "thiserror", @@ -1162,7 +1174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e32675ee2b3ce5df274c0ab52d19b28789632406277ca26bffee79a8e27dc133" dependencies = [ "libc", - "nix", + "nix 0.23.1", ] [[package]] diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs b/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs index aeb7e0e3e94d..90623d59cbcf 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/endpoints_test.rs @@ -7,21 +7,267 @@ #[cfg(test)] mod tests { use anyhow::Context; + use netlink_packet_route::MACVLAN_MODE_PRIVATE; use scopeguard::defer; use std::sync::Arc; use crate::network::{ - endpoint::IPVlanEndpoint, + endpoint::{IPVlanEndpoint, MacVlanEndpoint, VlanEndpoint}, network_model::{ + self, tc_filter_model::{fetch_index, TcFilterModel}, - NetworkModelType, + NetworkModelType, TC_FILTER_NET_MODEL_STR, }, network_pair::{NetworkInterface, NetworkPair, TapInterface}, }; + // this unit test tests the integrity of MacVlanEndpoint::new() + #[actix_rt::test] + async fn test_vlan_construction() { + let idx = 8193; + let mac_addr = String::from("02:78:CA:FE:00:04"); + let manual_vlan_iface_name = format!("eth{}", idx); + let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new() + let dummy_name = format!("dummy{}", idx); + let vlanid = 123; + + if let Ok((conn, handle, _)) = + rtnetlink::new_connection().context("failed to create netlink connection") + { + let thread_handler = tokio::spawn(conn); + defer!({ + thread_handler.abort(); + }); + + if let Ok(()) = handle + .link() + .add() + .dummy(dummy_name.clone()) + .execute() + .await + .context("failed to create dummy link") + { + let dummy_index = fetch_index(&handle, dummy_name.clone().as_str()) + .await + .expect("failed to get the index of dummy link"); + + // since IPVlanEndpoint::new() needs an EXISTING virt_iface (which is created + // by containerd normally), we have to manually create a virt_iface. + if let Ok(()) = handle + .link() + .add() + .vlan(manual_vlan_iface_name.clone(), dummy_index, vlanid) + .execute() + .await + .context("failed to create manual veth pair") + { + if let Ok(mut result) = VlanEndpoint::new(&handle, "", idx, 5) + .await + .context("failed to create new ipvlan endpoint") + { + let manual = VlanEndpoint { + net_pair: NetworkPair { + tap: TapInterface { + id: String::from("uniqueTestID_kata"), + name: format!("br{}_kata", idx), + tap_iface: NetworkInterface { + name: tap_iface_name.clone(), + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: manual_vlan_iface_name.clone(), + hard_addr: mac_addr.clone(), + ..Default::default() + }, + model: Arc::new(TcFilterModel::new().unwrap()), // impossible to panic + network_qos: false, + }, + }; + + result.net_pair.tap.id = String::from("uniqueTestID_kata"); + result.net_pair.tap.tap_iface.hard_addr = String::from(""); + result.net_pair.virt_iface.hard_addr = mac_addr.clone(); + + // check the integrity by compare all variables + assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id); + assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name); + assert_eq!( + manual.net_pair.tap.tap_iface.name, + result.net_pair.tap.tap_iface.name + ); + assert_eq!( + manual.net_pair.tap.tap_iface.hard_addr, + result.net_pair.tap.tap_iface.hard_addr + ); + assert_eq!( + manual.net_pair.tap.tap_iface.addrs, + result.net_pair.tap.tap_iface.addrs + ); + assert_eq!( + manual.net_pair.virt_iface.name, + result.net_pair.virt_iface.name + ); + assert_eq!( + manual.net_pair.virt_iface.hard_addr, + result.net_pair.virt_iface.hard_addr + ); + // using match branch to avoid deriving PartialEq trait + match manual.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} // ok + _ => unreachable!(), + } + match result.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} + _ => unreachable!(), + } + assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); + } + let link_index = fetch_index(&handle, manual_vlan_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + let link_index = fetch_index(&handle, tap_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + assert!(handle.link().del(dummy_index).execute().await.is_ok()); + } + } + } + } + + // this unit test tests the integrity of VlanEndpoint::new() + #[actix_rt::test] + async fn test_macvlan_construction() { + let idx = 8194; + let mac_addr = String::from("02:25:CA:FE:00:04"); + let manual_macvlan_iface_name = format!("eth{}", idx); + let tap_iface_name = format!("tap{}_kata", idx); // create by NetworkPair::new() + let model_str = TC_FILTER_NET_MODEL_STR; + let dummy_name = format!("dummy{}", idx); + + if let Ok((conn, handle, _)) = + rtnetlink::new_connection().context("failed to create netlink connection") + { + let thread_handler = tokio::spawn(conn); + defer!({ + thread_handler.abort(); + }); + + if let Ok(()) = handle + .link() + .add() + .dummy(dummy_name.clone()) + .execute() + .await + .context("failed to create dummy link") + { + let dummy_index = fetch_index(&handle, dummy_name.clone().as_str()) + .await + .expect("failed to get the index of dummy link"); + + // the mode here does not matter, could be any of available modes + if let Ok(()) = handle + .link() + .add() + .macvlan( + manual_macvlan_iface_name.clone(), + dummy_index, + MACVLAN_MODE_PRIVATE, + ) + .execute() + .await + .context("failed to create manual macvlan pair") + { + // model here does not matter, could be any of supported models + if let Ok(mut result) = MacVlanEndpoint::new( + &handle, + manual_macvlan_iface_name.clone().as_str(), + idx, + model_str, + 5, + ) + .await + .context("failed to create new macvlan endpoint") + { + let manual = MacVlanEndpoint { + net_pair: NetworkPair { + tap: TapInterface { + id: String::from("uniqueTestID_kata"), + name: format!("br{}_kata", idx), + tap_iface: NetworkInterface { + name: tap_iface_name.clone(), + ..Default::default() + }, + }, + virt_iface: NetworkInterface { + name: manual_macvlan_iface_name.clone(), + hard_addr: mac_addr.clone(), + ..Default::default() + }, + model: network_model::new(model_str) + .expect("failed to create new network model"), + network_qos: false, + }, + }; + + result.net_pair.tap.id = String::from("uniqueTestID_kata"); + result.net_pair.tap.tap_iface.hard_addr = String::from(""); + result.net_pair.virt_iface.hard_addr = mac_addr.clone(); + + // check the integrity by compare all variables + assert_eq!(manual.net_pair.tap.id, result.net_pair.tap.id); + assert_eq!(manual.net_pair.tap.name, result.net_pair.tap.name); + assert_eq!( + manual.net_pair.tap.tap_iface.name, + result.net_pair.tap.tap_iface.name + ); + assert_eq!( + manual.net_pair.tap.tap_iface.hard_addr, + result.net_pair.tap.tap_iface.hard_addr + ); + assert_eq!( + manual.net_pair.tap.tap_iface.addrs, + result.net_pair.tap.tap_iface.addrs + ); + assert_eq!( + manual.net_pair.virt_iface.name, + result.net_pair.virt_iface.name + ); + assert_eq!( + manual.net_pair.virt_iface.hard_addr, + result.net_pair.virt_iface.hard_addr + ); + // using match branch to avoid deriving PartialEq trait + // TcFilter model is hard-coded "model_str" variable + match manual.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} // ok + _ => unreachable!(), + } + match result.net_pair.model.model_type() { + NetworkModelType::TcFilter => {} + _ => unreachable!(), + } + assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); + } + // delete the manually created links + let link_index = fetch_index(&handle, manual_macvlan_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + let link_index = fetch_index(&handle, tap_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + assert!(handle.link().del(dummy_index).execute().await.is_ok()); + } + } + } + } + // this unit test tests the integrity of IPVlanEndpoint::new() - // by comparing the manual constructed object with object constructed by new() #[actix_rt::test] async fn test_ipvlan_construction() { let idx = 8192; @@ -30,7 +276,7 @@ mod tests { let tap_iface_name = format!("tap{}_kata", idx); // create by kata if let Ok((conn, handle, _)) = - rtnetlink::new_connection().context("failed to create new netlink connection") + rtnetlink::new_connection().context("failed to create netlink connection") { let thread_handler = tokio::spawn(conn); defer!({ @@ -45,11 +291,11 @@ mod tests { .veth("foo".to_string(), manual_virt_iface_name.clone()) .execute() .await - .context("failed to create virt_iface") + .context("failed to create manual veth pair") { if let Ok(mut result) = IPVlanEndpoint::new(&handle, "", idx, 5) .await - .context("failed to create new IPVlan Endpoint") + .context("failed to create new ipvlan endpoint") { let manual = IPVlanEndpoint { net_pair: NetworkPair { @@ -98,10 +344,6 @@ mod tests { manual.net_pair.virt_iface.hard_addr, result.net_pair.virt_iface.hard_addr ); - assert_eq!( - manual.net_pair.virt_iface.addrs, - result.net_pair.virt_iface.addrs - ); // using match branch to avoid deriving PartialEq trait match manual.net_pair.model.model_type() { NetworkModelType::TcFilter => {} // ok @@ -113,13 +355,14 @@ mod tests { } assert_eq!(manual.net_pair.network_qos, result.net_pair.network_qos); } - if let Ok(link_index) = fetch_index(&handle, manual_virt_iface_name.as_str()).await - { - assert!(handle.link().del(link_index).execute().await.is_ok()) - } - if let Ok(link_index) = fetch_index(&handle, tap_iface_name.as_str()).await { - assert!(handle.link().del(link_index).execute().await.is_ok()) - } + let link_index = fetch_index(&handle, manual_virt_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); + let link_index = fetch_index(&handle, tap_iface_name.as_str()) + .await + .expect("failed to fetch index"); + assert!(handle.link().del(link_index).execute().await.is_ok()); } } } diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs new file mode 100644 index 000000000000..21f22345d83b --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/macvlan_endpoint.rs @@ -0,0 +1,84 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{self, Error}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use hypervisor::{device::NetworkConfig, Device, Hypervisor}; + +use super::Endpoint; +use crate::network::{utils, NetworkPair}; + +#[derive(Debug)] +pub struct MacVlanEndpoint { + pub(crate) net_pair: NetworkPair, +} + +impl MacVlanEndpoint { + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + idx: u32, + model: &str, + queues: usize, + ) -> Result { + let net_pair = NetworkPair::new(handle, idx, name, model, queues) + .await + .context("error creating new networkInterfacePair")?; + Ok(MacVlanEndpoint { net_pair }) + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + Ok(NetworkConfig { + id: self.net_pair.virt_iface.name.clone(), + host_dev_name: iface.name.clone(), + guest_mac: Some(guest_mac), + }) + } +} + +#[async_trait] +impl Endpoint for MacVlanEndpoint { + async fn name(&self) -> String { + self.net_pair.virt_iface.name.clone() + } + + async fn hardware_addr(&self) -> String { + self.net_pair.tap.tap_iface.hard_addr.clone() + } + + async fn attach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .add_network_model() + .await + .context("add network model")?; + let config = self.get_network_config().context("get network config")?; + h.add_device(Device::Network(config)) + .await + .context("Error add device")?; + Ok(()) + } + + async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .del_network_model() + .await + .context("del network model")?; + let config = self.get_network_config().context("get network config")?; + h.remove_device(Device::Network(config)) + .await + .context("remove device")?; + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs index b8e4557b53d1..9e5e841c82b4 100644 --- a/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs +++ b/src/runtime-rs/crates/resource/src/network/endpoint/mod.rs @@ -10,6 +10,10 @@ mod veth_endpoint; pub use veth_endpoint::VethEndpoint; mod ipvlan_endpoint; pub use ipvlan_endpoint::IPVlanEndpoint; +mod vlan_endpoint; +pub use vlan_endpoint::VlanEndpoint; +mod macvlan_endpoint; +pub use macvlan_endpoint::MacVlanEndpoint; mod endpoints_test; use anyhow::Result; diff --git a/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs new file mode 100644 index 000000000000..14626318cfa2 --- /dev/null +++ b/src/runtime-rs/crates/resource/src/network/endpoint/vlan_endpoint.rs @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::io::{self, Error}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use super::Endpoint; +use crate::network::network_model::TC_FILTER_NET_MODEL_STR; +use crate::network::{utils, NetworkPair}; +use hypervisor::{device::NetworkConfig, Device, Hypervisor}; + +#[derive(Debug)] +pub struct VlanEndpoint { + pub(crate) net_pair: NetworkPair, +} + +impl VlanEndpoint { + pub async fn new( + handle: &rtnetlink::Handle, + name: &str, + idx: u32, + queues: usize, + ) -> Result { + let net_pair = NetworkPair::new(handle, idx, name, TC_FILTER_NET_MODEL_STR, queues) + .await + .context("error creating networkInterfacePair")?; + Ok(VlanEndpoint { net_pair }) + } + + fn get_network_config(&self) -> Result { + let iface = &self.net_pair.tap.tap_iface; + let guest_mac = utils::parse_mac(&iface.hard_addr).ok_or_else(|| { + Error::new( + io::ErrorKind::InvalidData, + format!("hard_addr {}", &iface.hard_addr), + ) + })?; + Ok(NetworkConfig { + id: self.net_pair.virt_iface.name.clone(), + host_dev_name: iface.name.clone(), + guest_mac: Some(guest_mac), + }) + } +} + +#[async_trait] +impl Endpoint for VlanEndpoint { + async fn name(&self) -> String { + self.net_pair.virt_iface.name.clone() + } + + async fn hardware_addr(&self) -> String { + self.net_pair.tap.tap_iface.hard_addr.clone() + } + + async fn attach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .add_network_model() + .await + .context("error adding network model")?; + let config = self.get_network_config().context("get network config")?; + h.add_device(Device::Network(config)) + .await + .context("error adding device by hypervisor")?; + + Ok(()) + } + + async fn detach(&self, h: &dyn Hypervisor) -> Result<()> { + self.net_pair + .del_network_model() + .await + .context("error deleting network model")?; + let config = self + .get_network_config() + .context("error getting network config")?; + h.remove_device(Device::Network(config)) + .await + .context("error removing device by hypervisor")?; + + Ok(()) + } +} diff --git a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs index 1d2a1e55f960..d89e8ab7e696 100644 --- a/src/runtime-rs/crates/resource/src/network/network_with_netns.rs +++ b/src/runtime-rs/crates/resource/src/network/network_with_netns.rs @@ -17,7 +17,9 @@ use scopeguard::defer; use tokio::sync::RwLock; use super::{ - endpoint::{Endpoint, IPVlanEndpoint, PhysicalEndpoint, VethEndpoint}, + endpoint::{ + Endpoint, IPVlanEndpoint, MacVlanEndpoint, PhysicalEndpoint, VethEndpoint, VlanEndpoint, + }, network_entity::NetworkEntity, network_info::network_info_from_link::NetworkInfoFromLink, utils::{link, netns}, @@ -186,12 +188,30 @@ async fn create_endpoint( .context("veth endpoint")?; Arc::new(ret) } + "vlan" => { + let ret = VlanEndpoint::new(handle, &attrs.name, idx, config.queues) + .await + .context("vlan endpoint")?; + Arc::new(ret) + } "ipvlan" => { let ret = IPVlanEndpoint::new(handle, &attrs.name, idx, config.queues) .await .context("ipvlan endpoint")?; Arc::new(ret) } + "macvlan" => { + let ret = MacVlanEndpoint::new( + handle, + &attrs.name, + idx, + &config.network_model, + config.queues, + ) + .await + .context("macvlan endpoint")?; + Arc::new(ret) + } _ => return Err(anyhow!("unsupported link type: {}", link_type)), } }; diff --git a/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs b/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs index fd4a8b1c99fe..efc43bb7049a 100644 --- a/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs +++ b/src/runtime-rs/crates/resource/src/network/utils/link/manager.rs @@ -103,6 +103,11 @@ fn link_info(mut infos: Vec) -> Box { link = Some(Box::new(IpVlan::default())); } } + InfoKind::MacVlan => { + if link.is_none() { + link = Some(Box::new(MacVlan::default())); + } + } InfoKind::Vlan => { if link.is_none() { link = Some(Box::new(Vlan::default())); @@ -129,6 +134,9 @@ fn link_info(mut infos: Vec) -> Box { InfoData::IpVlan(_) => { link = Some(Box::new(IpVlan::default())); } + InfoData::MacVlan(_) => { + link = Some(Box::new(MacVlan::default())); + } InfoData::Vlan(_) => { link = Some(Box::new(Vlan::default())); } @@ -247,6 +255,26 @@ impl Link for IpVlan { } } +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct MacVlan { + attrs: Option, + + /// on create only + pub peer_name: String, +} + +impl Link for MacVlan { + fn attrs(&self) -> &LinkAttrs { + self.attrs.as_ref().unwrap() + } + fn set_attrs(&mut self, attr: LinkAttrs) { + self.attrs = Some(attr) + } + fn r#type(&self) -> &'static str { + "macvlan" + } +} + #[derive(Debug, PartialEq, Eq, Clone, Default)] pub struct Vlan { attrs: Option, From fa85fd584e6719838d9d77d950b9309fd409a20d Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Fri, 22 Jul 2022 10:30:21 +0800 Subject: [PATCH 0146/1953] docs: add rust environment setup for kata 3.0 add more details for rust set up in kata 3.0 install guide Fixes: #4720 Signed-off-by: Zhongtao Hu --- ...ers-3.0-rust-runtime-installation-guide.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md index 8e50901ba9ff..122e43b51291 100644 --- a/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md +++ b/docs/install/kata-containers-3.0-rust-runtime-installation-guide.md @@ -44,6 +44,38 @@ architectures: ## Build from source installation +### Rust Environment Set Up + +* Download `Rustup` and install `Rust` + > **Notes:** + > Rust version 1.58 is needed + + Example for `x86_64` + ``` + $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + $ source $HOME/.cargo/env + $ rustup install 1.58 + $ rustup default 1.58-x86_64-unknown-linux-gnu + ``` + +* Musl support for fully static binary + + Example for `x86_64` + ``` + $ rustup target add x86_64-unknown-linux-musl + ``` +* [Musl `libc`](http://musl.libc.org/) install + + Example for musl 1.2.3 + ``` + $ wget https://git.musl-libc.org/cgit/musl/snapshot/musl-1.2.3.tar.gz + $ tar vxf musl-1.2.3.tar.gz + $ cd musl-1.2.3/ + $ ./configure --prefix=/usr/local/ + $ make && sudo make install + ``` + + ### Install Kata 3.0 Rust Runtime Shim ``` From e0194dcb5eddda583499edfa084761df08723d1d Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Sat, 23 Jul 2022 15:18:00 +0800 Subject: [PATCH 0147/1953] runtime-rs: update route destination with prefix Update route destination with prefix. Fixes: #4726 Signed-off-by: Quanwei Zhou --- .../resource/src/network/network_info/network_info_from_link.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs index a15f09796f08..e8d549491545 100644 --- a/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs +++ b/src/runtime-rs/crates/resource/src/network/network_info/network_info_from_link.rs @@ -163,7 +163,7 @@ fn generate_route(name: &str, route: &RouteMessage) -> Result> { Ok(Some(Route { dest: route .destination_prefix() - .map(|(addr, _)| addr.to_string()) + .map(|(addr, prefix)| format!("{}/{}", addr, prefix)) .unwrap_or_default(), gateway: route.gateway().map(|v| v.to_string()).unwrap_or_default(), device: name.to_string(), From c825065b27ffb742f3e534e52ea9767152ad69cb Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Sun, 24 Jul 2022 20:44:24 +0800 Subject: [PATCH 0148/1953] runtime-rs: fix tc filter setup failed Fix bug using tc filter and protocol needs to use network byte order. Fixes: #4726 Signed-off-by: Quanwei Zhou --- .../src/network/network_model/tc_filter_model.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs index 6c014a7bc083..ff689b9b845b 100644 --- a/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs +++ b/src/runtime-rs/crates/resource/src/network/network_model/tc_filter_model.rs @@ -20,6 +20,7 @@ impl TcFilterModel { Ok(Self {}) } } + #[async_trait] impl NetworkModel for TcFilterModel { fn model_type(&self) -> NetworkModelType { @@ -60,22 +61,25 @@ impl NetworkModel for TcFilterModel { handle .traffic_filter(tap_index as i32) .add() - .protocol(0x0003) - .egress() + .parent(0xffff0000) + // get protocol with network byte order + .protocol(0x0003_u16.to_be()) .redirect(virt_index) .execute() .await - .context("add tap egress")?; + .context("add redirect for tap")?; handle .traffic_filter(virt_index as i32) .add() - .protocol(0x0003) - .egress() + .parent(0xffff0000) + // get protocol with network byte order + .protocol(0x0003_u16.to_be()) .redirect(tap_index) .execute() .await - .context("add virt egress")?; + .context("add redirect for virt")?; + Ok(()) } From 0e24f47a4351a41927cfefd73966a1a95702117d Mon Sep 17 00:00:00 2001 From: liubin Date: Tue, 26 Jul 2022 10:32:44 +0800 Subject: [PATCH 0149/1953] agent: log RPC calls for debugging We can log all RPC calls to the agent for debugging purposes to check which RPC is called, which can help us to understand the container lifespan. Fixes: #4738 Signed-off-by: liubin --- src/agent/src/tracer.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/agent/src/tracer.rs b/src/agent/src/tracer.rs index 1854876da292..bad4a6f509ea 100644 --- a/src/agent/src/tracer.rs +++ b/src/agent/src/tracer.rs @@ -69,6 +69,8 @@ macro_rules! trace_rpc_call { propagator.extract(&extract_carrier_from_ttrpc($ctx)) }); + info!(sl!(), "rpc call from shim to agent: {:?}", $name); + // generate tracing span let rpc_span = span!(tracing::Level::INFO, $name, "mod"="rpc.rs", req=?$req); From 57c556a80150f1a205934107fdc30c3932d33bd8 Mon Sep 17 00:00:00 2001 From: Quanwei Zhou Date: Tue, 26 Jul 2022 11:43:00 +0800 Subject: [PATCH 0150/1953] runtime-rs: fix stop failed in azure Fix the stop failed in azure. Fixes: #4740 Signed-off-by: Quanwei Zhou --- src/runtime-rs/crates/resource/src/cgroups/mod.rs | 7 +++++-- .../crates/runtimes/virt_container/src/sandbox.rs | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/runtime-rs/crates/resource/src/cgroups/mod.rs b/src/runtime-rs/crates/resource/src/cgroups/mod.rs index 367e9fba8339..9a176c2fd8ac 100644 --- a/src/runtime-rs/crates/resource/src/cgroups/mod.rs +++ b/src/runtime-rs/crates/resource/src/cgroups/mod.rs @@ -100,13 +100,16 @@ impl CgroupsResource { for cg_pid in self.cgroup_manager.tasks() { self.cgroup_manager.remove_task(cg_pid); } - self.cgroup_manager.delete()?; + + self.cgroup_manager + .delete() + .context("delete cgroup manager")?; if let Some(overhead) = self.overhead_cgroup_manager.as_ref() { for cg_pid in overhead.tasks() { overhead.remove_task(cg_pid); } - overhead.delete()?; + overhead.delete().context("delete overhead")?; } Ok(()) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 6bcf3d2dfc2f..5f9625ad751b 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -210,13 +210,16 @@ impl Sandbox for VirtSandbox { async fn stop(&self) -> Result<()> { info!(sl!(), "begin stop sandbox"); - // TODO: stop sandbox + self.hypervisor.stop_vm().await.context("stop vm")?; Ok(()) } async fn shutdown(&self) -> Result<()> { info!(sl!(), "shutdown"); + self.stop().await.context("stop")?; + + info!(sl!(), "delete cgroup"); self.resource_manager .delete_cgroups() .await From 1ef3f8eac6c7b7544dc203c66d1cb5461ba5db6e Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Mon, 18 Jul 2022 17:31:01 +0800 Subject: [PATCH 0151/1953] runtime-rs: set share sandbox pid namespace Set the share sandbox pid namepsace from spec Fixes:#4680 Signed-off-by: Zhongtao Hu --- .../src/container_manager/container.rs | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 2e50b55e924d..860a5f0c2916 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -80,8 +80,8 @@ impl Container { let mut inner = self.inner.write().await; let toml_config = self.resource_manager.config().await; let config = &self.config; - amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp).context("load spec")?; - + let sandbox_pidns = amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp) + .context("load spec")?; // handler rootfs let rootfs = self .resource_manager @@ -143,7 +143,7 @@ impl Container { storages, oci: Some(spec), guest_hooks: None, - sandbox_pidns: false, + sandbox_pidns, rootfs_mounts: vec![], }; @@ -373,7 +373,7 @@ impl Container { } } -fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> { +fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result { // hook should be done on host spec.hooks = None; @@ -390,6 +390,8 @@ fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> { resource.network = None; } + // Host pidns path does not make sense in kata. Let's just align it with + // sandbox namespace whenever it is set. let mut ns: Vec = Vec::new(); for n in linux.namespaces.iter() { match n.r#type.as_str() { @@ -399,9 +401,27 @@ fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> { } linux.namespaces = ns; + + return Ok(handle_pid_namespace(&linux.namespaces)); } - Ok(()) + Ok(false) +} + +// handle_pid_namespace checks if Pid namespace for a container needs to be shared with its sandbox +// pid namespace. +fn handle_pid_namespace(namespaces: &[oci::LinuxNamespace]) -> bool { + for n in namespaces.iter() { + match n.r#type.as_str() { + oci::PIDNAMESPACE => { + if !n.path.is_empty() { + return true; + } + } + _ => continue, + } + } + false } #[cfg(test)] From b3147411e3849e59dbca0fafec0756820df5c65f Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Tue, 19 Jul 2022 00:26:11 +0800 Subject: [PATCH 0152/1953] runtime-rs:add unit test for set share pid ns Fixes:#4680 Signed-off-by: Zhongtao Hu --- .../src/container_manager/container.rs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 860a5f0c2916..2d414318bd7f 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -427,7 +427,7 @@ fn handle_pid_namespace(namespaces: &[oci::LinuxNamespace]) -> bool { #[cfg(test)] mod tests { use super::amend_spec; - + use crate::container_manager::container::handle_pid_namespace; #[test] fn test_amend_spec_disable_guest_seccomp() { let mut spec = oci::Spec { @@ -448,4 +448,38 @@ mod tests { amend_spec(&mut spec, true).unwrap(); assert!(spec.linux.as_ref().unwrap().seccomp.is_none()); } + #[test] + fn test_handle_pid_namespace() { + let namespaces = vec![ + oci::LinuxNamespace { + r#type: "pid".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "network".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "ipc".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "uts".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "mount".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "user".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "cgroup".to_string(), + path: "".to_string(), + }, + ]; + assert!(!handle_pid_namespace(&namespaces)); + } } From 56d49b50731edc9cdbda716b74ad2581ec3fb66b Mon Sep 17 00:00:00 2001 From: gntouts Date: Wed, 20 Jul 2022 16:07:21 +0000 Subject: [PATCH 0153/1953] versions: Update Firecracker version to v1.1.0 This patch upgrades Firecracker version from v0.23.4 to v1.1.0 * Generate swagger models for v1.1.0 (from firecracker.yaml) * Replace ht_enabled param to smt (API change) * Remove NUMA-related jailer param --node 0 Fixes: #4673 Depends-on: github.com/kata-containers/tests#4968 Signed-off-by: George Ntoutsos Signed-off-by: Anastassios Nanos --- src/runtime/go.mod | 10 +- src/runtime/go.sum | 164 +- .../asaskevich/govalidator/.gitignore | 15 + .../asaskevich/govalidator/.travis.yml | 20 +- .../asaskevich/govalidator/CODE_OF_CONDUCT.md | 43 + .../asaskevich/govalidator/CONTRIBUTING.md | 4 +- .../github.com/asaskevich/govalidator/LICENSE | 2 +- .../asaskevich/govalidator/README.md | 146 +- .../asaskevich/govalidator/arrays.go | 29 + .../asaskevich/govalidator/converter.go | 33 +- .../github.com/asaskevich/govalidator/doc.go | 3 + .../asaskevich/govalidator/error.go | 6 +- .../github.com/asaskevich/govalidator/go.mod | 3 + .../asaskevich/govalidator/numerics.go | 37 +- .../asaskevich/govalidator/patterns.go | 168 +- .../asaskevich/govalidator/types.go | 55 +- .../asaskevich/govalidator/utils.go | 46 +- .../asaskevich/govalidator/validator.go | 535 ++++- .../asaskevich/govalidator/wercker.yml | 2 +- .../vendor/github.com/globalsign/mgo/LICENSE | 25 - .../github.com/globalsign/mgo/bson/LICENSE | 25 - .../github.com/globalsign/mgo/bson/README.md | 12 - .../github.com/globalsign/mgo/bson/bson.go | 836 -------- .../globalsign/mgo/bson/compatibility.go | 29 - .../github.com/globalsign/mgo/bson/decimal.go | 312 --- .../github.com/globalsign/mgo/bson/decode.go | 1055 ---------- .../github.com/globalsign/mgo/bson/encode.go | 645 ------- .../github.com/globalsign/mgo/bson/json.go | 384 ---- .../github.com/globalsign/mgo/bson/stream.go | 90 - .../globalsign/mgo/internal/json/LICENSE | 27 - .../globalsign/mgo/internal/json/decode.go | 1685 ---------------- .../globalsign/mgo/internal/json/encode.go | 1260 ------------ .../globalsign/mgo/internal/json/extension.go | 95 - .../globalsign/mgo/internal/json/fold.go | 143 -- .../globalsign/mgo/internal/json/indent.go | 141 -- .../globalsign/mgo/internal/json/scanner.go | 697 ------- .../globalsign/mgo/internal/json/stream.go | 510 ----- .../globalsign/mgo/internal/json/tags.go | 44 - .../go-openapi/analysis/.codecov.yml | 5 + .../go-openapi/analysis/.gitattributes | 2 + .../go-openapi/analysis/.golangci.yml | 38 +- .../go-openapi/analysis/.travis.yml | 24 - .../github.com/go-openapi/analysis/README.md | 28 +- .../go-openapi/analysis/analyzer.go | 402 ++-- .../go-openapi/analysis/appveyor.yml | 32 + .../github.com/go-openapi/analysis/debug.go | 28 +- .../github.com/go-openapi/analysis/fixer.go | 57 +- .../github.com/go-openapi/analysis/flatten.go | 1584 +++++---------- .../go-openapi/analysis/flatten_name.go | 293 +++ .../go-openapi/analysis/flatten_options.go | 78 + .../github.com/go-openapi/analysis/go.mod | 16 +- .../github.com/go-openapi/analysis/go.sum | 183 +- .../analysis/internal/debug/debug.go | 41 + .../internal/flatten/normalize/normalize.go | 87 + .../internal/flatten/operations/operations.go | 90 + .../internal/flatten/replace/replace.go | 434 +++++ .../flatten/schutils/flatten_schema.go | 29 + .../analysis/internal/flatten/sortref/keys.go | 201 ++ .../internal/flatten/sortref/sort_ref.go | 141 ++ .../github.com/go-openapi/analysis/mixin.go | 197 +- .../github.com/go-openapi/analysis/schema.go | 96 +- .../go-openapi/errors/.gitattributes | 1 + .../go-openapi/errors/.golangci.yml | 27 + .../github.com/go-openapi/errors/.travis.yml | 14 - .../github.com/go-openapi/errors/README.md | 7 +- .../github.com/go-openapi/errors/api.go | 27 +- .../github.com/go-openapi/errors/auth.go | 4 +- .../github.com/go-openapi/errors/go.mod | 9 +- .../github.com/go-openapi/errors/go.sum | 25 +- .../github.com/go-openapi/errors/headers.go | 26 +- .../go-openapi/errors/middleware.go | 6 +- .../github.com/go-openapi/errors/parsing.go | 23 +- .../github.com/go-openapi/errors/schema.go | 101 +- .../go-openapi/jsonreference/go.mod | 3 +- .../go-openapi/jsonreference/go.sum | 15 +- .../github.com/go-openapi/loads/.drone.sec | 1 - .../github.com/go-openapi/loads/.drone.yml | 39 - .../github.com/go-openapi/loads/.golangci.yml | 44 + .../github.com/go-openapi/loads/.travis.yml | 25 +- .../github.com/go-openapi/loads/README.md | 5 +- .../internal/pre_go18.go => loads/doc.go} | 18 +- .../vendor/github.com/go-openapi/loads/go.mod | 26 +- .../vendor/github.com/go-openapi/loads/go.sum | 184 +- .../github.com/go-openapi/loads/loaders.go | 134 ++ .../github.com/go-openapi/loads/options.go | 61 + .../github.com/go-openapi/loads/spec.go | 201 +- .../github.com/go-openapi/runtime/.travis.yml | 19 +- .../go-openapi/runtime/bytestream.go | 4 + .../go-openapi/runtime/client/auth_info.go | 3 +- .../go-openapi/runtime/client/request.go | 213 +- .../go-openapi/runtime/client/runtime.go | 50 +- .../go-openapi/runtime/client_request.go | 3 + .../go-openapi/runtime/client_response.go | 9 +- .../go-openapi/runtime/constants.go | 2 + .../github.com/go-openapi/runtime/csv.go | 77 + .../github.com/go-openapi/runtime/go.mod | 29 +- .../github.com/go-openapi/runtime/go.sum | 249 ++- .../go-openapi/runtime/logger/logger.go | 10 +- .../go-openapi/runtime/middleware/context.go | 44 +- .../runtime/middleware/denco/router.go | 12 +- .../runtime/middleware/not_implemented.go | 23 +- .../runtime/middleware/parameter.go | 7 +- .../go-openapi/runtime/middleware/redoc.go | 10 +- .../go-openapi/runtime/middleware/request.go | 16 +- .../go-openapi/runtime/middleware/router.go | 10 +- .../go-openapi/runtime/middleware/spec.go | 2 +- .../runtime/middleware/swaggerui.go | 162 ++ .../runtime/middleware/untyped/api.go | 14 +- .../runtime/middleware/validation.go | 8 +- .../github.com/go-openapi/runtime/request.go | 64 +- .../runtime/security/authenticator.go | 67 +- .../github.com/go-openapi/runtime/text.go | 6 + .../github.com/go-openapi/spec/.golangci.yml | 19 + .../github.com/go-openapi/spec/.travis.yml | 24 +- .../github.com/go-openapi/spec/README.md | 32 +- .../github.com/go-openapi/spec/appveyor.yml | 32 + .../github.com/go-openapi/spec/bindata.go | 38 +- .../github.com/go-openapi/spec/cache.go | 54 +- .../go-openapi/spec/contact_info.go | 33 + .../github.com/go-openapi/spec/debug.go | 12 +- .../github.com/go-openapi/spec/errors.go | 19 + .../github.com/go-openapi/spec/expander.go | 570 +++--- .../vendor/github.com/go-openapi/spec/go.mod | 17 +- .../vendor/github.com/go-openapi/spec/go.sum | 81 +- .../github.com/go-openapi/spec/header.go | 6 + .../github.com/go-openapi/spec/items.go | 22 +- .../github.com/go-openapi/spec/license.go | 33 + .../github.com/go-openapi/spec/normalizer.go | 243 ++- .../go-openapi/spec/normalizer_nonwindows.go | 43 + .../go-openapi/spec/normalizer_windows.go | 154 ++ .../github.com/go-openapi/spec/operation.go | 1 - .../github.com/go-openapi/spec/parameter.go | 11 +- .../github.com/go-openapi/spec/properties.go | 91 + .../vendor/github.com/go-openapi/spec/ref.go | 4 +- .../github.com/go-openapi/spec/resolver.go | 127 ++ .../github.com/go-openapi/spec/response.go | 25 +- .../github.com/go-openapi/spec/schema.go | 120 +- .../go-openapi/spec/schema_loader.go | 234 ++- .../go-openapi/spec/security_scheme.go | 44 +- .../vendor/github.com/go-openapi/spec/spec.go | 14 +- .../github.com/go-openapi/spec/unused.go | 174 -- .../github.com/go-openapi/spec/validations.go | 215 +++ .../go-openapi/strfmt/.gitattributes | 2 + .../go-openapi/strfmt/.golangci.yml | 29 +- .../github.com/go-openapi/strfmt/.travis.yml | 20 - .../github.com/go-openapi/strfmt/README.md | 25 +- .../github.com/go-openapi/strfmt/bson.go | 128 +- .../github.com/go-openapi/strfmt/date.go | 101 +- .../github.com/go-openapi/strfmt/default.go | 1218 +++++++----- .../github.com/go-openapi/strfmt/duration.go | 74 +- .../github.com/go-openapi/strfmt/format.go | 72 +- .../github.com/go-openapi/strfmt/go.mod | 18 +- .../github.com/go-openapi/strfmt/go.sum | 76 +- .../github.com/go-openapi/strfmt/time.go | 173 +- .../github.com/go-openapi/strfmt/ulid.go | 225 +++ .../github.com/go-openapi/swag/.gitattributes | 2 + .../github.com/go-openapi/swag/.golangci.yml | 11 + .../github.com/go-openapi/swag/.travis.yml | 37 - .../internal/post_go18.go => swag/file.go} | 28 +- .../github.com/go-openapi/swag/post_go18.go | 1 + .../github.com/go-openapi/swag/post_go19.go | 1 + .../github.com/go-openapi/swag/pre_go18.go | 1 + .../github.com/go-openapi/swag/pre_go19.go | 1 + .../go-openapi/validate/.gitattributes | 2 + .../go-openapi/validate/.golangci.yml | 38 +- .../go-openapi/validate/.travis.yml | 35 - .../github.com/go-openapi/validate/README.md | 40 +- .../go-openapi/validate/appveyor.yml | 32 + .../github.com/go-openapi/validate/context.go | 56 + .../go-openapi/validate/default_validator.go | 131 +- .../go-openapi/validate/example_validator.go | 141 +- .../github.com/go-openapi/validate/formats.go | 14 +- .../github.com/go-openapi/validate/go.mod | 21 +- .../github.com/go-openapi/validate/go.sum | 194 +- .../github.com/go-openapi/validate/helpers.go | 87 +- .../go-openapi/validate/object_validator.go | 87 +- .../github.com/go-openapi/validate/result.go | 14 +- .../github.com/go-openapi/validate/rexp.go | 2 +- .../github.com/go-openapi/validate/schema.go | 30 +- .../go-openapi/validate/schema_option.go | 54 + .../go-openapi/validate/schema_props.go | 25 +- .../go-openapi/validate/slice_validator.go | 11 +- .../github.com/go-openapi/validate/spec.go | 227 ++- .../go-openapi/validate/spec_messages.go | 10 +- .../github.com/go-openapi/validate/type.go | 94 +- .../go-openapi/validate/validator.go | 33 +- .../github.com/go-openapi/validate/values.go | 106 +- .../github.com/go-stack/stack/.travis.yml | 15 + .../github.com/go-stack/stack/LICENSE.md | 21 + .../github.com/go-stack/stack/README.md | 38 + .../vendor/github.com/go-stack/stack/go.mod | 1 + .../vendor/github.com/go-stack/stack/stack.go | 400 ++++ .../mitchellh/mapstructure/.travis.yml | 8 - .../mitchellh/mapstructure/CHANGELOG.md | 52 + .../mitchellh/mapstructure/decode_hooks.go | 71 +- .../github.com/mitchellh/mapstructure/go.mod | 2 + .../mitchellh/mapstructure/mapstructure.go | 479 ++++- .../vendor/github.com/oklog/ulid/.gitignore | 29 + .../vendor/github.com/oklog/ulid/.travis.yml | 16 + .../vendor/github.com/oklog/ulid/AUTHORS.md | 2 + .../vendor/github.com/oklog/ulid/CHANGELOG.md | 33 + .../github.com/oklog/ulid/CONTRIBUTING.md | 17 + .../vendor/github.com/oklog/ulid/Gopkg.lock | 15 + .../vendor/github.com/oklog/ulid/Gopkg.toml | 26 + .../vendor/github.com/oklog/ulid/LICENSE | 201 ++ .../vendor/github.com/oklog/ulid/README.md | 150 ++ .../vendor/github.com/oklog/ulid/ulid.go | 614 ++++++ .../go.mongodb.org/mongo-driver/LICENSE | 201 ++ .../go.mongodb.org/mongo-driver/bson/bson.go | 52 + .../mongo-driver/bson/bson_1_8.go | 81 + .../bson/bsoncodec/array_codec.go | 50 + .../mongo-driver/bson/bsoncodec/bsoncodec.go | 216 +++ .../bson/bsoncodec/byte_slice_codec.go | 111 ++ .../bson/bsoncodec/cond_addr_codec.go | 63 + .../bson/bsoncodec/default_value_decoders.go | 1717 +++++++++++++++++ .../bson/bsoncodec/default_value_encoders.go | 773 ++++++++ .../mongo-driver/bson/bsoncodec/doc.go | 84 + .../bson/bsoncodec/empty_interface_codec.go | 140 ++ .../mongo-driver/bson/bsoncodec/map_codec.go | 297 +++ .../mongo-driver/bson/bsoncodec/mode.go | 65 + .../bson/bsoncodec/pointer_codec.go | 109 ++ .../mongo-driver/bson/bsoncodec/proxy.go | 14 + .../mongo-driver/bson/bsoncodec/registry.go | 474 +++++ .../bson/bsoncodec/slice_codec.go | 199 ++ .../bson/bsoncodec/string_codec.go | 119 ++ .../bson/bsoncodec/struct_codec.go | 671 +++++++ .../bson/bsoncodec/struct_tag_parser.go | 139 ++ .../mongo-driver/bson/bsoncodec/time_codec.go | 127 ++ .../mongo-driver/bson/bsoncodec/types.go | 82 + .../mongo-driver/bson/bsoncodec/uint_codec.go | 173 ++ .../bsonoptions/byte_slice_codec_options.go | 38 + .../empty_interface_codec_options.go | 38 + .../bson/bsonoptions/map_codec_options.go | 67 + .../bson/bsonoptions/slice_codec_options.go | 38 + .../bson/bsonoptions/string_codec_options.go | 41 + .../bson/bsonoptions/struct_codec_options.go | 87 + .../bson/bsonoptions/time_codec_options.go | 38 + .../bson/bsonoptions/uint_codec_options.go | 38 + .../mongo-driver/bson/bsonrw/copier.go | 445 +++++ .../mongo-driver/bson/bsonrw/doc.go | 9 + .../bson/bsonrw/extjson_parser.go | 806 ++++++++ .../bson/bsonrw/extjson_reader.go | 644 +++++++ .../bson/bsonrw/extjson_tables.go | 223 +++ .../bson/bsonrw/extjson_wrappers.go | 492 +++++ .../bson/bsonrw/extjson_writer.go | 737 +++++++ .../mongo-driver/bson/bsonrw/json_scanner.go | 528 +++++ .../mongo-driver/bson/bsonrw/mode.go | 108 ++ .../mongo-driver/bson/bsonrw/reader.go | 63 + .../mongo-driver/bson/bsonrw/value_reader.go | 877 +++++++++ .../mongo-driver/bson/bsonrw/value_writer.go | 606 ++++++ .../mongo-driver/bson/bsonrw/writer.go | 102 + .../mongo-driver/bson/bsontype/bsontype.go | 95 + .../mongo-driver/bson/decoder.go | 118 ++ .../go.mongodb.org/mongo-driver/bson/doc.go | 138 ++ .../mongo-driver/bson/encoder.go | 99 + .../mongo-driver/bson/marshal.go | 223 +++ .../mongo-driver/bson/primitive/decimal.go | 425 ++++ .../mongo-driver/bson/primitive/objectid.go | 184 ++ .../mongo-driver/bson/primitive/primitive.go | 217 +++ .../mongo-driver/bson/primitive_codecs.go | 111 ++ .../go.mongodb.org/mongo-driver/bson/raw.go | 92 + .../mongo-driver/bson/raw_element.go | 51 + .../mongo-driver/bson/raw_value.go | 309 +++ .../mongo-driver/bson/registry.go | 24 + .../go.mongodb.org/mongo-driver/bson/types.go | 85 + .../mongo-driver/bson/unmarshal.go | 101 + .../mongo-driver/x/bsonx/bsoncore/array.go | 164 ++ .../x/bsonx/bsoncore/bson_arraybuilder.go | 201 ++ .../x/bsonx/bsoncore/bson_documentbuilder.go | 189 ++ .../mongo-driver/x/bsonx/bsoncore/bsoncore.go | 862 +++++++++ .../mongo-driver/x/bsonx/bsoncore/document.go | 410 ++++ .../x/bsonx/bsoncore/document_sequence.go | 183 ++ .../mongo-driver/x/bsonx/bsoncore/element.go | 152 ++ .../mongo-driver/x/bsonx/bsoncore/tables.go | 223 +++ .../mongo-driver/x/bsonx/bsoncore/value.go | 980 ++++++++++ src/runtime/vendor/modules.txt | 44 +- src/runtime/virtcontainers/fc.go | 53 +- .../virtcontainers/pkg/firecracker/README | 2 +- ...er_client.go => firecracker_api_client.go} | 32 +- .../pkg/firecracker/client/models/balloon.go | 91 + .../client/models/balloon_stats.go | 152 ++ .../client/models/balloon_stats_update.go | 71 + .../client/models/balloon_update.go | 71 + .../firecracker/client/models/boot_source.go | 9 +- .../firecracker/client/models/cpu_template.go | 28 +- .../pkg/firecracker/client/models/drive.go | 138 +- .../pkg/firecracker/client/models/error.go | 29 +- .../client/models/firecracker_version.go | 71 + .../client/models/full_vm_configuration.go | 495 +++++ .../client/models/instance_action_info.go | 12 +- .../client/models/instance_info.go | 82 +- .../pkg/firecracker/client/models/logger.go | 13 +- .../client/models/machine_configuration.go | 77 +- .../client/models/memory_backend.go | 124 ++ .../pkg/firecracker/client/models/metrics.go | 9 +- .../firecracker/client/models/mmds_config.go | 84 +- .../client/models/mmds_contents_object.go | 11 + .../client/models/network_interface.go | 63 +- .../client/models/partial_drive.go | 58 +- .../models/partial_network_interface.go | 60 +- .../firecracker/client/models/rate_limiter.go | 60 +- .../client/models/snapshot_create_params.go | 20 +- .../client/models/snapshot_load_params.go | 65 +- .../firecracker/client/models/token_bucket.go | 22 +- .../pkg/firecracker/client/models/vm.go | 12 +- .../pkg/firecracker/client/models/vsock.go | 23 +- .../operations/create_snapshot_parameters.go | 52 +- .../operations/create_snapshot_responses.go | 20 +- .../create_sync_action_parameters.go | 50 +- .../create_sync_action_responses.go | 20 +- .../describe_balloon_config_parameters.go | 126 ++ .../describe_balloon_config_responses.go | 153 ++ .../describe_balloon_stats_parameters.go | 126 ++ .../describe_balloon_stats_responses.go | 153 ++ .../describe_instance_parameters.go | 45 +- .../operations/describe_instance_responses.go | 17 +- .../get_export_vm_config_parameters.go | 126 ++ .../get_export_vm_config_responses.go | 115 ++ .../get_firecracker_version_parameters.go | 126 ++ .../get_firecracker_version_responses.go | 115 ++ .../get_machine_configuration_parameters.go | 45 +- .../get_machine_configuration_responses.go | 17 +- .../client/operations/get_mmds_parameters.go | 45 +- .../client/operations/get_mmds_responses.go | 25 +- .../operations/load_snapshot_parameters.go | 52 +- .../operations/load_snapshot_responses.go | 20 +- .../client/operations/operations_client.go | 882 +++++++-- .../operations/patch_balloon_parameters.go | 151 ++ .../operations/patch_balloon_responses.go | 142 ++ ...patch_balloon_stats_interval_parameters.go | 151 ++ .../patch_balloon_stats_interval_responses.go | 142 ++ .../patch_guest_drive_by_id_parameters.go | 57 +- .../patch_guest_drive_by_id_responses.go | 20 +- ...uest_network_interface_by_id_parameters.go | 57 +- ...guest_network_interface_by_id_responses.go | 20 +- .../patch_machine_configuration_parameters.go | 52 +- .../patch_machine_configuration_responses.go | 20 +- .../operations/patch_mmds_parameters.go | 56 +- .../client/operations/patch_mmds_responses.go | 22 +- .../client/operations/patch_vm_parameters.go | 52 +- .../client/operations/patch_vm_responses.go | 20 +- .../operations/put_balloon_parameters.go | 151 ++ .../operations/put_balloon_responses.go | 142 ++ .../put_guest_boot_source_parameters.go | 52 +- .../put_guest_boot_source_responses.go | 20 +- .../put_guest_drive_by_id_parameters.go | 57 +- .../put_guest_drive_by_id_responses.go | 20 +- ...uest_network_interface_by_id_parameters.go | 57 +- ...guest_network_interface_by_id_responses.go | 20 +- .../operations/put_guest_vsock_parameters.go | 52 +- .../operations/put_guest_vsock_responses.go | 20 +- .../operations/put_logger_parameters.go | 52 +- .../client/operations/put_logger_responses.go | 20 +- .../put_machine_configuration_parameters.go | 52 +- .../put_machine_configuration_responses.go | 20 +- .../operations/put_metrics_parameters.go | 52 +- .../operations/put_metrics_responses.go | 20 +- .../operations/put_mmds_config_parameters.go | 52 +- .../operations/put_mmds_config_responses.go | 22 +- .../client/operations/put_mmds_parameters.go | 56 +- .../client/operations/put_mmds_responses.go | 22 +- .../pkg/firecracker/firecracker.yaml | 425 +++- versions.yaml | 2 +- 363 files changed, 34922 insertions(+), 13425 deletions(-) create mode 100644 src/runtime/vendor/github.com/asaskevich/govalidator/.gitignore create mode 100644 src/runtime/vendor/github.com/asaskevich/govalidator/CODE_OF_CONDUCT.md create mode 100644 src/runtime/vendor/github.com/asaskevich/govalidator/doc.go create mode 100644 src/runtime/vendor/github.com/asaskevich/govalidator/go.mod delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/LICENSE delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/LICENSE delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/README.md delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/bson.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/compatibility.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/decimal.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/decode.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/encode.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/json.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/bson/stream.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/LICENSE delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/decode.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/encode.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/extension.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/fold.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/indent.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/scanner.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/stream.go delete mode 100644 src/runtime/vendor/github.com/globalsign/mgo/internal/json/tags.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/.codecov.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/.gitattributes delete mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/.travis.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/appveyor.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/flatten_name.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/flatten_options.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/debug/debug.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/flatten/normalize/normalize.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/flatten/operations/operations.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/flatten/replace/replace.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/flatten/schutils/flatten_schema.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/flatten/sortref/keys.go create mode 100644 src/runtime/vendor/github.com/go-openapi/analysis/internal/flatten/sortref/sort_ref.go create mode 100644 src/runtime/vendor/github.com/go-openapi/errors/.gitattributes delete mode 100644 src/runtime/vendor/github.com/go-openapi/errors/.travis.yml delete mode 100644 src/runtime/vendor/github.com/go-openapi/loads/.drone.sec delete mode 100644 src/runtime/vendor/github.com/go-openapi/loads/.drone.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/loads/.golangci.yml rename src/runtime/vendor/github.com/go-openapi/{analysis/internal/pre_go18.go => loads/doc.go} (60%) create mode 100644 src/runtime/vendor/github.com/go-openapi/loads/loaders.go create mode 100644 src/runtime/vendor/github.com/go-openapi/loads/options.go create mode 100644 src/runtime/vendor/github.com/go-openapi/runtime/csv.go create mode 100644 src/runtime/vendor/github.com/go-openapi/runtime/middleware/swaggerui.go create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/appveyor.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/errors.go create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/normalizer_nonwindows.go create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/normalizer_windows.go create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/properties.go create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/resolver.go delete mode 100644 src/runtime/vendor/github.com/go-openapi/spec/unused.go create mode 100644 src/runtime/vendor/github.com/go-openapi/spec/validations.go create mode 100644 src/runtime/vendor/github.com/go-openapi/strfmt/.gitattributes delete mode 100644 src/runtime/vendor/github.com/go-openapi/strfmt/.travis.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/strfmt/ulid.go create mode 100644 src/runtime/vendor/github.com/go-openapi/swag/.gitattributes delete mode 100644 src/runtime/vendor/github.com/go-openapi/swag/.travis.yml rename src/runtime/vendor/github.com/go-openapi/{analysis/internal/post_go18.go => swag/file.go} (61%) create mode 100644 src/runtime/vendor/github.com/go-openapi/validate/.gitattributes delete mode 100644 src/runtime/vendor/github.com/go-openapi/validate/.travis.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/validate/appveyor.yml create mode 100644 src/runtime/vendor/github.com/go-openapi/validate/context.go create mode 100644 src/runtime/vendor/github.com/go-openapi/validate/schema_option.go create mode 100644 src/runtime/vendor/github.com/go-stack/stack/.travis.yml create mode 100644 src/runtime/vendor/github.com/go-stack/stack/LICENSE.md create mode 100644 src/runtime/vendor/github.com/go-stack/stack/README.md create mode 100644 src/runtime/vendor/github.com/go-stack/stack/go.mod create mode 100644 src/runtime/vendor/github.com/go-stack/stack/stack.go delete mode 100644 src/runtime/vendor/github.com/mitchellh/mapstructure/.travis.yml create mode 100644 src/runtime/vendor/github.com/oklog/ulid/.gitignore create mode 100644 src/runtime/vendor/github.com/oklog/ulid/.travis.yml create mode 100644 src/runtime/vendor/github.com/oklog/ulid/AUTHORS.md create mode 100644 src/runtime/vendor/github.com/oklog/ulid/CHANGELOG.md create mode 100644 src/runtime/vendor/github.com/oklog/ulid/CONTRIBUTING.md create mode 100644 src/runtime/vendor/github.com/oklog/ulid/Gopkg.lock create mode 100644 src/runtime/vendor/github.com/oklog/ulid/Gopkg.toml create mode 100644 src/runtime/vendor/github.com/oklog/ulid/LICENSE create mode 100644 src/runtime/vendor/github.com/oklog/ulid/README.md create mode 100644 src/runtime/vendor/github.com/oklog/ulid/ulid.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/LICENSE create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bson.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bson_1_8.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/array_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/bsoncodec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/byte_slice_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/cond_addr_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/default_value_decoders.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/default_value_encoders.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/doc.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/empty_interface_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/map_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/mode.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/pointer_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/proxy.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/registry.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/slice_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/string_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/struct_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/struct_tag_parser.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/time_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/types.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsoncodec/uint_codec.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/byte_slice_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/empty_interface_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/map_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/slice_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/string_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/struct_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/time_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonoptions/uint_codec_options.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/copier.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/doc.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/extjson_parser.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/extjson_reader.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/extjson_tables.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/extjson_wrappers.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/extjson_writer.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/json_scanner.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/mode.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/reader.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/value_reader.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/value_writer.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsonrw/writer.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/bsontype/bsontype.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/decoder.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/doc.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/encoder.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/marshal.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/primitive/decimal.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/primitive/objectid.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/primitive/primitive.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/primitive_codecs.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/raw.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/raw_element.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/raw_value.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/registry.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/types.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/bson/unmarshal.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/array.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/bson_arraybuilder.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/bson_documentbuilder.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/bsoncore.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/document.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/document_sequence.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/element.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/tables.go create mode 100644 src/runtime/vendor/go.mongodb.org/mongo-driver/x/bsonx/bsoncore/value.go rename src/runtime/virtcontainers/pkg/firecracker/client/{firecracker_client.go => firecracker_api_client.go} (82%) create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/balloon.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/balloon_stats.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/balloon_stats_update.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/balloon_update.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/firecracker_version.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/full_vm_configuration.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/memory_backend.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/models/mmds_contents_object.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/describe_balloon_config_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/describe_balloon_config_responses.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/describe_balloon_stats_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/describe_balloon_stats_responses.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/get_export_vm_config_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/get_export_vm_config_responses.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/get_firecracker_version_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/get_firecracker_version_responses.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/patch_balloon_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/patch_balloon_responses.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/patch_balloon_stats_interval_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/patch_balloon_stats_interval_responses.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/put_balloon_parameters.go create mode 100644 src/runtime/virtcontainers/pkg/firecracker/client/operations/put_balloon_responses.go diff --git a/src/runtime/go.mod b/src/runtime/go.mod index 36a2f618b981..5b38e1b1a284 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -20,11 +20,11 @@ require ( github.com/docker/go-units v0.4.0 github.com/fsnotify/fsnotify v1.4.9 github.com/go-ini/ini v1.28.2 - github.com/go-openapi/errors v0.18.0 - github.com/go-openapi/runtime v0.18.0 - github.com/go-openapi/strfmt v0.18.0 - github.com/go-openapi/swag v0.19.14 - github.com/go-openapi/validate v0.18.0 + github.com/go-openapi/errors v0.20.2 + github.com/go-openapi/runtime v0.19.21 + github.com/go-openapi/strfmt v0.21.1 + github.com/go-openapi/swag v0.21.1 + github.com/go-openapi/validate v0.22.0 github.com/godbus/dbus/v5 v5.0.6 github.com/gogo/protobuf v1.3.2 github.com/hashicorp/go-multierror v1.1.1 diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 63870b2325a2..252cc46991c0 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -111,8 +111,11 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -375,7 +378,6 @@ github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49P github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -401,53 +403,114 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.17.2 h1:eYp14J1o8TTSCzndHBtsNuckikV1PfZOSnx4BcBeu0c= -github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0 h1:+RnmJ5MQccF7jwWAoMzwOpzJEspZ18ZIWfg9Z2eiXq8= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.17.2/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.17.2/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.17.2 h1:tEXYu6Xc0pevpzzQx5ghrMN9F7IVpN/+u4iD3rkYE5o= -github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.18.0 h1:ddoL4Uo/729XbNAS9UIsG7Oqa8R8l2edBe6Pq/i8AHM= -github.com/go-openapi/runtime v0.18.0/go.mod h1:uI6pHuxWYTy94zZxgcwJkUWa9wbIlhteGfloI10GD4U= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.21 h1:81PiYus9l6fwwS4EwhJD+tQb3EPZBeWfgdAVTfFD25Q= +github.com/go-openapi/runtime v0.19.21/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.17.2/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0 h1:FqqmmVCKn3di+ilU/+1m957T1CnMz3IteVUcV3aGXWA= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1 h1:G6s2t5V5kGCHLVbSdZ/6lI8Wm4OzoPFkc3/cjAsKQrM= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.18.0 h1:PVXYcP1GkTl+XIAJnyJxOmK6CSG5Q1UcvoCvNO++5Kg= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= +github.com/go-openapi/validate v0.22.0 h1:b0QecH6VslW/TxtpKgzpO1SNG7GU2FsaqKdP1E2T50Y= +github.com/go-openapi/validate v0.22.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= @@ -500,6 +563,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -539,7 +603,6 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= @@ -608,6 +671,7 @@ github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbB github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -624,12 +688,16 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -648,11 +716,15 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -680,8 +752,11 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -699,6 +774,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -712,6 +788,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -769,9 +846,10 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -824,6 +902,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -842,6 +922,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -891,6 +972,8 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -914,11 +997,17 @@ github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvV github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -941,6 +1030,13 @@ go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lL go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -993,11 +1089,16 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1048,6 +1149,7 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1073,6 +1175,7 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -1087,6 +1190,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1113,6 +1217,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1130,12 +1235,16 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1203,6 +1312,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1252,11 +1362,16 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1419,6 +1534,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/.gitignore b/src/runtime/vendor/github.com/asaskevich/govalidator/.gitignore new file mode 100644 index 000000000000..8d69a9418aa3 --- /dev/null +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/.gitignore @@ -0,0 +1,15 @@ +bin/ +.idea/ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/.travis.yml b/src/runtime/vendor/github.com/asaskevich/govalidator/.travis.yml index e29f8eef5efd..bb83c6670df6 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/.travis.yml +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/.travis.yml @@ -1,14 +1,12 @@ language: go - +dist: xenial go: - - 1.1 - - 1.2 - - 1.3 - - 1.4 - - 1.5 - - 1.6 - - tip + - '1.10' + - '1.11' + - '1.12' + - '1.13' + - 'tip' -notifications: - email: - - bwatas@gmail.com +script: + - go test -coverpkg=./... -coverprofile=coverage.info -timeout=5s + - bash <(curl -s https://codecov.io/bash) diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/CODE_OF_CONDUCT.md b/src/runtime/vendor/github.com/asaskevich/govalidator/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..4b462b0d81b1 --- /dev/null +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/CODE_OF_CONDUCT.md @@ -0,0 +1,43 @@ +# Contributor Code of Conduct + +This project adheres to [The Code Manifesto](http://codemanifesto.com) +as its guidelines for contributor interactions. + +## The Code Manifesto + +We want to work in an ecosystem that empowers developers to reach their +potential — one that encourages growth and effective collaboration. A space +that is safe for all. + +A space such as this benefits everyone that participates in it. It encourages +new developers to enter our field. It is through discussion and collaboration +that we grow, and through growth that we improve. + +In the effort to create such a place, we hold to these values: + +1. **Discrimination limits us.** This includes discrimination on the basis of + race, gender, sexual orientation, gender identity, age, nationality, + technology and any other arbitrary exclusion of a group of people. +2. **Boundaries honor us.** Your comfort levels are not everyone’s comfort + levels. Remember that, and if brought to your attention, heed it. +3. **We are our biggest assets.** None of us were born masters of our trade. + Each of us has been helped along the way. Return that favor, when and where + you can. +4. **We are resources for the future.** As an extension of #3, share what you + know. Make yourself a resource to help those that come after you. +5. **Respect defines us.** Treat others as you wish to be treated. Make your + discussions, criticisms and debates from a position of respectfulness. Ask + yourself, is it true? Is it necessary? Is it constructive? Anything less is + unacceptable. +6. **Reactions require grace.** Angry responses are valid, but abusive language + and vindictive actions are toxic. When something happens that offends you, + handle it assertively, but be respectful. Escalate reasonably, and try to + allow the offender an opportunity to explain themselves, and possibly + correct the issue. +7. **Opinions are just that: opinions.** Each and every one of us, due to our + background and upbringing, have varying opinions. That is perfectly + acceptable. Remember this: if you respect your own opinions, you should + respect the opinions of others. +8. **To err is human.** You might not intend it, but mistakes do happen and + contribute to build experience. Tolerate honest mistakes, and don't + hesitate to apologize if you make one yourself. diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/CONTRIBUTING.md b/src/runtime/vendor/github.com/asaskevich/govalidator/CONTRIBUTING.md index f0f7e3a8add0..7ed268a1edd9 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/CONTRIBUTING.md +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/CONTRIBUTING.md @@ -11,7 +11,7 @@ If you don't know what to do, there are some features and functions that need to - [ ] Update actual [list of functions](https://github.com/asaskevich/govalidator#list-of-functions) - [ ] Update [list of validators](https://github.com/asaskevich/govalidator#validatestruct-2) that available for `ValidateStruct` and add new - [ ] Implement new validators: `IsFQDN`, `IsIMEI`, `IsPostalCode`, `IsISIN`, `IsISRC` etc -- [ ] Implement [validation by maps](https://github.com/asaskevich/govalidator/issues/224) +- [x] Implement [validation by maps](https://github.com/asaskevich/govalidator/issues/224) - [ ] Implement fuzzing testing - [ ] Implement some struct/map/array utilities - [ ] Implement map/array validation @@ -37,7 +37,7 @@ Anyone can file an expense. If the expense makes sense for the development of th ### Contributors Thank you to all the people who have already contributed to govalidator! - + ### Backers diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/LICENSE b/src/runtime/vendor/github.com/asaskevich/govalidator/LICENSE index 2f9a31fadf67..cacba9102400 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/LICENSE +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Alex Saskevich +Copyright (c) 2014-2020 Alex Saskevich Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/README.md b/src/runtime/vendor/github.com/asaskevich/govalidator/README.md index 40f9a87811b3..39121ea8e37c 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/README.md +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/README.md @@ -1,7 +1,8 @@ govalidator =========== -[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/asaskevich/govalidator?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![GoDoc](https://godoc.org/github.com/asaskevich/govalidator?status.png)](https://godoc.org/github.com/asaskevich/govalidator) [![Coverage Status](https://img.shields.io/coveralls/asaskevich/govalidator.svg)](https://coveralls.io/r/asaskevich/govalidator?branch=master) [![wercker status](https://app.wercker.com/status/1ec990b09ea86c910d5f08b0e02c6043/s "wercker status")](https://app.wercker.com/project/bykey/1ec990b09ea86c910d5f08b0e02c6043) -[![Build Status](https://travis-ci.org/asaskevich/govalidator.svg?branch=master)](https://travis-ci.org/asaskevich/govalidator) [![Go Report Card](https://goreportcard.com/badge/github.com/asaskevich/govalidator)](https://goreportcard.com/report/github.com/asaskevich/govalidator) [![GoSearch](http://go-search.org/badge?id=github.com%2Fasaskevich%2Fgovalidator)](http://go-search.org/view?id=github.com%2Fasaskevich%2Fgovalidator) [![Backers on Open Collective](https://opencollective.com/govalidator/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/govalidator/sponsors/badge.svg)](#sponsors) [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator?ref=badge_shield) +[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/asaskevich/govalidator?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![GoDoc](https://godoc.org/github.com/asaskevich/govalidator?status.png)](https://godoc.org/github.com/asaskevich/govalidator) +[![Build Status](https://travis-ci.org/asaskevich/govalidator.svg?branch=master)](https://travis-ci.org/asaskevich/govalidator) +[![Coverage](https://codecov.io/gh/asaskevich/govalidator/branch/master/graph/badge.svg)](https://codecov.io/gh/asaskevich/govalidator) [![Go Report Card](https://goreportcard.com/badge/github.com/asaskevich/govalidator)](https://goreportcard.com/report/github.com/asaskevich/govalidator) [![GoSearch](http://go-search.org/badge?id=github.com%2Fasaskevich%2Fgovalidator)](http://go-search.org/view?id=github.com%2Fasaskevich%2Fgovalidator) [![Backers on Open Collective](https://opencollective.com/govalidator/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/govalidator/sponsors/badge.svg)](#sponsors) [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator?ref=badge_shield) A package of validators and sanitizers for strings, structs and collections. Based on [validator.js](https://github.com/chriso/validator.js). @@ -13,7 +14,7 @@ Type the following command in your terminal: or you can get specified release of the package with `gopkg.in`: - go get gopkg.in/asaskevich/govalidator.v4 + go get gopkg.in/asaskevich/govalidator.v10 After it the package is ready to use. @@ -83,14 +84,14 @@ This was changed to prevent data races when accessing custom validators. import "github.com/asaskevich/govalidator" // before -govalidator.CustomTypeTagMap["customByteArrayValidator"] = CustomTypeValidator(func(i interface{}, o interface{}) bool { +govalidator.CustomTypeTagMap["customByteArrayValidator"] = func(i interface{}, o interface{}) bool { // ... -}) +} // after -govalidator.CustomTypeTagMap.Set("customByteArrayValidator", CustomTypeValidator(func(i interface{}, o interface{}) bool { +govalidator.CustomTypeTagMap.Set("customByteArrayValidator", func(i interface{}, o interface{}) bool { // ... -})) +}) ``` #### List of functions: @@ -108,23 +109,34 @@ func Filter(array []interface{}, iterator ConditionIterator) []interface{} func Find(array []interface{}, iterator ConditionIterator) interface{} func GetLine(s string, index int) (string, error) func GetLines(s string) []string -func InRange(value, left, right float64) bool +func HasLowerCase(str string) bool +func HasUpperCase(str string) bool +func HasWhitespace(str string) bool +func HasWhitespaceOnly(str string) bool +func InRange(value interface{}, left interface{}, right interface{}) bool +func InRangeFloat32(value, left, right float32) bool +func InRangeFloat64(value, left, right float64) bool +func InRangeInt(value, left, right interface{}) bool func IsASCII(str string) bool func IsAlpha(str string) bool func IsAlphanumeric(str string) bool func IsBase64(str string) bool func IsByteLength(str string, min, max int) bool func IsCIDR(str string) bool +func IsCRC32(str string) bool +func IsCRC32b(str string) bool func IsCreditCard(str string) bool func IsDNSName(str string) bool func IsDataURI(str string) bool func IsDialString(str string) bool func IsDivisibleBy(str, num string) bool func IsEmail(str string) bool +func IsExistingEmail(email string) bool func IsFilePath(str string) (bool, int) func IsFloat(str string) bool func IsFullWidth(str string) bool func IsHalfWidth(str string) bool +func IsHash(str string, algorithm string) bool func IsHexadecimal(str string) bool func IsHexcolor(str string) bool func IsHost(str string) bool @@ -136,22 +148,27 @@ func IsISBN10(str string) bool func IsISBN13(str string) bool func IsISO3166Alpha2(str string) bool func IsISO3166Alpha3(str string) bool +func IsISO4217(str string) bool func IsISO693Alpha2(str string) bool func IsISO693Alpha3b(str string) bool -func IsISO4217(str string) bool func IsIn(str string, params ...string) bool +func IsInRaw(str string, params ...string) bool func IsInt(str string) bool func IsJSON(str string) bool func IsLatitude(str string) bool func IsLongitude(str string) bool func IsLowerCase(str string) bool func IsMAC(str string) bool +func IsMD4(str string) bool +func IsMD5(str string) bool +func IsMagnetURI(str string) bool func IsMongoID(str string) bool func IsMultibyte(str string) bool func IsNatural(value float64) bool func IsNegative(value float64) bool func IsNonNegative(value float64) bool func IsNonPositive(value float64) bool +func IsNotNull(str string) bool func IsNull(str string) bool func IsNumeric(str string) bool func IsPort(str string) bool @@ -162,9 +179,21 @@ func IsRFC3339WithoutZone(str string) bool func IsRGBcolor(str string) bool func IsRequestURI(rawurl string) bool func IsRequestURL(rawurl string) bool +func IsRipeMD128(str string) bool +func IsRipeMD160(str string) bool +func IsRsaPub(str string, params ...string) bool +func IsRsaPublicKey(str string, keylen int) bool +func IsSHA1(str string) bool +func IsSHA256(str string) bool +func IsSHA384(str string) bool +func IsSHA512(str string) bool func IsSSN(str string) bool func IsSemver(str string) bool +func IsTiger128(str string) bool +func IsTiger160(str string) bool +func IsTiger192(str string) bool func IsTime(str string, format string) bool +func IsType(v interface{}, params ...string) bool func IsURL(str string) bool func IsUTFDigit(str string) bool func IsUTFLetter(str string) bool @@ -174,16 +203,20 @@ func IsUUID(str string) bool func IsUUIDv3(str string) bool func IsUUIDv4(str string) bool func IsUUIDv5(str string) bool +func IsUnixTime(str string) bool func IsUpperCase(str string) bool func IsVariableWidth(str string) bool func IsWhole(value float64) bool func LeftTrim(str, chars string) string func Map(array []interface{}, iterator ResultIterator) []interface{} func Matches(str, pattern string) bool +func MaxStringLength(str string, params ...string) bool +func MinStringLength(str string, params ...string) bool func NormalizeEmail(str string) (string, error) func PadBoth(str string, padStr string, padLen int) string func PadLeft(str string, padStr string, padLen int) string func PadRight(str string, padStr string, padLen int) string +func PrependPathToErrors(err error, path string) error func Range(str string, params ...string) bool func RemoveTags(s string) string func ReplacePattern(str, pattern, replace string) string @@ -192,18 +225,21 @@ func RightTrim(str, chars string) string func RuneLength(str string, params ...string) bool func SafeFileName(str string) string func SetFieldsRequiredByDefault(value bool) +func SetNilPtrAllowedByRequired(value bool) func Sign(value float64) float64 func StringLength(str string, params ...string) bool func StringMatches(s string, params ...string) bool func StripLow(str string, keepNewLines bool) string func ToBoolean(str string) (bool, error) func ToFloat(str string) (float64, error) -func ToInt(str string) (int64, error) +func ToInt(value interface{}) (res int64, err error) func ToJSON(obj interface{}) (string, error) func ToString(obj interface{}) string func Trim(str, chars string) string func Truncate(str string, length int, ending string) string +func TruncatingErrorf(str string, args ...interface{}) error func UnderscoreToCamelCase(s string) string +func ValidateMap(inputMap map[string]interface{}, validationMap map[string]interface{}) (bool, error) func ValidateStruct(s interface{}) (bool, error) func WhiteList(str, chars string) string type ConditionIterator @@ -214,6 +250,8 @@ type Errors func (es Errors) Error() string func (es Errors) Errors() []error type ISO3166Entry +type ISO693Entry +type InterfaceParamValidator type Iterator type ParamValidator type ResultIterator @@ -227,6 +265,27 @@ type Validator ```go println(govalidator.IsURL(`http://user@pass:domain.com/path/page`)) ``` +###### IsType +```go +println(govalidator.IsType("Bob", "string")) +println(govalidator.IsType(1, "int")) +i := 1 +println(govalidator.IsType(&i, "*int")) +``` + +IsType can be used through the tag `type` which is essential for map validation: +```go +type User struct { + Name string `valid:"type(string)"` + Age int `valid:"type(int)"` + Meta interface{} `valid:"type(string)"` +} +result, err := govalidator.ValidateStruct(User{"Bob", 20, "meta"}) +if err != nil { + println("error: " + err.Error()) +} +println(result) +``` ###### ToString ```go type User struct { @@ -334,6 +393,13 @@ Validators with parameters "matches(pattern)": StringMatches, "in(string1|string2|...|stringN)": IsIn, "rsapub(keylength)" : IsRsaPub, +"minstringlength(int): MinStringLength, +"maxstringlength(int): MaxStringLength, +``` +Validators with parameters for any type + +```go +"type(type)": IsType, ``` And here is small example of usage: @@ -370,6 +436,41 @@ if err != nil { } println(result) ``` +###### ValidateMap [#2](https://github.com/asaskevich/govalidator/pull/338) +If you want to validate maps, you can use the map to be validated and a validation map that contain the same tags used in ValidateStruct, both maps have to be in the form `map[string]interface{}` + +So here is small example of usage: +```go +var mapTemplate = map[string]interface{}{ + "name":"required,alpha", + "family":"required,alpha", + "email":"required,email", + "cell-phone":"numeric", + "address":map[string]interface{}{ + "line1":"required,alphanum", + "line2":"alphanum", + "postal-code":"numeric", + }, +} + +var inputMap = map[string]interface{}{ + "name":"Bob", + "family":"Smith", + "email":"foo@bar.baz", + "address":map[string]interface{}{ + "line1":"", + "line2":"", + "postal-code":"", + }, +} + +result, err := govalidator.ValidateMap(inputMap, mapTemplate) +if err != nil { + println("error: " + err.Error()) +} +println(result) +``` + ###### WhiteList ```go // Remove all characters from string ignoring characters between "a" and "z" @@ -389,7 +490,7 @@ type StructWithCustomByteArray struct { CustomMinLength int `valid:"-"` } -govalidator.CustomTypeTagMap.Set("customByteArrayValidator", CustomTypeValidator(func(i interface{}, context interface{}) bool { +govalidator.CustomTypeTagMap.Set("customByteArrayValidator", func(i interface{}, context interface{}) bool { switch v := context.(type) { // you can type switch on the context interface being validated case StructWithCustomByteArray: // you can check and validate against some other field in the context, @@ -409,14 +510,25 @@ govalidator.CustomTypeTagMap.Set("customByteArrayValidator", CustomTypeValidator } } return false -})) -govalidator.CustomTypeTagMap.Set("customMinLengthValidator", CustomTypeValidator(func(i interface{}, context interface{}) bool { +}) +govalidator.CustomTypeTagMap.Set("customMinLengthValidator", func(i interface{}, context interface{}) bool { switch v := context.(type) { // this validates a field against the value in another field, i.e. dependent validation case StructWithCustomByteArray: return len(v.ID) >= v.CustomMinLength } return false -})) +}) +``` + +###### Loop over Error() +By default .Error() returns all errors in a single String. To access each error you can do this: +```go + if err != nil { + errs := err.(govalidator.Errors).Errors() + for _, e := range errs { + fmt.Println(e.Error()) + } + } ``` ###### Custom error messages @@ -445,7 +557,7 @@ If you don't know what to do, there are some features and functions that need to - [ ] Update actual [list of functions](https://github.com/asaskevich/govalidator#list-of-functions) - [ ] Update [list of validators](https://github.com/asaskevich/govalidator#validatestruct-2) that available for `ValidateStruct` and add new - [ ] Implement new validators: `IsFQDN`, `IsIMEI`, `IsPostalCode`, `IsISIN`, `IsISRC` etc -- [ ] Implement [validation by maps](https://github.com/asaskevich/govalidator/issues/224) +- [x] Implement [validation by maps](https://github.com/asaskevich/govalidator/issues/224) - [ ] Implement fuzzing testing - [ ] Implement some struct/map/array utilities - [ ] Implement map/array validation @@ -475,7 +587,7 @@ This project exists thanks to all the people who contribute. [[Contribute](CONTR * [Matt Sanford](https://github.com/mzsanford) * [Simon ccl1115](https://github.com/ccl1115) - + ### Backers @@ -504,4 +616,4 @@ Support this project by becoming a sponsor. Your logo will show up here with a l ## License -[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator?ref=badge_large) \ No newline at end of file +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fasaskevich%2Fgovalidator?ref=badge_large) diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/arrays.go b/src/runtime/vendor/github.com/asaskevich/govalidator/arrays.go index 5bace2654d3b..3e1da7cb480e 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/arrays.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/arrays.go @@ -9,6 +9,35 @@ type ResultIterator func(interface{}, int) interface{} // ConditionIterator is the function that accepts element of slice/array and its index and returns boolean type ConditionIterator func(interface{}, int) bool +// ReduceIterator is the function that accepts two element of slice/array and returns result of merging those values +type ReduceIterator func(interface{}, interface{}) interface{} + +// Some validates that any item of array corresponds to ConditionIterator. Returns boolean. +func Some(array []interface{}, iterator ConditionIterator) bool { + res := false + for index, data := range array { + res = res || iterator(data, index) + } + return res +} + +// Every validates that every item of array corresponds to ConditionIterator. Returns boolean. +func Every(array []interface{}, iterator ConditionIterator) bool { + res := true + for index, data := range array { + res = res && iterator(data, index) + } + return res +} + +// Reduce boils down a list of values into a single value by ReduceIterator +func Reduce(array []interface{}, iterator ReduceIterator, initialValue interface{}) interface{} { + for _, data := range array { + initialValue = iterator(initialValue, data) + } + return initialValue +} + // Each iterates over the slice and apply Iterator to every item func Each(array []interface{}, iterator Iterator) { for index, data := range array { diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/converter.go b/src/runtime/vendor/github.com/asaskevich/govalidator/converter.go index cf1e5d569ba0..d68e990fc256 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/converter.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/converter.go @@ -10,7 +10,7 @@ import ( // ToString convert the input to a string. func ToString(obj interface{}) string { res := fmt.Sprintf("%v", obj) - return string(res) + return res } // ToJSON convert the input to a valid JSON string @@ -23,12 +23,27 @@ func ToJSON(obj interface{}) (string, error) { } // ToFloat convert the input string to a float, or 0.0 if the input is not a float. -func ToFloat(str string) (float64, error) { - res, err := strconv.ParseFloat(str, 64) - if err != nil { - res = 0.0 +func ToFloat(value interface{}) (res float64, err error) { + val := reflect.ValueOf(value) + + switch value.(type) { + case int, int8, int16, int32, int64: + res = float64(val.Int()) + case uint, uint8, uint16, uint32, uint64: + res = float64(val.Uint()) + case float32, float64: + res = val.Float() + case string: + res, err = strconv.ParseFloat(val.String(), 64) + if err != nil { + res = 0 + } + default: + err = fmt.Errorf("ToInt: unknown interface type %T", value) + res = 0 } - return res, err + + return } // ToInt convert the input string or any int type to an integer type 64, or 0 if the input is not an integer. @@ -40,6 +55,8 @@ func ToInt(value interface{}) (res int64, err error) { res = val.Int() case uint, uint8, uint16, uint32, uint64: res = int64(val.Uint()) + case float32, float64: + res = int64(val.Float()) case string: if IsInt(val.String()) { res, err = strconv.ParseInt(val.String(), 0, 64) @@ -47,11 +64,11 @@ func ToInt(value interface{}) (res int64, err error) { res = 0 } } else { - err = fmt.Errorf("math: square root of negative number %g", value) + err = fmt.Errorf("ToInt: invalid numeric format %g", value) res = 0 } default: - err = fmt.Errorf("math: square root of negative number %g", value) + err = fmt.Errorf("ToInt: unknown interface type %T", value) res = 0 } diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/doc.go b/src/runtime/vendor/github.com/asaskevich/govalidator/doc.go new file mode 100644 index 000000000000..55dce62dc8c3 --- /dev/null +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/doc.go @@ -0,0 +1,3 @@ +package govalidator + +// A package of validators and sanitizers for strings, structures and collections. diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/error.go b/src/runtime/vendor/github.com/asaskevich/govalidator/error.go index 655b750cb8f6..1da2336f47ee 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/error.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/error.go @@ -1,6 +1,9 @@ package govalidator -import "strings" +import ( + "sort" + "strings" +) // Errors is an array of multiple errors and conforms to the error interface. type Errors []error @@ -15,6 +18,7 @@ func (es Errors) Error() string { for _, e := range es { errs = append(errs, e.Error()) } + sort.Strings(errs) return strings.Join(errs, ";") } diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/go.mod b/src/runtime/vendor/github.com/asaskevich/govalidator/go.mod new file mode 100644 index 000000000000..42d5b1f63857 --- /dev/null +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/go.mod @@ -0,0 +1,3 @@ +module github.com/asaskevich/govalidator + +go 1.13 diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/numerics.go b/src/runtime/vendor/github.com/asaskevich/govalidator/numerics.go index 7e6c652e140c..5041d9e86844 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/numerics.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/numerics.go @@ -2,7 +2,6 @@ package govalidator import ( "math" - "reflect" ) // Abs returns absolute value of number @@ -41,7 +40,7 @@ func IsNonPositive(value float64) bool { return value <= 0 } -// InRange returns true if value lies between left and right border +// InRangeInt returns true if value lies between left and right border func InRangeInt(value, left, right interface{}) bool { value64, _ := ToInt(value) left64, _ := ToInt(left) @@ -52,7 +51,7 @@ func InRangeInt(value, left, right interface{}) bool { return value64 >= left64 && value64 <= right64 } -// InRange returns true if value lies between left and right border +// InRangeFloat32 returns true if value lies between left and right border func InRangeFloat32(value, left, right float32) bool { if left > right { left, right = right, left @@ -60,7 +59,7 @@ func InRangeFloat32(value, left, right float32) bool { return value >= left && value <= right } -// InRange returns true if value lies between left and right border +// InRangeFloat64 returns true if value lies between left and right border func InRangeFloat64(value, left, right float64) bool { if left > right { left, right = right, left @@ -68,20 +67,24 @@ func InRangeFloat64(value, left, right float64) bool { return value >= left && value <= right } -// InRange returns true if value lies between left and right border, generic type to handle int, float32 or float64, all types must the same type +// InRange returns true if value lies between left and right border, generic type to handle int, float32, float64 and string. +// All types must the same type. +// False if value doesn't lie in range or if it incompatible or not comparable func InRange(value interface{}, left interface{}, right interface{}) bool { - - reflectValue := reflect.TypeOf(value).Kind() - reflectLeft := reflect.TypeOf(left).Kind() - reflectRight := reflect.TypeOf(right).Kind() - - if reflectValue == reflect.Int && reflectLeft == reflect.Int && reflectRight == reflect.Int { - return InRangeInt(value.(int), left.(int), right.(int)) - } else if reflectValue == reflect.Float32 && reflectLeft == reflect.Float32 && reflectRight == reflect.Float32 { - return InRangeFloat32(value.(float32), left.(float32), right.(float32)) - } else if reflectValue == reflect.Float64 && reflectLeft == reflect.Float64 && reflectRight == reflect.Float64 { - return InRangeFloat64(value.(float64), left.(float64), right.(float64)) - } else { + switch value.(type) { + case int: + intValue, _ := ToInt(value) + intLeft, _ := ToInt(left) + intRight, _ := ToInt(right) + return InRangeInt(intValue, intLeft, intRight) + case float32, float64: + intValue, _ := ToFloat(value) + intLeft, _ := ToFloat(left) + intRight, _ := ToFloat(right) + return InRangeFloat64(intValue, intLeft, intRight) + case string: + return value.(string) >= left.(string) && value.(string) <= right.(string) + default: return false } } diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/patterns.go b/src/runtime/vendor/github.com/asaskevich/govalidator/patterns.go index 61a05d438e18..106ed94f80ad 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/patterns.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/patterns.go @@ -4,49 +4,52 @@ import "regexp" // Basic regular expressions for validating strings const ( - Email string = "^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$" - CreditCard string = "^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11})$" - ISBN10 string = "^(?:[0-9]{9}X|[0-9]{10})$" - ISBN13 string = "^(?:[0-9]{13})$" - UUID3 string = "^[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$" - UUID4 string = "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" - UUID5 string = "^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" - UUID string = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" - Alpha string = "^[a-zA-Z]+$" - Alphanumeric string = "^[a-zA-Z0-9]+$" - Numeric string = "^[0-9]+$" - Int string = "^(?:[-+]?(?:0|[1-9][0-9]*))$" - Float string = "^(?:[-+]?(?:[0-9]+))?(?:\\.[0-9]*)?(?:[eE][\\+\\-]?(?:[0-9]+))?$" - Hexadecimal string = "^[0-9a-fA-F]+$" - Hexcolor string = "^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$" - RGBcolor string = "^rgb\\(\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*,\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*,\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*\\)$" - ASCII string = "^[\x00-\x7F]+$" - Multibyte string = "[^\x00-\x7F]" - FullWidth string = "[^\u0020-\u007E\uFF61-\uFF9F\uFFA0-\uFFDC\uFFE8-\uFFEE0-9a-zA-Z]" - HalfWidth string = "[\u0020-\u007E\uFF61-\uFF9F\uFFA0-\uFFDC\uFFE8-\uFFEE0-9a-zA-Z]" - Base64 string = "^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=|[A-Za-z0-9+\\/]{4})$" - PrintableASCII string = "^[\x20-\x7E]+$" - DataURI string = "^data:.+\\/(.+);base64$" - Latitude string = "^[-+]?([1-8]?\\d(\\.\\d+)?|90(\\.0+)?)$" - Longitude string = "^[-+]?(180(\\.0+)?|((1[0-7]\\d)|([1-9]?\\d))(\\.\\d+)?)$" - DNSName string = `^([a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62}){1}(\.[a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62})*[\._]?$` - IP string = `(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))` - URLSchema string = `((ftp|tcp|udp|wss?|https?):\/\/)` - URLUsername string = `(\S+(:\S*)?@)` - URLPath string = `((\/|\?|#)[^\s]*)` - URLPort string = `(:(\d{1,5}))` - URLIP string = `([1-9]\d?|1\d\d|2[01]\d|22[0-3])(\.(1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-4]))` - URLSubdomain string = `((www\.)|([a-zA-Z0-9]+([-_\.]?[a-zA-Z0-9])*[a-zA-Z0-9]\.[a-zA-Z0-9]+))` - URL string = `^` + URLSchema + `?` + URLUsername + `?` + `((` + URLIP + `|(\[` + IP + `\])|(([a-zA-Z0-9]([a-zA-Z0-9-_]+)?[a-zA-Z0-9]([-\.][a-zA-Z0-9]+)*)|(` + URLSubdomain + `?))?(([a-zA-Z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-zA-Z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-zA-Z\x{00a1}-\x{ffff}]{1,}))?))\.?` + URLPort + `?` + URLPath + `?$` - SSN string = `^\d{3}[- ]?\d{2}[- ]?\d{4}$` - WinPath string = `^[a-zA-Z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*$` - UnixPath string = `^(/[^/\x00]*)+/?$` - Semver string = "^v?(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(-(0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\\+[0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*)?$" - tagName string = "valid" - hasLowerCase string = ".*[[:lower:]]" - hasUpperCase string = ".*[[:upper:]]" - hasWhitespace string = ".*[[:space:]]" - hasWhitespaceOnly string = "^[[:space:]]+$" + Email string = "^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$" + CreditCard string = "^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|(222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|(?:2131|1800|35\\d{3})\\d{11}|6[27][0-9]{14})$" + ISBN10 string = "^(?:[0-9]{9}X|[0-9]{10})$" + ISBN13 string = "^(?:[0-9]{13})$" + UUID3 string = "^[0-9a-f]{8}-[0-9a-f]{4}-3[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$" + UUID4 string = "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + UUID5 string = "^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + UUID string = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + Alpha string = "^[a-zA-Z]+$" + Alphanumeric string = "^[a-zA-Z0-9]+$" + Numeric string = "^[0-9]+$" + Int string = "^(?:[-+]?(?:0|[1-9][0-9]*))$" + Float string = "^(?:[-+]?(?:[0-9]+))?(?:\\.[0-9]*)?(?:[eE][\\+\\-]?(?:[0-9]+))?$" + Hexadecimal string = "^[0-9a-fA-F]+$" + Hexcolor string = "^#?([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$" + RGBcolor string = "^rgb\\(\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*,\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*,\\s*(0|[1-9]\\d?|1\\d\\d?|2[0-4]\\d|25[0-5])\\s*\\)$" + ASCII string = "^[\x00-\x7F]+$" + Multibyte string = "[^\x00-\x7F]" + FullWidth string = "[^\u0020-\u007E\uFF61-\uFF9F\uFFA0-\uFFDC\uFFE8-\uFFEE0-9a-zA-Z]" + HalfWidth string = "[\u0020-\u007E\uFF61-\uFF9F\uFFA0-\uFFDC\uFFE8-\uFFEE0-9a-zA-Z]" + Base64 string = "^(?:[A-Za-z0-9+\\/]{4})*(?:[A-Za-z0-9+\\/]{2}==|[A-Za-z0-9+\\/]{3}=|[A-Za-z0-9+\\/]{4})$" + PrintableASCII string = "^[\x20-\x7E]+$" + DataURI string = "^data:.+\\/(.+);base64$" + MagnetURI string = "^magnet:\\?xt=urn:[a-zA-Z0-9]+:[a-zA-Z0-9]{32,40}&dn=.+&tr=.+$" + Latitude string = "^[-+]?([1-8]?\\d(\\.\\d+)?|90(\\.0+)?)$" + Longitude string = "^[-+]?(180(\\.0+)?|((1[0-7]\\d)|([1-9]?\\d))(\\.\\d+)?)$" + DNSName string = `^([a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62}){1}(\.[a-zA-Z0-9_]{1}[a-zA-Z0-9_-]{0,62})*[\._]?$` + IP string = `(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))` + URLSchema string = `((ftp|tcp|udp|wss?|https?):\/\/)` + URLUsername string = `(\S+(:\S*)?@)` + URLPath string = `((\/|\?|#)[^\s]*)` + URLPort string = `(:(\d{1,5}))` + URLIP string = `([1-9]\d?|1\d\d|2[01]\d|22[0-3]|24\d|25[0-5])(\.(\d{1,2}|1\d\d|2[0-4]\d|25[0-5])){2}(?:\.([0-9]\d?|1\d\d|2[0-4]\d|25[0-5]))` + URLSubdomain string = `((www\.)|([a-zA-Z0-9]+([-_\.]?[a-zA-Z0-9])*[a-zA-Z0-9]\.[a-zA-Z0-9]+))` + URL = `^` + URLSchema + `?` + URLUsername + `?` + `((` + URLIP + `|(\[` + IP + `\])|(([a-zA-Z0-9]([a-zA-Z0-9-_]+)?[a-zA-Z0-9]([-\.][a-zA-Z0-9]+)*)|(` + URLSubdomain + `?))?(([a-zA-Z\x{00a1}-\x{ffff}0-9]+-?-?)*[a-zA-Z\x{00a1}-\x{ffff}0-9]+)(?:\.([a-zA-Z\x{00a1}-\x{ffff}]{1,}))?))\.?` + URLPort + `?` + URLPath + `?$` + SSN string = `^\d{3}[- ]?\d{2}[- ]?\d{4}$` + WinPath string = `^[a-zA-Z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*$` + UnixPath string = `^(/[^/\x00]*)+/?$` + Semver string = "^v?(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(-(0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\\+[0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*)?$" + tagName string = "valid" + hasLowerCase string = ".*[[:lower:]]" + hasUpperCase string = ".*[[:upper:]]" + hasWhitespace string = ".*[[:space:]]" + hasWhitespaceOnly string = "^[[:space:]]+$" + IMEI string = "^[0-9a-f]{14}$|^\\d{15}$|^\\d{18}$" + IMSI string = "^\\d{14,15}$" ) // Used by IsFilePath func @@ -60,42 +63,45 @@ const ( ) var ( - userRegexp = regexp.MustCompile("^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~.-]+$") - hostRegexp = regexp.MustCompile("^[^\\s]+\\.[^\\s]+$") - userDotRegexp = regexp.MustCompile("(^[.]{1})|([.]{1}$)|([.]{2,})") - rxEmail = regexp.MustCompile(Email) - rxCreditCard = regexp.MustCompile(CreditCard) - rxISBN10 = regexp.MustCompile(ISBN10) - rxISBN13 = regexp.MustCompile(ISBN13) - rxUUID3 = regexp.MustCompile(UUID3) - rxUUID4 = regexp.MustCompile(UUID4) - rxUUID5 = regexp.MustCompile(UUID5) - rxUUID = regexp.MustCompile(UUID) - rxAlpha = regexp.MustCompile(Alpha) - rxAlphanumeric = regexp.MustCompile(Alphanumeric) - rxNumeric = regexp.MustCompile(Numeric) - rxInt = regexp.MustCompile(Int) - rxFloat = regexp.MustCompile(Float) - rxHexadecimal = regexp.MustCompile(Hexadecimal) - rxHexcolor = regexp.MustCompile(Hexcolor) - rxRGBcolor = regexp.MustCompile(RGBcolor) - rxASCII = regexp.MustCompile(ASCII) - rxPrintableASCII = regexp.MustCompile(PrintableASCII) - rxMultibyte = regexp.MustCompile(Multibyte) - rxFullWidth = regexp.MustCompile(FullWidth) - rxHalfWidth = regexp.MustCompile(HalfWidth) - rxBase64 = regexp.MustCompile(Base64) - rxDataURI = regexp.MustCompile(DataURI) - rxLatitude = regexp.MustCompile(Latitude) - rxLongitude = regexp.MustCompile(Longitude) - rxDNSName = regexp.MustCompile(DNSName) - rxURL = regexp.MustCompile(URL) - rxSSN = regexp.MustCompile(SSN) - rxWinPath = regexp.MustCompile(WinPath) - rxUnixPath = regexp.MustCompile(UnixPath) - rxSemver = regexp.MustCompile(Semver) - rxHasLowerCase = regexp.MustCompile(hasLowerCase) - rxHasUpperCase = regexp.MustCompile(hasUpperCase) - rxHasWhitespace = regexp.MustCompile(hasWhitespace) - rxHasWhitespaceOnly = regexp.MustCompile(hasWhitespaceOnly) + userRegexp = regexp.MustCompile("^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~.-]+$") + hostRegexp = regexp.MustCompile("^[^\\s]+\\.[^\\s]+$") + userDotRegexp = regexp.MustCompile("(^[.]{1})|([.]{1}$)|([.]{2,})") + rxEmail = regexp.MustCompile(Email) + rxCreditCard = regexp.MustCompile(CreditCard) + rxISBN10 = regexp.MustCompile(ISBN10) + rxISBN13 = regexp.MustCompile(ISBN13) + rxUUID3 = regexp.MustCompile(UUID3) + rxUUID4 = regexp.MustCompile(UUID4) + rxUUID5 = regexp.MustCompile(UUID5) + rxUUID = regexp.MustCompile(UUID) + rxAlpha = regexp.MustCompile(Alpha) + rxAlphanumeric = regexp.MustCompile(Alphanumeric) + rxNumeric = regexp.MustCompile(Numeric) + rxInt = regexp.MustCompile(Int) + rxFloat = regexp.MustCompile(Float) + rxHexadecimal = regexp.MustCompile(Hexadecimal) + rxHexcolor = regexp.MustCompile(Hexcolor) + rxRGBcolor = regexp.MustCompile(RGBcolor) + rxASCII = regexp.MustCompile(ASCII) + rxPrintableASCII = regexp.MustCompile(PrintableASCII) + rxMultibyte = regexp.MustCompile(Multibyte) + rxFullWidth = regexp.MustCompile(FullWidth) + rxHalfWidth = regexp.MustCompile(HalfWidth) + rxBase64 = regexp.MustCompile(Base64) + rxDataURI = regexp.MustCompile(DataURI) + rxMagnetURI = regexp.MustCompile(MagnetURI) + rxLatitude = regexp.MustCompile(Latitude) + rxLongitude = regexp.MustCompile(Longitude) + rxDNSName = regexp.MustCompile(DNSName) + rxURL = regexp.MustCompile(URL) + rxSSN = regexp.MustCompile(SSN) + rxWinPath = regexp.MustCompile(WinPath) + rxUnixPath = regexp.MustCompile(UnixPath) + rxSemver = regexp.MustCompile(Semver) + rxHasLowerCase = regexp.MustCompile(hasLowerCase) + rxHasUpperCase = regexp.MustCompile(hasUpperCase) + rxHasWhitespace = regexp.MustCompile(hasWhitespace) + rxHasWhitespaceOnly = regexp.MustCompile(hasWhitespaceOnly) + rxIMEI = regexp.MustCompile(IMEI) + rxIMSI = regexp.MustCompile(IMSI) ) diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/types.go b/src/runtime/vendor/github.com/asaskevich/govalidator/types.go index 4f7e9274ade0..54218bf05a2f 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/types.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/types.go @@ -14,8 +14,11 @@ type Validator func(str string) bool // The second parameter should be the context (in the case of validating a struct: the whole object being validated). type CustomTypeValidator func(i interface{}, o interface{}) bool -// ParamValidator is a wrapper for validator functions that accepts additional parameters. +// ParamValidator is a wrapper for validator functions that accept additional parameters. type ParamValidator func(str string, params ...string) bool + +// InterfaceParamValidator is a wrapper for functions that accept variants parameters for an interface value +type InterfaceParamValidator func(in interface{}, params ...string) bool type tagOptionsMap map[string]tagOption func (t tagOptionsMap) orderedKeys() []string { @@ -46,26 +49,40 @@ type UnsupportedTypeError struct { // It implements the methods to sort by string. type stringValues []reflect.Value +// InterfaceParamTagMap is a map of functions accept variants parameters for an interface value +var InterfaceParamTagMap = map[string]InterfaceParamValidator{ + "type": IsType, +} + +// InterfaceParamTagRegexMap maps interface param tags to their respective regexes. +var InterfaceParamTagRegexMap = map[string]*regexp.Regexp{ + "type": regexp.MustCompile(`^type\((.*)\)$`), +} + // ParamTagMap is a map of functions accept variants parameters var ParamTagMap = map[string]ParamValidator{ - "length": ByteLength, - "range": Range, - "runelength": RuneLength, - "stringlength": StringLength, - "matches": StringMatches, - "in": isInRaw, - "rsapub": IsRsaPub, + "length": ByteLength, + "range": Range, + "runelength": RuneLength, + "stringlength": StringLength, + "matches": StringMatches, + "in": IsInRaw, + "rsapub": IsRsaPub, + "minstringlength": MinStringLength, + "maxstringlength": MaxStringLength, } // ParamTagRegexMap maps param tags to their respective regexes. var ParamTagRegexMap = map[string]*regexp.Regexp{ - "range": regexp.MustCompile("^range\\((\\d+)\\|(\\d+)\\)$"), - "length": regexp.MustCompile("^length\\((\\d+)\\|(\\d+)\\)$"), - "runelength": regexp.MustCompile("^runelength\\((\\d+)\\|(\\d+)\\)$"), - "stringlength": regexp.MustCompile("^stringlength\\((\\d+)\\|(\\d+)\\)$"), - "in": regexp.MustCompile(`^in\((.*)\)`), - "matches": regexp.MustCompile(`^matches\((.+)\)$`), - "rsapub": regexp.MustCompile("^rsapub\\((\\d+)\\)$"), + "range": regexp.MustCompile("^range\\((\\d+)\\|(\\d+)\\)$"), + "length": regexp.MustCompile("^length\\((\\d+)\\|(\\d+)\\)$"), + "runelength": regexp.MustCompile("^runelength\\((\\d+)\\|(\\d+)\\)$"), + "stringlength": regexp.MustCompile("^stringlength\\((\\d+)\\|(\\d+)\\)$"), + "in": regexp.MustCompile(`^in\((.*)\)`), + "matches": regexp.MustCompile(`^matches\((.+)\)$`), + "rsapub": regexp.MustCompile("^rsapub\\((\\d+)\\)$"), + "minstringlength": regexp.MustCompile("^minstringlength\\((\\d+)\\)$"), + "maxstringlength": regexp.MustCompile("^maxstringlength\\((\\d+)\\)$"), } type customTypeTagMap struct { @@ -114,6 +131,7 @@ var TagMap = map[string]Validator{ "int": IsInt, "float": IsFloat, "null": IsNull, + "notnull": IsNotNull, "uuid": IsUUID, "uuidv3": IsUUIDv3, "uuidv4": IsUUIDv4, @@ -146,6 +164,7 @@ var TagMap = map[string]Validator{ "ISO3166Alpha2": IsISO3166Alpha2, "ISO3166Alpha3": IsISO3166Alpha3, "ISO4217": IsISO4217, + "IMEI": IsIMEI, } // ISO3166Entry stores country codes @@ -430,10 +449,10 @@ var ISO4217List = []string{ "PAB", "PEN", "PGK", "PHP", "PKR", "PLN", "PYG", "QAR", "RON", "RSD", "RUB", "RWF", - "SAR", "SBD", "SCR", "SDG", "SEK", "SGD", "SHP", "SLL", "SOS", "SRD", "SSP", "STD", "SVC", "SYP", "SZL", + "SAR", "SBD", "SCR", "SDG", "SEK", "SGD", "SHP", "SLL", "SOS", "SRD", "SSP", "STD", "STN", "SVC", "SYP", "SZL", "THB", "TJS", "TMT", "TND", "TOP", "TRY", "TTD", "TWD", "TZS", - "UAH", "UGX", "USD", "USN", "UYI", "UYU", "UZS", - "VEF", "VND", "VUV", + "UAH", "UGX", "USD", "USN", "UYI", "UYU", "UYW", "UZS", + "VEF", "VES", "VND", "VUV", "WST", "XAF", "XAG", "XAU", "XBA", "XBB", "XBC", "XBD", "XCD", "XDR", "XOF", "XPD", "XPF", "XPT", "XSU", "XTS", "XUA", "XXX", "YER", diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/utils.go b/src/runtime/vendor/github.com/asaskevich/govalidator/utils.go index a0b706a743ce..f4c30f824a22 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/utils.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/utils.go @@ -12,20 +12,20 @@ import ( "unicode/utf8" ) -// Contains check if the string contains the substring. +// Contains checks if the string contains the substring. func Contains(str, substring string) bool { return strings.Contains(str, substring) } -// Matches check if string matches the pattern (pattern is regular expression) +// Matches checks if string matches the pattern (pattern is regular expression) // In case of error return false func Matches(str, pattern string) bool { match, _ := regexp.MatchString(pattern, str) return match } -// LeftTrim trim characters from the left-side of the input. -// If second argument is empty, it's will be remove leading spaces. +// LeftTrim trims characters from the left side of the input. +// If second argument is empty, it will remove leading spaces. func LeftTrim(str, chars string) string { if chars == "" { return strings.TrimLeftFunc(str, unicode.IsSpace) @@ -34,8 +34,8 @@ func LeftTrim(str, chars string) string { return r.ReplaceAllString(str, "") } -// RightTrim trim characters from the right-side of the input. -// If second argument is empty, it's will be remove spaces. +// RightTrim trims characters from the right side of the input. +// If second argument is empty, it will remove trailing spaces. func RightTrim(str, chars string) string { if chars == "" { return strings.TrimRightFunc(str, unicode.IsSpace) @@ -44,27 +44,27 @@ func RightTrim(str, chars string) string { return r.ReplaceAllString(str, "") } -// Trim trim characters from both sides of the input. -// If second argument is empty, it's will be remove spaces. +// Trim trims characters from both sides of the input. +// If second argument is empty, it will remove spaces. func Trim(str, chars string) string { return LeftTrim(RightTrim(str, chars), chars) } -// WhiteList remove characters that do not appear in the whitelist. +// WhiteList removes characters that do not appear in the whitelist. func WhiteList(str, chars string) string { pattern := "[^" + chars + "]+" r, _ := regexp.Compile(pattern) return r.ReplaceAllString(str, "") } -// BlackList remove characters that appear in the blacklist. +// BlackList removes characters that appear in the blacklist. func BlackList(str, chars string) string { pattern := "[" + chars + "]+" r, _ := regexp.Compile(pattern) return r.ReplaceAllString(str, "") } -// StripLow remove characters with a numerical value < 32 and 127, mostly control characters. +// StripLow removes characters with a numerical value < 32 and 127, mostly control characters. // If keep_new_lines is true, newline characters are preserved (\n and \r, hex 0xA and 0xD). func StripLow(str string, keepNewLines bool) string { chars := "" @@ -76,13 +76,13 @@ func StripLow(str string, keepNewLines bool) string { return BlackList(str, chars) } -// ReplacePattern replace regular expression pattern in string +// ReplacePattern replaces regular expression pattern in string func ReplacePattern(str, pattern, replace string) string { r, _ := regexp.Compile(pattern) return r.ReplaceAllString(str, replace) } -// Escape replace <, >, & and " with HTML entities. +// Escape replaces <, >, & and " with HTML entities. var Escape = html.EscapeString func addSegment(inrune, segment []rune) []rune { @@ -120,7 +120,7 @@ func CamelCaseToUnderscore(str string) string { return string(output) } -// Reverse return reversed string +// Reverse returns reversed string func Reverse(s string) string { r := []rune(s) for i, j := 0, len(r)-1; i < j; i, j = i+1, j-1 { @@ -129,12 +129,12 @@ func Reverse(s string) string { return string(r) } -// GetLines split string by "\n" and return array of lines +// GetLines splits string by "\n" and return array of lines func GetLines(s string) []string { return strings.Split(s, "\n") } -// GetLine return specified line of multiline string +// GetLine returns specified line of multiline string func GetLine(s string, index int) (string, error) { lines := GetLines(s) if index < 0 || index >= len(lines) { @@ -143,12 +143,12 @@ func GetLine(s string, index int) (string, error) { return lines[index], nil } -// RemoveTags remove all tags from HTML string +// RemoveTags removes all tags from HTML string func RemoveTags(s string) string { return ReplacePattern(s, "<[^>]*>", "") } -// SafeFileName return safe string that can be used in file names +// SafeFileName returns safe string that can be used in file names func SafeFileName(str string) string { name := strings.ToLower(str) name = path.Clean(path.Base(name)) @@ -210,23 +210,23 @@ func Truncate(str string, length int, ending string) string { return str } -// PadLeft pad left side of string if size of string is less then indicated pad length +// PadLeft pads left side of a string if size of string is less then indicated pad length func PadLeft(str string, padStr string, padLen int) string { return buildPadStr(str, padStr, padLen, true, false) } -// PadRight pad right side of string if size of string is less then indicated pad length +// PadRight pads right side of a string if size of string is less then indicated pad length func PadRight(str string, padStr string, padLen int) string { return buildPadStr(str, padStr, padLen, false, true) } -// PadBoth pad sides of string if size of string is less then indicated pad length +// PadBoth pads both sides of a string if size of string is less then indicated pad length func PadBoth(str string, padStr string, padLen int) string { return buildPadStr(str, padStr, padLen, true, true) } -// PadString either left, right or both sides, not the padding string can be unicode and more then one -// character +// PadString either left, right or both sides. +// Note that padding string can be unicode and more then one character func buildPadStr(str string, padStr string, padLen int, padLeft bool, padRight bool) string { // When padded length is less then the current string size diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/validator.go b/src/runtime/vendor/github.com/asaskevich/govalidator/validator.go index b18bbcb4c99f..5c918fc4bc7d 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/validator.go +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/validator.go @@ -32,7 +32,7 @@ var ( const maxURLRuneCount = 2083 const minURLRuneCount = 3 -const RF3339WithoutZone = "2006-01-02T15:04:05" +const rfc3339WithoutZone = "2006-01-02T15:04:05" // SetFieldsRequiredByDefault causes validation to fail when struct fields // do not include validations or are not explicitly marked as exempt (using `valid:"-"` or `valid:"email,optional"`). @@ -63,13 +63,13 @@ func SetNilPtrAllowedByRequired(value bool) { nilPtrAllowedByRequired = value } -// IsEmail check if the string is an email. +// IsEmail checks if the string is an email. func IsEmail(str string) bool { // TODO uppercase letters are not supported return rxEmail.MatchString(str) } -// IsExistingEmail check if the string is an email of existing domain +// IsExistingEmail checks if the string is an email of existing domain func IsExistingEmail(email string) bool { if len(email) < 6 || len(email) > 254 { @@ -84,13 +84,13 @@ func IsExistingEmail(email string) bool { if len(user) > 64 { return false } - if userDotRegexp.MatchString(user) || !userRegexp.MatchString(user) || !hostRegexp.MatchString(host) { - return false - } switch host { case "localhost", "example.com": return true } + if userDotRegexp.MatchString(user) || !userRegexp.MatchString(user) || !hostRegexp.MatchString(host) { + return false + } if _, err := net.LookupMX(host); err != nil { if _, err := net.LookupIP(host); err != nil { return false @@ -100,7 +100,7 @@ func IsExistingEmail(email string) bool { return true } -// IsURL check if the string is an URL. +// IsURL checks if the string is an URL. func IsURL(str string) bool { if str == "" || utf8.RuneCountInString(str) >= maxURLRuneCount || len(str) <= minURLRuneCount || strings.HasPrefix(str, ".") { return false @@ -124,7 +124,7 @@ func IsURL(str string) bool { return rxURL.MatchString(str) } -// IsRequestURL check if the string rawurl, assuming +// IsRequestURL checks if the string rawurl, assuming // it was received in an HTTP request, is a valid // URL confirm to RFC 3986 func IsRequestURL(rawurl string) bool { @@ -138,7 +138,7 @@ func IsRequestURL(rawurl string) bool { return true } -// IsRequestURI check if the string rawurl, assuming +// IsRequestURI checks if the string rawurl, assuming // it was received in an HTTP request, is an // absolute URI or an absolute path. func IsRequestURI(rawurl string) bool { @@ -146,7 +146,7 @@ func IsRequestURI(rawurl string) bool { return err == nil } -// IsAlpha check if the string contains only letters (a-zA-Z). Empty string is valid. +// IsAlpha checks if the string contains only letters (a-zA-Z). Empty string is valid. func IsAlpha(str string) bool { if IsNull(str) { return true @@ -154,7 +154,7 @@ func IsAlpha(str string) bool { return rxAlpha.MatchString(str) } -//IsUTFLetter check if the string contains only unicode letter characters. +//IsUTFLetter checks if the string contains only unicode letter characters. //Similar to IsAlpha but for all languages. Empty string is valid. func IsUTFLetter(str string) bool { if IsNull(str) { @@ -170,7 +170,7 @@ func IsUTFLetter(str string) bool { } -// IsAlphanumeric check if the string contains only letters and numbers. Empty string is valid. +// IsAlphanumeric checks if the string contains only letters and numbers. Empty string is valid. func IsAlphanumeric(str string) bool { if IsNull(str) { return true @@ -178,7 +178,7 @@ func IsAlphanumeric(str string) bool { return rxAlphanumeric.MatchString(str) } -// IsUTFLetterNumeric check if the string contains only unicode letters and numbers. Empty string is valid. +// IsUTFLetterNumeric checks if the string contains only unicode letters and numbers. Empty string is valid. func IsUTFLetterNumeric(str string) bool { if IsNull(str) { return true @@ -192,7 +192,7 @@ func IsUTFLetterNumeric(str string) bool { } -// IsNumeric check if the string contains only numbers. Empty string is valid. +// IsNumeric checks if the string contains only numbers. Empty string is valid. func IsNumeric(str string) bool { if IsNull(str) { return true @@ -200,7 +200,7 @@ func IsNumeric(str string) bool { return rxNumeric.MatchString(str) } -// IsUTFNumeric check if the string contains only unicode numbers of any kind. +// IsUTFNumeric checks if the string contains only unicode numbers of any kind. // Numbers can be 0-9 but also Fractions ¾,Roman Ⅸ and Hangzhou 〩. Empty string is valid. func IsUTFNumeric(str string) bool { if IsNull(str) { @@ -222,7 +222,7 @@ func IsUTFNumeric(str string) bool { } -// IsUTFDigit check if the string contains only unicode radix-10 decimal digits. Empty string is valid. +// IsUTFDigit checks if the string contains only unicode radix-10 decimal digits. Empty string is valid. func IsUTFDigit(str string) bool { if IsNull(str) { return true @@ -243,22 +243,22 @@ func IsUTFDigit(str string) bool { } -// IsHexadecimal check if the string is a hexadecimal number. +// IsHexadecimal checks if the string is a hexadecimal number. func IsHexadecimal(str string) bool { return rxHexadecimal.MatchString(str) } -// IsHexcolor check if the string is a hexadecimal color. +// IsHexcolor checks if the string is a hexadecimal color. func IsHexcolor(str string) bool { return rxHexcolor.MatchString(str) } -// IsRGBcolor check if the string is a valid RGB color in form rgb(RRR, GGG, BBB). +// IsRGBcolor checks if the string is a valid RGB color in form rgb(RRR, GGG, BBB). func IsRGBcolor(str string) bool { return rxRGBcolor.MatchString(str) } -// IsLowerCase check if the string is lowercase. Empty string is valid. +// IsLowerCase checks if the string is lowercase. Empty string is valid. func IsLowerCase(str string) bool { if IsNull(str) { return true @@ -266,7 +266,7 @@ func IsLowerCase(str string) bool { return str == strings.ToLower(str) } -// IsUpperCase check if the string is uppercase. Empty string is valid. +// IsUpperCase checks if the string is uppercase. Empty string is valid. func IsUpperCase(str string) bool { if IsNull(str) { return true @@ -274,7 +274,7 @@ func IsUpperCase(str string) bool { return str == strings.ToUpper(str) } -// HasLowerCase check if the string contains at least 1 lowercase. Empty string is valid. +// HasLowerCase checks if the string contains at least 1 lowercase. Empty string is valid. func HasLowerCase(str string) bool { if IsNull(str) { return true @@ -282,7 +282,7 @@ func HasLowerCase(str string) bool { return rxHasLowerCase.MatchString(str) } -// HasUpperCase check if the string contians as least 1 uppercase. Empty string is valid. +// HasUpperCase checks if the string contains as least 1 uppercase. Empty string is valid. func HasUpperCase(str string) bool { if IsNull(str) { return true @@ -290,7 +290,7 @@ func HasUpperCase(str string) bool { return rxHasUpperCase.MatchString(str) } -// IsInt check if the string is an integer. Empty string is valid. +// IsInt checks if the string is an integer. Empty string is valid. func IsInt(str string) bool { if IsNull(str) { return true @@ -298,12 +298,12 @@ func IsInt(str string) bool { return rxInt.MatchString(str) } -// IsFloat check if the string is a float. +// IsFloat checks if the string is a float. func IsFloat(str string) bool { return str != "" && rxFloat.MatchString(str) } -// IsDivisibleBy check if the string is a number that's divisible by another. +// IsDivisibleBy checks if the string is a number that's divisible by another. // If second argument is not valid integer or zero, it's return false. // Otherwise, if first argument is not valid integer or zero, it's return true (Invalid string converts to zero). func IsDivisibleBy(str, num string) bool { @@ -316,47 +316,52 @@ func IsDivisibleBy(str, num string) bool { return (p == 0) || (p%q == 0) } -// IsNull check if the string is null. +// IsNull checks if the string is null. func IsNull(str string) bool { return len(str) == 0 } +// IsNotNull checks if the string is not null. +func IsNotNull(str string) bool { + return !IsNull(str) +} + // HasWhitespaceOnly checks the string only contains whitespace func HasWhitespaceOnly(str string) bool { - return len(str) > 0 && rxHasWhitespaceOnly.MatchString(str) + return len(str) > 0 && rxHasWhitespaceOnly.MatchString(str) } // HasWhitespace checks if the string contains any whitespace func HasWhitespace(str string) bool { - return len(str) > 0 && rxHasWhitespace.MatchString(str) + return len(str) > 0 && rxHasWhitespace.MatchString(str) } -// IsByteLength check if the string's length (in bytes) falls in a range. +// IsByteLength checks if the string's length (in bytes) falls in a range. func IsByteLength(str string, min, max int) bool { return len(str) >= min && len(str) <= max } -// IsUUIDv3 check if the string is a UUID version 3. +// IsUUIDv3 checks if the string is a UUID version 3. func IsUUIDv3(str string) bool { return rxUUID3.MatchString(str) } -// IsUUIDv4 check if the string is a UUID version 4. +// IsUUIDv4 checks if the string is a UUID version 4. func IsUUIDv4(str string) bool { return rxUUID4.MatchString(str) } -// IsUUIDv5 check if the string is a UUID version 5. +// IsUUIDv5 checks if the string is a UUID version 5. func IsUUIDv5(str string) bool { return rxUUID5.MatchString(str) } -// IsUUID check if the string is a UUID (version 3, 4 or 5). +// IsUUID checks if the string is a UUID (version 3, 4 or 5). func IsUUID(str string) bool { return rxUUID.MatchString(str) } -// IsCreditCard check if the string is a credit card. +// IsCreditCard checks if the string is a credit card. func IsCreditCard(str string) bool { sanitized := notNumberRegexp.ReplaceAllString(str, "") if !rxCreditCard.MatchString(sanitized) { @@ -372,7 +377,7 @@ func IsCreditCard(str string) bool { if shouldDouble { tmpNum *= 2 if tmpNum >= 10 { - sum += ((tmpNum % 10) + 1) + sum += (tmpNum % 10) + 1 } else { sum += tmpNum } @@ -385,18 +390,18 @@ func IsCreditCard(str string) bool { return sum%10 == 0 } -// IsISBN10 check if the string is an ISBN version 10. +// IsISBN10 checks if the string is an ISBN version 10. func IsISBN10(str string) bool { return IsISBN(str, 10) } -// IsISBN13 check if the string is an ISBN version 13. +// IsISBN13 checks if the string is an ISBN version 13. func IsISBN13(str string) bool { return IsISBN(str, 13) } -// IsISBN check if the string is an ISBN (version 10 or 13). -// If version value is not equal to 10 or 13, it will be check both variants. +// IsISBN checks if the string is an ISBN (version 10 or 13). +// If version value is not equal to 10 or 13, it will be checks both variants. func IsISBN(str string, version int) bool { sanitized := whiteSpacesAndMinus.ReplaceAllString(str, "") var checksum int32 @@ -430,13 +435,13 @@ func IsISBN(str string, version int) bool { return IsISBN(str, 10) || IsISBN(str, 13) } -// IsJSON check if the string is valid JSON (note: uses json.Unmarshal). +// IsJSON checks if the string is valid JSON (note: uses json.Unmarshal). func IsJSON(str string) bool { var js json.RawMessage return json.Unmarshal([]byte(str), &js) == nil } -// IsMultibyte check if the string contains one or more multibyte chars. Empty string is valid. +// IsMultibyte checks if the string contains one or more multibyte chars. Empty string is valid. func IsMultibyte(str string) bool { if IsNull(str) { return true @@ -444,7 +449,7 @@ func IsMultibyte(str string) bool { return rxMultibyte.MatchString(str) } -// IsASCII check if the string contains ASCII chars only. Empty string is valid. +// IsASCII checks if the string contains ASCII chars only. Empty string is valid. func IsASCII(str string) bool { if IsNull(str) { return true @@ -452,7 +457,7 @@ func IsASCII(str string) bool { return rxASCII.MatchString(str) } -// IsPrintableASCII check if the string contains printable ASCII chars only. Empty string is valid. +// IsPrintableASCII checks if the string contains printable ASCII chars only. Empty string is valid. func IsPrintableASCII(str string) bool { if IsNull(str) { return true @@ -460,7 +465,7 @@ func IsPrintableASCII(str string) bool { return rxPrintableASCII.MatchString(str) } -// IsFullWidth check if the string contains any full-width chars. Empty string is valid. +// IsFullWidth checks if the string contains any full-width chars. Empty string is valid. func IsFullWidth(str string) bool { if IsNull(str) { return true @@ -468,7 +473,7 @@ func IsFullWidth(str string) bool { return rxFullWidth.MatchString(str) } -// IsHalfWidth check if the string contains any half-width chars. Empty string is valid. +// IsHalfWidth checks if the string contains any half-width chars. Empty string is valid. func IsHalfWidth(str string) bool { if IsNull(str) { return true @@ -476,7 +481,7 @@ func IsHalfWidth(str string) bool { return rxHalfWidth.MatchString(str) } -// IsVariableWidth check if the string contains a mixture of full and half-width chars. Empty string is valid. +// IsVariableWidth checks if the string contains a mixture of full and half-width chars. Empty string is valid. func IsVariableWidth(str string) bool { if IsNull(str) { return true @@ -484,12 +489,12 @@ func IsVariableWidth(str string) bool { return rxHalfWidth.MatchString(str) && rxFullWidth.MatchString(str) } -// IsBase64 check if a string is base64 encoded. +// IsBase64 checks if a string is base64 encoded. func IsBase64(str string) bool { return rxBase64.MatchString(str) } -// IsFilePath check is a string is Win or Unix file path and returns it's type. +// IsFilePath checks is a string is Win or Unix file path and returns it's type. func IsFilePath(str string) (bool, int) { if rxWinPath.MatchString(str) { //check windows path limit see: @@ -513,6 +518,11 @@ func IsDataURI(str string) bool { return IsBase64(dataURI[1]) } +// IsMagnetURI checks if a string is valid magnet URI +func IsMagnetURI(str string) bool { + return rxMagnetURI.MatchString(str) +} + // IsISO3166Alpha2 checks if a string is valid two-letter country code func IsISO3166Alpha2(str string) bool { for _, entry := range ISO3166List { @@ -565,7 +575,7 @@ func IsDNSName(str string) bool { // IsHash checks if a string is a hash of type algorithm. // Algorithm is one of ['md4', 'md5', 'sha1', 'sha256', 'sha384', 'sha512', 'ripemd128', 'ripemd160', 'tiger128', 'tiger160', 'tiger192', 'crc32', 'crc32b'] func IsHash(str string, algorithm string) bool { - len := "0" + var len string algo := strings.ToLower(algorithm) if algo == "crc32" || algo == "crc32b" { @@ -589,9 +599,73 @@ func IsHash(str string, algorithm string) bool { return Matches(str, "^[a-f0-9]{"+len+"}$") } +// IsSHA512 checks is a string is a SHA512 hash. Alias for `IsHash(str, "sha512")` +func IsSHA512(str string) bool { + return IsHash(str, "sha512") +} + +// IsSHA384 checks is a string is a SHA384 hash. Alias for `IsHash(str, "sha384")` +func IsSHA384(str string) bool { + return IsHash(str, "sha384") +} + +// IsSHA256 checks is a string is a SHA256 hash. Alias for `IsHash(str, "sha256")` +func IsSHA256(str string) bool { + return IsHash(str, "sha256") +} + +// IsTiger192 checks is a string is a Tiger192 hash. Alias for `IsHash(str, "tiger192")` +func IsTiger192(str string) bool { + return IsHash(str, "tiger192") +} + +// IsTiger160 checks is a string is a Tiger160 hash. Alias for `IsHash(str, "tiger160")` +func IsTiger160(str string) bool { + return IsHash(str, "tiger160") +} + +// IsRipeMD160 checks is a string is a RipeMD160 hash. Alias for `IsHash(str, "ripemd160")` +func IsRipeMD160(str string) bool { + return IsHash(str, "ripemd160") +} + +// IsSHA1 checks is a string is a SHA-1 hash. Alias for `IsHash(str, "sha1")` +func IsSHA1(str string) bool { + return IsHash(str, "sha1") +} + +// IsTiger128 checks is a string is a Tiger128 hash. Alias for `IsHash(str, "tiger128")` +func IsTiger128(str string) bool { + return IsHash(str, "tiger128") +} + +// IsRipeMD128 checks is a string is a RipeMD128 hash. Alias for `IsHash(str, "ripemd128")` +func IsRipeMD128(str string) bool { + return IsHash(str, "ripemd128") +} + +// IsCRC32 checks is a string is a CRC32 hash. Alias for `IsHash(str, "crc32")` +func IsCRC32(str string) bool { + return IsHash(str, "crc32") +} + +// IsCRC32b checks is a string is a CRC32b hash. Alias for `IsHash(str, "crc32b")` +func IsCRC32b(str string) bool { + return IsHash(str, "crc32b") +} + +// IsMD5 checks is a string is a MD5 hash. Alias for `IsHash(str, "md5")` +func IsMD5(str string) bool { + return IsHash(str, "md5") +} + +// IsMD4 checks is a string is a MD4 hash. Alias for `IsHash(str, "md4")` +func IsMD4(str string) bool { + return IsHash(str, "md4") +} + // IsDialString validates the given string for usage with the various Dial() functions func IsDialString(str string) bool { - if h, p, err := net.SplitHostPort(str); err == nil && h != "" && p != "" && (IsDNSName(h) || IsIP(h)) && IsPort(p) { return true } @@ -599,7 +673,7 @@ func IsDialString(str string) bool { return false } -// IsIP checks if a string is either IP version 4 or 6. +// IsIP checks if a string is either IP version 4 or 6. Alias for `net.ParseIP` func IsIP(str string) bool { return net.ParseIP(str) != nil } @@ -612,25 +686,25 @@ func IsPort(str string) bool { return false } -// IsIPv4 check if the string is an IP version 4. +// IsIPv4 checks if the string is an IP version 4. func IsIPv4(str string) bool { ip := net.ParseIP(str) return ip != nil && strings.Contains(str, ".") } -// IsIPv6 check if the string is an IP version 6. +// IsIPv6 checks if the string is an IP version 6. func IsIPv6(str string) bool { ip := net.ParseIP(str) return ip != nil && strings.Contains(str, ":") } -// IsCIDR check if the string is an valid CIDR notiation (IPV4 & IPV6) +// IsCIDR checks if the string is an valid CIDR notiation (IPV4 & IPV6) func IsCIDR(str string) bool { _, _, err := net.ParseCIDR(str) return err == nil } -// IsMAC check if a string is valid MAC address. +// IsMAC checks if a string is valid MAC address. // Possible MAC formats: // 01:23:45:67:89:ab // 01:23:45:67:89:ab:cd:ef @@ -648,22 +722,70 @@ func IsHost(str string) bool { return IsIP(str) || IsDNSName(str) } -// IsMongoID check if the string is a valid hex-encoded representation of a MongoDB ObjectId. +// IsMongoID checks if the string is a valid hex-encoded representation of a MongoDB ObjectId. func IsMongoID(str string) bool { return rxHexadecimal.MatchString(str) && (len(str) == 24) } -// IsLatitude check if a string is valid latitude. +// IsLatitude checks if a string is valid latitude. func IsLatitude(str string) bool { return rxLatitude.MatchString(str) } -// IsLongitude check if a string is valid longitude. +// IsLongitude checks if a string is valid longitude. func IsLongitude(str string) bool { return rxLongitude.MatchString(str) } -// IsRsaPublicKey check if a string is valid public key with provided length +// IsIMEI checks if a string is valid IMEI +func IsIMEI(str string) bool { + return rxIMEI.MatchString(str) +} + +// IsIMSI checks if a string is valid IMSI +func IsIMSI(str string) bool { + if !rxIMSI.MatchString(str) { + return false + } + + mcc, err := strconv.ParseInt(str[0:3], 10, 32) + if err != nil { + return false + } + + switch mcc { + case 202, 204, 206, 208, 212, 213, 214, 216, 218, 219: + case 220, 221, 222, 226, 228, 230, 231, 232, 234, 235: + case 238, 240, 242, 244, 246, 247, 248, 250, 255, 257: + case 259, 260, 262, 266, 268, 270, 272, 274, 276, 278: + case 280, 282, 283, 284, 286, 288, 289, 290, 292, 293: + case 294, 295, 297, 302, 308, 310, 311, 312, 313, 314: + case 315, 316, 330, 332, 334, 338, 340, 342, 344, 346: + case 348, 350, 352, 354, 356, 358, 360, 362, 363, 364: + case 365, 366, 368, 370, 372, 374, 376, 400, 401, 402: + case 404, 405, 406, 410, 412, 413, 414, 415, 416, 417: + case 418, 419, 420, 421, 422, 424, 425, 426, 427, 428: + case 429, 430, 431, 432, 434, 436, 437, 438, 440, 441: + case 450, 452, 454, 455, 456, 457, 460, 461, 466, 467: + case 470, 472, 502, 505, 510, 514, 515, 520, 525, 528: + case 530, 536, 537, 539, 540, 541, 542, 543, 544, 545: + case 546, 547, 548, 549, 550, 551, 552, 553, 554, 555: + case 602, 603, 604, 605, 606, 607, 608, 609, 610, 611: + case 612, 613, 614, 615, 616, 617, 618, 619, 620, 621: + case 622, 623, 624, 625, 626, 627, 628, 629, 630, 631: + case 632, 633, 634, 635, 636, 637, 638, 639, 640, 641: + case 642, 643, 645, 646, 647, 648, 649, 650, 651, 652: + case 653, 654, 655, 657, 658, 659, 702, 704, 706, 708: + case 710, 712, 714, 716, 722, 724, 730, 732, 734, 736: + case 738, 740, 742, 744, 746, 748, 750, 995: + return true + default: + return false + } + return true +} + +// IsRsaPublicKey checks if a string is valid public key with provided length func IsRsaPublicKey(str string, keylen int) bool { bb := bytes.NewBufferString(str) pemBytes, err := ioutil.ReadAll(bb) @@ -717,7 +839,7 @@ func toJSONName(tag string) string { return name } -func PrependPathToErrors(err error, path string) error { +func prependPathToErrors(err error, path string) error { switch err2 := err.(type) { case Error: err2.Path = append([]string{path}, err2.Path...) @@ -725,16 +847,125 @@ func PrependPathToErrors(err error, path string) error { case Errors: errors := err2.Errors() for i, err3 := range errors { - errors[i] = PrependPathToErrors(err3, path) + errors[i] = prependPathToErrors(err3, path) } return err2 } - fmt.Println(err) return err } +// ValidateArray performs validation according to condition iterator that validates every element of the array +func ValidateArray(array []interface{}, iterator ConditionIterator) bool { + return Every(array, iterator) +} + +// ValidateMap use validation map for fields. +// result will be equal to `false` if there are any errors. +// s is the map containing the data to be validated. +// m is the validation map in the form: +// map[string]interface{}{"name":"required,alpha","address":map[string]interface{}{"line1":"required,alphanum"}} +func ValidateMap(s map[string]interface{}, m map[string]interface{}) (bool, error) { + if s == nil { + return true, nil + } + result := true + var err error + var errs Errors + var index int + val := reflect.ValueOf(s) + for key, value := range s { + presentResult := true + validator, ok := m[key] + if !ok { + presentResult = false + var err error + err = fmt.Errorf("all map keys has to be present in the validation map; got %s", key) + err = prependPathToErrors(err, key) + errs = append(errs, err) + } + valueField := reflect.ValueOf(value) + mapResult := true + typeResult := true + structResult := true + resultField := true + switch subValidator := validator.(type) { + case map[string]interface{}: + var err error + if v, ok := value.(map[string]interface{}); !ok { + mapResult = false + err = fmt.Errorf("map validator has to be for the map type only; got %s", valueField.Type().String()) + err = prependPathToErrors(err, key) + errs = append(errs, err) + } else { + mapResult, err = ValidateMap(v, subValidator) + if err != nil { + mapResult = false + err = prependPathToErrors(err, key) + errs = append(errs, err) + } + } + case string: + if (valueField.Kind() == reflect.Struct || + (valueField.Kind() == reflect.Ptr && valueField.Elem().Kind() == reflect.Struct)) && + subValidator != "-" { + var err error + structResult, err = ValidateStruct(valueField.Interface()) + if err != nil { + err = prependPathToErrors(err, key) + errs = append(errs, err) + } + } + resultField, err = typeCheck(valueField, reflect.StructField{ + Name: key, + PkgPath: "", + Type: val.Type(), + Tag: reflect.StructTag(fmt.Sprintf("%s:%q", tagName, subValidator)), + Offset: 0, + Index: []int{index}, + Anonymous: false, + }, val, nil) + if err != nil { + errs = append(errs, err) + } + case nil: + // already handlerd when checked before + default: + typeResult = false + err = fmt.Errorf("map validator has to be either map[string]interface{} or string; got %s", valueField.Type().String()) + err = prependPathToErrors(err, key) + errs = append(errs, err) + } + result = result && presentResult && typeResult && resultField && structResult && mapResult + index++ + } + // checks required keys + requiredResult := true + for key, value := range m { + if schema, ok := value.(string); ok { + tags := parseTagIntoMap(schema) + if required, ok := tags["required"]; ok { + if _, ok := s[key]; !ok { + requiredResult = false + if required.customErrorMessage != "" { + err = Error{key, fmt.Errorf(required.customErrorMessage), true, "required", []string{}} + } else { + err = Error{key, fmt.Errorf("required field missing"), false, "required", []string{}} + } + errs = append(errs, err) + } + } + } + } + + if len(errs) > 0 { + err = errs + } + return result && requiredResult, err +} + // ValidateStruct use tags for fields. // result will be equal to `false` if there are any errors. +// todo currently there is no guarantee that errors will be returned in predictable order (tests may to fail) func ValidateStruct(s interface{}) (bool, error) { if s == nil { return true, nil @@ -766,7 +997,7 @@ func ValidateStruct(s interface{}) (bool, error) { var err error structResult, err = ValidateStruct(valueField.Interface()) if err != nil { - err = PrependPathToErrors(err, typeField.Name) + err = prependPathToErrors(err, typeField.Name) errs = append(errs, err) } } @@ -803,6 +1034,42 @@ func ValidateStruct(s interface{}) (bool, error) { return result, err } +// ValidateStructAsync performs async validation of the struct and returns results through the channels +func ValidateStructAsync(s interface{}) (<-chan bool, <-chan error) { + res := make(chan bool) + errors := make(chan error) + + go func() { + defer close(res) + defer close(errors) + + isValid, isFailed := ValidateStruct(s) + + res <- isValid + errors <- isFailed + }() + + return res, errors +} + +// ValidateMapAsync performs async validation of the map and returns results through the channels +func ValidateMapAsync(s map[string]interface{}, m map[string]interface{}) (<-chan bool, <-chan error) { + res := make(chan bool) + errors := make(chan error) + + go func() { + defer close(res) + defer close(errors) + + isValid, isFailed := ValidateMap(s, m) + + res <- isValid + errors <- isFailed + }() + + return res, errors +} + // parseTagIntoMap parses a struct tag `valid:required~Some error message,length(2|3)` into map[string]string{"required": "Some error message", "length(2|3)": ""} func parseTagIntoMap(tag string) tagOptionsMap { optionsMap := make(tagOptionsMap) @@ -851,28 +1118,45 @@ func IsSSN(str string) bool { return rxSSN.MatchString(str) } -// IsSemver check if string is valid semantic version +// IsSemver checks if string is valid semantic version func IsSemver(str string) bool { return rxSemver.MatchString(str) } -// IsTime check if string is valid according to given format +// IsType checks if interface is of some type +func IsType(v interface{}, params ...string) bool { + if len(params) == 1 { + typ := params[0] + return strings.Replace(reflect.TypeOf(v).String(), " ", "", -1) == strings.Replace(typ, " ", "", -1) + } + return false +} + +// IsTime checks if string is valid according to given format func IsTime(str string, format string) bool { _, err := time.Parse(format, str) return err == nil } -// IsRFC3339 check if string is valid timestamp value according to RFC3339 +// IsUnixTime checks if string is valid unix timestamp value +func IsUnixTime(str string) bool { + if _, err := strconv.Atoi(str); err == nil { + return true + } + return false +} + +// IsRFC3339 checks if string is valid timestamp value according to RFC3339 func IsRFC3339(str string) bool { return IsTime(str, time.RFC3339) } -// IsRFC3339WithoutZone check if string is valid timestamp value according to RFC3339 which excludes the timezone. +// IsRFC3339WithoutZone checks if string is valid timestamp value according to RFC3339 which excludes the timezone. func IsRFC3339WithoutZone(str string) bool { - return IsTime(str, RF3339WithoutZone) + return IsTime(str, rfc3339WithoutZone) } -// IsISO4217 check if string is valid ISO currency code +// IsISO4217 checks if string is valid ISO currency code func IsISO4217(str string) bool { for _, currency := range ISO4217List { if str == currency { @@ -883,7 +1167,7 @@ func IsISO4217(str string) bool { return false } -// ByteLength check string's length +// ByteLength checks string's length func ByteLength(str string, params ...string) bool { if len(params) == 2 { min, _ := ToInt(params[0]) @@ -894,13 +1178,13 @@ func ByteLength(str string, params ...string) bool { return false } -// RuneLength check string's length +// RuneLength checks string's length // Alias for StringLength func RuneLength(str string, params ...string) bool { return StringLength(str, params...) } -// IsRsaPub check whether string is valid RSA key +// IsRsaPub checks whether string is valid RSA key // Alias for IsRsaPublicKey func IsRsaPub(str string, params ...string) bool { if len(params) == 1 { @@ -920,7 +1204,7 @@ func StringMatches(s string, params ...string) bool { return false } -// StringLength check string's length (including multi byte strings) +// StringLength checks string's length (including multi byte strings) func StringLength(str string, params ...string) bool { if len(params) == 2 { @@ -933,7 +1217,31 @@ func StringLength(str string, params ...string) bool { return false } -// Range check string's length +// MinStringLength checks string's minimum length (including multi byte strings) +func MinStringLength(str string, params ...string) bool { + + if len(params) == 1 { + strLength := utf8.RuneCountInString(str) + min, _ := ToInt(params[0]) + return strLength >= int(min) + } + + return false +} + +// MaxStringLength checks string's maximum length (including multi byte strings) +func MaxStringLength(str string, params ...string) bool { + + if len(params) == 1 { + strLength := utf8.RuneCountInString(str) + max, _ := ToInt(params[0]) + return strLength <= int(max) + } + + return false +} + +// Range checks string's length func Range(str string, params ...string) bool { if len(params) == 2 { value, _ := ToFloat(str) @@ -945,7 +1253,8 @@ func Range(str string, params ...string) bool { return false } -func isInRaw(str string, params ...string) bool { +// IsInRaw checks if string is in list of allowed values +func IsInRaw(str string, params ...string) bool { if len(params) == 1 { rawParams := params[0] @@ -957,7 +1266,7 @@ func isInRaw(str string, params ...string) bool { return false } -// IsIn check if string str is a member of the set of strings params +// IsIn checks if string str is a member of the set of strings params func IsIn(str string, params ...string) bool { for _, param := range params { if str == param { @@ -995,7 +1304,7 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options tag := t.Tag.Get(tagName) - // Check if the field should be ignored + // checks if the field should be ignored switch tag { case "": if v.Kind() != reflect.Slice && v.Kind() != reflect.Map { @@ -1015,7 +1324,7 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options } if isEmptyValue(v) { - // an empty value is not validated, check only required + // an empty value is not validated, checks only required isValid, resultErr = checkRequired(v, t, options) for key := range options { delete(options, key) @@ -1062,26 +1371,65 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options }() } + for _, validatorSpec := range optionsOrder { + validatorStruct := options[validatorSpec] + var negate bool + validator := validatorSpec + customMsgExists := len(validatorStruct.customErrorMessage) > 0 + + // checks whether the tag looks like '!something' or 'something' + if validator[0] == '!' { + validator = validator[1:] + negate = true + } + + // checks for interface param validators + for key, value := range InterfaceParamTagRegexMap { + ps := value.FindStringSubmatch(validator) + if len(ps) == 0 { + continue + } + + validatefunc, ok := InterfaceParamTagMap[key] + if !ok { + continue + } + + delete(options, validatorSpec) + + field := fmt.Sprint(v) + if result := validatefunc(v.Interface(), ps[1:]...); (!result && !negate) || (result && negate) { + if customMsgExists { + return false, Error{t.Name, TruncatingErrorf(validatorStruct.customErrorMessage, field, validator), customMsgExists, stripParams(validatorSpec), []string{}} + } + if negate { + return false, Error{t.Name, fmt.Errorf("%s does validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}} + } + return false, Error{t.Name, fmt.Errorf("%s does not validate as %s", field, validator), customMsgExists, stripParams(validatorSpec), []string{}} + } + } + } + switch v.Kind() { case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.Float32, reflect.Float64, reflect.String: - // for each tag option check the map of validator functions + // for each tag option checks the map of validator functions for _, validatorSpec := range optionsOrder { validatorStruct := options[validatorSpec] var negate bool validator := validatorSpec customMsgExists := len(validatorStruct.customErrorMessage) > 0 - // Check whether the tag looks like '!something' or 'something' + // checks whether the tag looks like '!something' or 'something' if validator[0] == '!' { validator = validator[1:] negate = true } - // Check for param validators + // checks for param validators for key, value := range ParamTagRegexMap { ps := value.FindStringSubmatch(validator) if len(ps) == 0 { @@ -1121,10 +1469,10 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options delete(options, validatorSpec) switch v.Kind() { - case reflect.String, - reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64: + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: field := fmt.Sprint(v) // make value into string, then validate with regex if result := validatefunc(field); !result && !negate || result && negate { if customMsgExists { @@ -1162,7 +1510,7 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options } else { resultItem, err = ValidateStruct(v.MapIndex(k).Interface()) if err != nil { - err = PrependPathToErrors(err, t.Name+"."+sv[i].Interface().(string)) + err = prependPathToErrors(err, t.Name+"."+sv[i].Interface().(string)) return false, err } } @@ -1182,7 +1530,7 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options } else { resultItem, err = ValidateStruct(v.Index(i).Interface()) if err != nil { - err = PrependPathToErrors(err, t.Name+"."+strconv.Itoa(i)) + err = prependPathToErrors(err, t.Name+"."+strconv.Itoa(i)) return false, err } } @@ -1196,13 +1544,13 @@ func typeCheck(v reflect.Value, t reflect.StructField, o reflect.Value, options } return ValidateStruct(v.Interface()) case reflect.Ptr: - // If the value is a pointer then check its element + // If the value is a pointer then checks its element if v.IsNil() { return true, nil } return typeCheck(v.Elem(), t, o, options) case reflect.Struct: - return ValidateStruct(v.Interface()) + return true, nil default: return false, &UnsupportedTypeError{v.Type()} } @@ -1212,6 +1560,7 @@ func stripParams(validatorString string) string { return paramsRegexp.ReplaceAllString(validatorString, "") } +// isEmptyValue checks whether value empty or not func isEmptyValue(v reflect.Value) bool { switch v.Kind() { case reflect.String, reflect.Array: @@ -1252,11 +1601,11 @@ func ErrorsByField(e error) map[string]string { } // prototype for ValidateStruct - switch e.(type) { + switch e := e.(type) { case Error: - m[e.(Error).Name] = e.(Error).Err.Error() + m[e.Name] = e.Err.Error() case Errors: - for _, item := range e.(Errors).Errors() { + for _, item := range e.Errors() { n := ErrorsByField(item) for k, v := range n { m[k] = v diff --git a/src/runtime/vendor/github.com/asaskevich/govalidator/wercker.yml b/src/runtime/vendor/github.com/asaskevich/govalidator/wercker.yml index cac7a5fcf063..bc5f7b0864bd 100644 --- a/src/runtime/vendor/github.com/asaskevich/govalidator/wercker.yml +++ b/src/runtime/vendor/github.com/asaskevich/govalidator/wercker.yml @@ -12,4 +12,4 @@ build: - script: name: go test code: | - go test -race ./... + go test -race -v ./... diff --git a/src/runtime/vendor/github.com/globalsign/mgo/LICENSE b/src/runtime/vendor/github.com/globalsign/mgo/LICENSE deleted file mode 100644 index 770c7672b45d..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -mgo - MongoDB driver for Go - -Copyright (c) 2010-2013 - Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/LICENSE b/src/runtime/vendor/github.com/globalsign/mgo/bson/LICENSE deleted file mode 100644 index 890326017b85..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -BSON library for Go - -Copyright (c) 2010-2012 - Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/README.md b/src/runtime/vendor/github.com/globalsign/mgo/bson/README.md deleted file mode 100644 index 5c5819e612b7..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/README.md +++ /dev/null @@ -1,12 +0,0 @@ -[![GoDoc](https://godoc.org/github.com/globalsign/mgo/bson?status.svg)](https://godoc.org/github.com/globalsign/mgo/bson) - -An Implementation of BSON for Go --------------------------------- - -Package bson is an implementation of the [BSON specification](http://bsonspec.org) for Go. - -While the BSON package implements the BSON spec as faithfully as possible, there -is some MongoDB specific behaviour (such as map keys `$in`, `$all`, etc) in the -`bson` package. The priority is for backwards compatibility for the `mgo` -driver, though fixes for obviously buggy behaviour is welcome (and features, etc -behind feature flags). diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/bson.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/bson.go deleted file mode 100644 index eb87ef6208a2..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/bson.go +++ /dev/null @@ -1,836 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Package bson is an implementation of the BSON specification for Go: -// -// http://bsonspec.org -// -// It was created as part of the mgo MongoDB driver for Go, but is standalone -// and may be used on its own without the driver. -package bson - -import ( - "bytes" - "crypto/md5" - "crypto/rand" - "encoding/binary" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "os" - "reflect" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" -) - -//go:generate go run bson_corpus_spec_test_generator.go - -// -------------------------------------------------------------------------- -// The public API. - -// Element types constants from BSON specification. -const ( - ElementFloat64 byte = 0x01 - ElementString byte = 0x02 - ElementDocument byte = 0x03 - ElementArray byte = 0x04 - ElementBinary byte = 0x05 - Element06 byte = 0x06 - ElementObjectId byte = 0x07 - ElementBool byte = 0x08 - ElementDatetime byte = 0x09 - ElementNil byte = 0x0A - ElementRegEx byte = 0x0B - ElementDBPointer byte = 0x0C - ElementJavaScriptWithoutScope byte = 0x0D - ElementSymbol byte = 0x0E - ElementJavaScriptWithScope byte = 0x0F - ElementInt32 byte = 0x10 - ElementTimestamp byte = 0x11 - ElementInt64 byte = 0x12 - ElementDecimal128 byte = 0x13 - ElementMinKey byte = 0xFF - ElementMaxKey byte = 0x7F - - BinaryGeneric byte = 0x00 - BinaryFunction byte = 0x01 - BinaryBinaryOld byte = 0x02 - BinaryUUIDOld byte = 0x03 - BinaryUUID byte = 0x04 - BinaryMD5 byte = 0x05 - BinaryUserDefined byte = 0x80 -) - -// Getter interface: a value implementing the bson.Getter interface will have its GetBSON -// method called when the given value has to be marshalled, and the result -// of this method will be marshaled in place of the actual object. -// -// If GetBSON returns return a non-nil error, the marshalling procedure -// will stop and error out with the provided value. -type Getter interface { - GetBSON() (interface{}, error) -} - -// Setter interface: a value implementing the bson.Setter interface will receive the BSON -// value via the SetBSON method during unmarshaling, and the object -// itself will not be changed as usual. -// -// If setting the value works, the method should return nil or alternatively -// bson.ErrSetZero to set the respective field to its zero value (nil for -// pointer types). If SetBSON returns a value of type bson.TypeError, the -// BSON value will be omitted from a map or slice being decoded and the -// unmarshalling will continue. If it returns any other non-nil error, the -// unmarshalling procedure will stop and error out with the provided value. -// -// This interface is generally useful in pointer receivers, since the method -// will want to change the receiver. A type field that implements the Setter -// interface doesn't have to be a pointer, though. -// -// Unlike the usual behavior, unmarshalling onto a value that implements a -// Setter interface will NOT reset the value to its zero state. This allows -// the value to decide by itself how to be unmarshalled. -// -// For example: -// -// type MyString string -// -// func (s *MyString) SetBSON(raw bson.Raw) error { -// return raw.Unmarshal(s) -// } -// -type Setter interface { - SetBSON(raw Raw) error -} - -// ErrSetZero may be returned from a SetBSON method to have the value set to -// its respective zero value. When used in pointer values, this will set the -// field to nil rather than to the pre-allocated value. -var ErrSetZero = errors.New("set to zero") - -// M is a convenient alias for a map[string]interface{} map, useful for -// dealing with BSON in a native way. For instance: -// -// bson.M{"a": 1, "b": true} -// -// There's no special handling for this type in addition to what's done anyway -// for an equivalent map type. Elements in the map will be dumped in an -// undefined ordered. See also the bson.D type for an ordered alternative. -type M map[string]interface{} - -// D represents a BSON document containing ordered elements. For example: -// -// bson.D{{"a", 1}, {"b", true}} -// -// In some situations, such as when creating indexes for MongoDB, the order in -// which the elements are defined is important. If the order is not important, -// using a map is generally more comfortable. See bson.M and bson.RawD. -type D []DocElem - -// DocElem is an element of the bson.D document representation. -type DocElem struct { - Name string - Value interface{} -} - -// Map returns a map out of the ordered element name/value pairs in d. -func (d D) Map() (m M) { - m = make(M, len(d)) - for _, item := range d { - m[item.Name] = item.Value - } - return m -} - -// The Raw type represents raw unprocessed BSON documents and elements. -// Kind is the kind of element as defined per the BSON specification, and -// Data is the raw unprocessed data for the respective element. -// Using this type it is possible to unmarshal or marshal values partially. -// -// Relevant documentation: -// -// http://bsonspec.org/#/specification -// -type Raw struct { - Kind byte - Data []byte -} - -// RawD represents a BSON document containing raw unprocessed elements. -// This low-level representation may be useful when lazily processing -// documents of uncertain content, or when manipulating the raw content -// documents in general. -type RawD []RawDocElem - -// RawDocElem elements of RawD type. -type RawDocElem struct { - Name string - Value Raw -} - -// ObjectId is a unique ID identifying a BSON value. It must be exactly 12 bytes -// long. MongoDB objects by default have such a property set in their "_id" -// property. -// -// http://www.mongodb.org/display/DOCS/Object+Ids -type ObjectId string - -// ObjectIdHex returns an ObjectId from the provided hex representation. -// Calling this function with an invalid hex representation will -// cause a runtime panic. See the IsObjectIdHex function. -func ObjectIdHex(s string) ObjectId { - d, err := hex.DecodeString(s) - if err != nil || len(d) != 12 { - panic(fmt.Sprintf("invalid input to ObjectIdHex: %q", s)) - } - return ObjectId(d) -} - -// IsObjectIdHex returns whether s is a valid hex representation of -// an ObjectId. See the ObjectIdHex function. -func IsObjectIdHex(s string) bool { - if len(s) != 24 { - return false - } - _, err := hex.DecodeString(s) - return err == nil -} - -// objectIdCounter is atomically incremented when generating a new ObjectId -// using NewObjectId() function. It's used as a counter part of an id. -var objectIdCounter = readRandomUint32() - -// readRandomUint32 returns a random objectIdCounter. -func readRandomUint32() uint32 { - var b [4]byte - _, err := io.ReadFull(rand.Reader, b[:]) - if err != nil { - panic(fmt.Errorf("cannot read random object id: %v", err)) - } - return uint32((uint32(b[0]) << 0) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24)) -} - -// machineId stores machine id generated once and used in subsequent calls -// to NewObjectId function. -var machineId = readMachineId() -var processId = os.Getpid() - -// readMachineId generates and returns a machine id. -// If this function fails to get the hostname it will cause a runtime error. -func readMachineId() []byte { - var sum [3]byte - id := sum[:] - hostname, err1 := os.Hostname() - if err1 != nil { - _, err2 := io.ReadFull(rand.Reader, id) - if err2 != nil { - panic(fmt.Errorf("cannot get hostname: %v; %v", err1, err2)) - } - return id - } - hw := md5.New() - hw.Write([]byte(hostname)) - copy(id, hw.Sum(nil)) - return id -} - -// NewObjectId returns a new unique ObjectId. -func NewObjectId() ObjectId { - var b [12]byte - // Timestamp, 4 bytes, big endian - binary.BigEndian.PutUint32(b[:], uint32(time.Now().Unix())) - // Machine, first 3 bytes of md5(hostname) - b[4] = machineId[0] - b[5] = machineId[1] - b[6] = machineId[2] - // Pid, 2 bytes, specs don't specify endianness, but we use big endian. - b[7] = byte(processId >> 8) - b[8] = byte(processId) - // Increment, 3 bytes, big endian - i := atomic.AddUint32(&objectIdCounter, 1) - b[9] = byte(i >> 16) - b[10] = byte(i >> 8) - b[11] = byte(i) - return ObjectId(b[:]) -} - -// NewObjectIdWithTime returns a dummy ObjectId with the timestamp part filled -// with the provided number of seconds from epoch UTC, and all other parts -// filled with zeroes. It's not safe to insert a document with an id generated -// by this method, it is useful only for queries to find documents with ids -// generated before or after the specified timestamp. -func NewObjectIdWithTime(t time.Time) ObjectId { - var b [12]byte - binary.BigEndian.PutUint32(b[:4], uint32(t.Unix())) - return ObjectId(string(b[:])) -} - -// String returns a hex string representation of the id. -// Example: ObjectIdHex("4d88e15b60f486e428412dc9"). -func (id ObjectId) String() string { - return fmt.Sprintf(`ObjectIdHex("%x")`, string(id)) -} - -// Hex returns a hex representation of the ObjectId. -func (id ObjectId) Hex() string { - return hex.EncodeToString([]byte(id)) -} - -// MarshalJSON turns a bson.ObjectId into a json.Marshaller. -func (id ObjectId) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%x"`, string(id))), nil -} - -var nullBytes = []byte("null") - -// UnmarshalJSON turns *bson.ObjectId into a json.Unmarshaller. -func (id *ObjectId) UnmarshalJSON(data []byte) error { - if len(data) > 0 && (data[0] == '{' || data[0] == 'O') { - var v struct { - Id json.RawMessage `json:"$oid"` - Func struct { - Id json.RawMessage - } `json:"$oidFunc"` - } - err := jdec(data, &v) - if err == nil { - if len(v.Id) > 0 { - data = []byte(v.Id) - } else { - data = []byte(v.Func.Id) - } - } - } - if len(data) == 2 && data[0] == '"' && data[1] == '"' || bytes.Equal(data, nullBytes) { - *id = "" - return nil - } - if len(data) != 26 || data[0] != '"' || data[25] != '"' { - return fmt.Errorf("invalid ObjectId in JSON: %s", string(data)) - } - var buf [12]byte - _, err := hex.Decode(buf[:], data[1:25]) - if err != nil { - return fmt.Errorf("invalid ObjectId in JSON: %s (%s)", string(data), err) - } - *id = ObjectId(string(buf[:])) - return nil -} - -// MarshalText turns bson.ObjectId into an encoding.TextMarshaler. -func (id ObjectId) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf("%x", string(id))), nil -} - -// UnmarshalText turns *bson.ObjectId into an encoding.TextUnmarshaler. -func (id *ObjectId) UnmarshalText(data []byte) error { - if len(data) == 1 && data[0] == ' ' || len(data) == 0 { - *id = "" - return nil - } - if len(data) != 24 { - return fmt.Errorf("invalid ObjectId: %s", data) - } - var buf [12]byte - _, err := hex.Decode(buf[:], data[:]) - if err != nil { - return fmt.Errorf("invalid ObjectId: %s (%s)", data, err) - } - *id = ObjectId(string(buf[:])) - return nil -} - -// Valid returns true if id is valid. A valid id must contain exactly 12 bytes. -func (id ObjectId) Valid() bool { - return len(id) == 12 -} - -// byteSlice returns byte slice of id from start to end. -// Calling this function with an invalid id will cause a runtime panic. -func (id ObjectId) byteSlice(start, end int) []byte { - if len(id) != 12 { - panic(fmt.Sprintf("invalid ObjectId: %q", string(id))) - } - return []byte(string(id)[start:end]) -} - -// Time returns the timestamp part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Time() time.Time { - // First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch. - secs := int64(binary.BigEndian.Uint32(id.byteSlice(0, 4))) - return time.Unix(secs, 0) -} - -// Machine returns the 3-byte machine id part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Machine() []byte { - return id.byteSlice(4, 7) -} - -// Pid returns the process id part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Pid() uint16 { - return binary.BigEndian.Uint16(id.byteSlice(7, 9)) -} - -// Counter returns the incrementing value part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Counter() int32 { - b := id.byteSlice(9, 12) - // Counter is stored as big-endian 3-byte value - return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])) -} - -// The Symbol type is similar to a string and is used in languages with a -// distinct symbol type. -type Symbol string - -// Now returns the current time with millisecond precision. MongoDB stores -// timestamps with the same precision, so a Time returned from this method -// will not change after a roundtrip to the database. That's the only reason -// why this function exists. Using the time.Now function also works fine -// otherwise. -func Now() time.Time { - return time.Unix(0, time.Now().UnixNano()/1e6*1e6) -} - -// MongoTimestamp is a special internal type used by MongoDB that for some -// strange reason has its own datatype defined in BSON. -type MongoTimestamp int64 - -// Time returns the time part of ts which is stored with second precision. -func (ts MongoTimestamp) Time() time.Time { - return time.Unix(int64(uint64(ts)>>32), 0) -} - -// Counter returns the counter part of ts. -func (ts MongoTimestamp) Counter() uint32 { - return uint32(ts) -} - -// NewMongoTimestamp creates a timestamp using the given -// date `t` (with second precision) and counter `c` (unique for `t`). -// -// Returns an error if time `t` is not between 1970-01-01T00:00:00Z -// and 2106-02-07T06:28:15Z (inclusive). -// -// Note that two MongoTimestamps should never have the same (time, counter) combination: -// the caller must ensure the counter `c` is increased if creating multiple MongoTimestamp -// values for the same time `t` (ignoring fractions of seconds). -func NewMongoTimestamp(t time.Time, c uint32) (MongoTimestamp, error) { - u := t.Unix() - if u < 0 || u > math.MaxUint32 { - return -1, errors.New("invalid value for time") - } - - i := int64(u<<32 | int64(c)) - - return MongoTimestamp(i), nil -} - -type orderKey int64 - -// MaxKey is a special value that compares higher than all other possible BSON -// values in a MongoDB database. -var MaxKey = orderKey(1<<63 - 1) - -// MinKey is a special value that compares lower than all other possible BSON -// values in a MongoDB database. -var MinKey = orderKey(-1 << 63) - -type undefined struct{} - -// Undefined represents the undefined BSON value. -var Undefined undefined - -// Binary is a representation for non-standard binary values. Any kind should -// work, but the following are known as of this writing: -// -// 0x00 - Generic. This is decoded as []byte(data), not Binary{0x00, data}. -// 0x01 - Function (!?) -// 0x02 - Obsolete generic. -// 0x03 - UUID -// 0x05 - MD5 -// 0x80 - User defined. -// -type Binary struct { - Kind byte - Data []byte -} - -// RegEx represents a regular expression. The Options field may contain -// individual characters defining the way in which the pattern should be -// applied, and must be sorted. Valid options as of this writing are 'i' for -// case insensitive matching, 'm' for multi-line matching, 'x' for verbose -// mode, 'l' to make \w, \W, and similar be locale-dependent, 's' for dot-all -// mode (a '.' matches everything), and 'u' to make \w, \W, and similar match -// unicode. The value of the Options parameter is not verified before being -// marshaled into the BSON format. -type RegEx struct { - Pattern string - Options string -} - -// JavaScript is a type that holds JavaScript code. If Scope is non-nil, it -// will be marshaled as a mapping from identifiers to values that may be -// used when evaluating the provided Code. -type JavaScript struct { - Code string - Scope interface{} -} - -// DBPointer refers to a document id in a namespace. -// -// This type is deprecated in the BSON specification and should not be used -// except for backwards compatibility with ancient applications. -type DBPointer struct { - Namespace string - Id ObjectId -} - -const initialBufferSize = 64 - -func handleErr(err *error) { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } else if _, ok := r.(externalPanic); ok { - panic(r) - } else if s, ok := r.(string); ok { - *err = errors.New(s) - } else if e, ok := r.(error); ok { - *err = e - } else { - panic(r) - } - } -} - -// Marshal serializes the in value, which may be a map or a struct value. -// In the case of struct values, only exported fields will be serialized, -// and the order of serialized fields will match that of the struct itself. -// The lowercased field name is used as the key for each exported field, -// but this behavior may be changed using the respective field tag. -// The tag may also contain flags to tweak the marshalling behavior for -// the field. The tag formats accepted are: -// -// "[][,[,]]" -// -// `(...) bson:"[][,[,]]" (...)` -// -// The following flags are currently supported: -// -// omitempty Only include the field if it's not set to the zero -// value for the type or to empty slices or maps. -// -// minsize Marshal an int64 value as an int32, if that's feasible -// while preserving the numeric value. -// -// inline Inline the field, which must be a struct or a map, -// causing all of its fields or keys to be processed as if -// they were part of the outer struct. For maps, keys must -// not conflict with the bson keys of other struct fields. -// -// Some examples: -// -// type T struct { -// A bool -// B int "myb" -// C string "myc,omitempty" -// D string `bson:",omitempty" json:"jsonkey"` -// E int64 ",minsize" -// F int64 "myf,omitempty,minsize" -// } -// -func Marshal(in interface{}) (out []byte, err error) { - return MarshalBuffer(in, make([]byte, 0, initialBufferSize)) -} - -// MarshalBuffer behaves the same way as Marshal, except that instead of -// allocating a new byte slice it tries to use the received byte slice and -// only allocates more memory if necessary to fit the marshaled value. -func MarshalBuffer(in interface{}, buf []byte) (out []byte, err error) { - defer handleErr(&err) - e := &encoder{buf} - e.addDoc(reflect.ValueOf(in)) - return e.out, nil -} - -// Unmarshal deserializes data from in into the out value. The out value -// must be a map, a pointer to a struct, or a pointer to a bson.D value. -// In the case of struct values, only exported fields will be deserialized. -// The lowercased field name is used as the key for each exported field, -// but this behavior may be changed using the respective field tag. -// The tag may also contain flags to tweak the marshalling behavior for -// the field. The tag formats accepted are: -// -// "[][,[,]]" -// -// `(...) bson:"[][,[,]]" (...)` -// -// The following flags are currently supported during unmarshal (see the -// Marshal method for other flags): -// -// inline Inline the field, which must be a struct or a map. -// Inlined structs are handled as if its fields were part -// of the outer struct. An inlined map causes keys that do -// not match any other struct field to be inserted in the -// map rather than being discarded as usual. -// -// The target field or element types of out may not necessarily match -// the BSON values of the provided data. The following conversions are -// made automatically: -// -// - Numeric types are converted if at least the integer part of the -// value would be preserved correctly -// - Bools are converted to numeric types as 1 or 0 -// - Numeric types are converted to bools as true if not 0 or false otherwise -// - Binary and string BSON data is converted to a string, array or byte slice -// -// If the value would not fit the type and cannot be converted, it's -// silently skipped. -// -// Pointer values are initialized when necessary. -func Unmarshal(in []byte, out interface{}) (err error) { - if raw, ok := out.(*Raw); ok { - raw.Kind = 3 - raw.Data = in - return nil - } - defer handleErr(&err) - v := reflect.ValueOf(out) - switch v.Kind() { - case reflect.Ptr: - fallthrough - case reflect.Map: - d := newDecoder(in) - d.readDocTo(v) - if d.i < len(d.in) { - return errors.New("document is corrupted") - } - case reflect.Struct: - return errors.New("unmarshal can't deal with struct values. Use a pointer") - default: - return errors.New("unmarshal needs a map or a pointer to a struct") - } - return nil -} - -// Unmarshal deserializes raw into the out value. If the out value type -// is not compatible with raw, a *bson.TypeError is returned. -// -// See the Unmarshal function documentation for more details on the -// unmarshalling process. -func (raw Raw) Unmarshal(out interface{}) (err error) { - defer handleErr(&err) - v := reflect.ValueOf(out) - switch v.Kind() { - case reflect.Ptr: - v = v.Elem() - fallthrough - case reflect.Map: - d := newDecoder(raw.Data) - good := d.readElemTo(v, raw.Kind) - if !good { - return &TypeError{v.Type(), raw.Kind} - } - case reflect.Struct: - return errors.New("raw Unmarshal can't deal with struct values. Use a pointer") - default: - return errors.New("raw Unmarshal needs a map or a valid pointer") - } - return nil -} - -// TypeError store details for type error occuring -// during unmarshaling -type TypeError struct { - Type reflect.Type - Kind byte -} - -func (e *TypeError) Error() string { - return fmt.Sprintf("BSON kind 0x%02x isn't compatible with type %s", e.Kind, e.Type.String()) -} - -// -------------------------------------------------------------------------- -// Maintain a mapping of keys to structure field indexes - -type structInfo struct { - FieldsMap map[string]fieldInfo - FieldsList []fieldInfo - InlineMap int - Zero reflect.Value -} - -type fieldInfo struct { - Key string - Num int - OmitEmpty bool - MinSize bool - Inline []int -} - -var structMap = make(map[reflect.Type]*structInfo) -var structMapMutex sync.RWMutex - -type externalPanic string - -func (e externalPanic) String() string { - return string(e) -} - -func getStructInfo(st reflect.Type) (*structInfo, error) { - structMapMutex.RLock() - sinfo, found := structMap[st] - structMapMutex.RUnlock() - if found { - return sinfo, nil - } - n := st.NumField() - fieldsMap := make(map[string]fieldInfo) - fieldsList := make([]fieldInfo, 0, n) - inlineMap := -1 - for i := 0; i != n; i++ { - field := st.Field(i) - if field.PkgPath != "" && !field.Anonymous { - continue // Private field - } - - info := fieldInfo{Num: i} - - tag := field.Tag.Get("bson") - - // Fall-back to JSON struct tag, if feature flag is set. - if tag == "" && useJSONTagFallback { - tag = field.Tag.Get("json") - } - - // If there's no bson/json tag available. - if tag == "" { - // If there's no tag, and also no tag: value splits (i.e. no colon) - // then assume the entire tag is the value - if strings.Index(string(field.Tag), ":") < 0 { - tag = string(field.Tag) - } - } - - if tag == "-" { - continue - } - - inline := false - fields := strings.Split(tag, ",") - if len(fields) > 1 { - for _, flag := range fields[1:] { - switch flag { - case "omitempty": - info.OmitEmpty = true - case "minsize": - info.MinSize = true - case "inline": - inline = true - default: - msg := fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st) - panic(externalPanic(msg)) - } - } - tag = fields[0] - } - - if inline { - switch field.Type.Kind() { - case reflect.Map: - if inlineMap >= 0 { - return nil, errors.New("Multiple ,inline maps in struct " + st.String()) - } - if field.Type.Key() != reflect.TypeOf("") { - return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) - } - inlineMap = info.Num - case reflect.Ptr: - // allow only pointer to struct - if kind := field.Type.Elem().Kind(); kind != reflect.Struct { - return nil, errors.New("Option ,inline allows a pointer only to a struct, was given pointer to " + kind.String()) - } - - field.Type = field.Type.Elem() - fallthrough - case reflect.Struct: - sinfo, err := getStructInfo(field.Type) - if err != nil { - return nil, err - } - for _, finfo := range sinfo.FieldsList { - if _, found := fieldsMap[finfo.Key]; found { - msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() - return nil, errors.New(msg) - } - if finfo.Inline == nil { - finfo.Inline = []int{i, finfo.Num} - } else { - finfo.Inline = append([]int{i}, finfo.Inline...) - } - fieldsMap[finfo.Key] = finfo - fieldsList = append(fieldsList, finfo) - } - default: - panic("Option ,inline needs a struct value or a pointer to a struct or map field") - } - continue - } - - if tag != "" { - info.Key = tag - } else { - info.Key = strings.ToLower(field.Name) - } - - if _, found = fieldsMap[info.Key]; found { - msg := "Duplicated key '" + info.Key + "' in struct " + st.String() - return nil, errors.New(msg) - } - - fieldsList = append(fieldsList, info) - fieldsMap[info.Key] = info - } - sinfo = &structInfo{ - fieldsMap, - fieldsList, - inlineMap, - reflect.New(st).Elem(), - } - structMapMutex.Lock() - structMap[st] = sinfo - structMapMutex.Unlock() - return sinfo, nil -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/compatibility.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/compatibility.go deleted file mode 100644 index 66efd465facc..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/compatibility.go +++ /dev/null @@ -1,29 +0,0 @@ -package bson - -// Current state of the JSON tag fallback option. -var useJSONTagFallback = false -var useRespectNilValues = false - -// SetJSONTagFallback enables or disables the JSON-tag fallback for structure tagging. When this is enabled, structures -// without BSON tags on a field will fall-back to using the JSON tag (if present). -func SetJSONTagFallback(state bool) { - useJSONTagFallback = state -} - -// JSONTagFallbackState returns the current status of the JSON tag fallback compatability option. See SetJSONTagFallback -// for more information. -func JSONTagFallbackState() bool { - return useJSONTagFallback -} - -// SetRespectNilValues enables or disables serializing nil slices or maps to `null` values. -// In other words it enables `encoding/json` compatible behaviour. -func SetRespectNilValues(state bool) { - useRespectNilValues = state -} - -// RespectNilValuesState returns the current status of the JSON nil slices and maps fallback compatibility option. -// See SetRespectNilValues for more information. -func RespectNilValuesState() bool { - return useRespectNilValues -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/decimal.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/decimal.go deleted file mode 100644 index 672ba1825940..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/decimal.go +++ /dev/null @@ -1,312 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package bson - -import ( - "fmt" - "strconv" - "strings" -) - -// Decimal128 holds decimal128 BSON values. -type Decimal128 struct { - h, l uint64 -} - -func (d Decimal128) String() string { - var pos int // positive sign - var e int // exponent - var h, l uint64 // significand high/low - - if d.h>>63&1 == 0 { - pos = 1 - } - - switch d.h >> 58 & (1<<5 - 1) { - case 0x1F: - return "NaN" - case 0x1E: - return "-Inf"[pos:] - } - - l = d.l - if d.h>>61&3 == 3 { - // Bits: 1*sign 2*ignored 14*exponent 111*significand. - // Implicit 0b100 prefix in significand. - e = int(d.h>>47&(1<<14-1)) - 6176 - //h = 4<<47 | d.h&(1<<47-1) - // Spec says all of these values are out of range. - h, l = 0, 0 - } else { - // Bits: 1*sign 14*exponent 113*significand - e = int(d.h>>49&(1<<14-1)) - 6176 - h = d.h & (1<<49 - 1) - } - - // Would be handled by the logic below, but that's trivial and common. - if h == 0 && l == 0 && e == 0 { - return "-0"[pos:] - } - - var repr [48]byte // Loop 5 times over 9 digits plus dot, negative sign, and leading zero. - var last = len(repr) - var i = len(repr) - var dot = len(repr) + e - var rem uint32 -Loop: - for d9 := 0; d9 < 5; d9++ { - h, l, rem = divmod(h, l, 1e9) - for d1 := 0; d1 < 9; d1++ { - // Handle "-0.0", "0.00123400", "-1.00E-6", "1.050E+3", etc. - if i < len(repr) && (dot == i || l == 0 && h == 0 && rem > 0 && rem < 10 && (dot < i-6 || e > 0)) { - e += len(repr) - i - i-- - repr[i] = '.' - last = i - 1 - dot = len(repr) // Unmark. - } - c := '0' + byte(rem%10) - rem /= 10 - i-- - repr[i] = c - // Handle "0E+3", "1E+3", etc. - if l == 0 && h == 0 && rem == 0 && i == len(repr)-1 && (dot < i-5 || e > 0) { - last = i - break Loop - } - if c != '0' { - last = i - } - // Break early. Works without it, but why. - if dot > i && l == 0 && h == 0 && rem == 0 { - break Loop - } - } - } - repr[last-1] = '-' - last-- - - if e > 0 { - return string(repr[last+pos:]) + "E+" + strconv.Itoa(e) - } - if e < 0 { - return string(repr[last+pos:]) + "E" + strconv.Itoa(e) - } - return string(repr[last+pos:]) -} - -func divmod(h, l uint64, div uint32) (qh, ql uint64, rem uint32) { - div64 := uint64(div) - a := h >> 32 - aq := a / div64 - ar := a % div64 - b := ar<<32 + h&(1<<32-1) - bq := b / div64 - br := b % div64 - c := br<<32 + l>>32 - cq := c / div64 - cr := c % div64 - d := cr<<32 + l&(1<<32-1) - dq := d / div64 - dr := d % div64 - return (aq<<32 | bq), (cq<<32 | dq), uint32(dr) -} - -var dNaN = Decimal128{0x1F << 58, 0} -var dPosInf = Decimal128{0x1E << 58, 0} -var dNegInf = Decimal128{0x3E << 58, 0} - -func dErr(s string) (Decimal128, error) { - return dNaN, fmt.Errorf("cannot parse %q as a decimal128", s) -} - -// ParseDecimal128 parse a string and return the corresponding value as -// a decimal128 -func ParseDecimal128(s string) (Decimal128, error) { - orig := s - if s == "" { - return dErr(orig) - } - neg := s[0] == '-' - if neg || s[0] == '+' { - s = s[1:] - } - - if (len(s) == 3 || len(s) == 8) && (s[0] == 'N' || s[0] == 'n' || s[0] == 'I' || s[0] == 'i') { - if s == "NaN" || s == "nan" || strings.EqualFold(s, "nan") { - return dNaN, nil - } - if s == "Inf" || s == "inf" || strings.EqualFold(s, "inf") || strings.EqualFold(s, "infinity") { - if neg { - return dNegInf, nil - } - return dPosInf, nil - } - return dErr(orig) - } - - var h, l uint64 - var e int - - var add, ovr uint32 - var mul uint32 = 1 - var dot = -1 - var digits = 0 - var i = 0 - for i < len(s) { - c := s[i] - if mul == 1e9 { - h, l, ovr = muladd(h, l, mul, add) - mul, add = 1, 0 - if ovr > 0 || h&((1<<15-1)<<49) > 0 { - return dErr(orig) - } - } - if c >= '0' && c <= '9' { - i++ - if c > '0' || digits > 0 { - digits++ - } - if digits > 34 { - if c == '0' { - // Exact rounding. - e++ - continue - } - return dErr(orig) - } - mul *= 10 - add *= 10 - add += uint32(c - '0') - continue - } - if c == '.' { - i++ - if dot >= 0 || i == 1 && len(s) == 1 { - return dErr(orig) - } - if i == len(s) { - break - } - if s[i] < '0' || s[i] > '9' || e > 0 { - return dErr(orig) - } - dot = i - continue - } - break - } - if i == 0 { - return dErr(orig) - } - if mul > 1 { - h, l, ovr = muladd(h, l, mul, add) - if ovr > 0 || h&((1<<15-1)<<49) > 0 { - return dErr(orig) - } - } - if dot >= 0 { - e += dot - i - } - if i+1 < len(s) && (s[i] == 'E' || s[i] == 'e') { - i++ - eneg := s[i] == '-' - if eneg || s[i] == '+' { - i++ - if i == len(s) { - return dErr(orig) - } - } - n := 0 - for i < len(s) && n < 1e4 { - c := s[i] - i++ - if c < '0' || c > '9' { - return dErr(orig) - } - n *= 10 - n += int(c - '0') - } - if eneg { - n = -n - } - e += n - for e < -6176 { - // Subnormal. - var div uint32 = 1 - for div < 1e9 && e < -6176 { - div *= 10 - e++ - } - var rem uint32 - h, l, rem = divmod(h, l, div) - if rem > 0 { - return dErr(orig) - } - } - for e > 6111 { - // Clamped. - var mul uint32 = 1 - for mul < 1e9 && e > 6111 { - mul *= 10 - e-- - } - h, l, ovr = muladd(h, l, mul, 0) - if ovr > 0 || h&((1<<15-1)<<49) > 0 { - return dErr(orig) - } - } - if e < -6176 || e > 6111 { - return dErr(orig) - } - } - - if i < len(s) { - return dErr(orig) - } - - h |= uint64(e+6176) & uint64(1<<14-1) << 49 - if neg { - h |= 1 << 63 - } - return Decimal128{h, l}, nil -} - -func muladd(h, l uint64, mul uint32, add uint32) (resh, resl uint64, overflow uint32) { - mul64 := uint64(mul) - a := mul64 * (l & (1<<32 - 1)) - b := a>>32 + mul64*(l>>32) - c := b>>32 + mul64*(h&(1<<32-1)) - d := c>>32 + mul64*(h>>32) - - a = a&(1<<32-1) + uint64(add) - b = b&(1<<32-1) + a>>32 - c = c&(1<<32-1) + b>>32 - d = d&(1<<32-1) + c>>32 - - return (d<<32 | c&(1<<32-1)), (b<<32 | a&(1<<32-1)), uint32(d >> 32) -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/decode.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/decode.go deleted file mode 100644 index 658856add04b..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/decode.go +++ /dev/null @@ -1,1055 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// gobson - BSON library for Go. - -package bson - -import ( - "errors" - "fmt" - "io" - "math" - "net/url" - "reflect" - "strconv" - "sync" - "time" -) - -type decoder struct { - in []byte - i int - docType reflect.Type -} - -var typeM = reflect.TypeOf(M{}) - -func newDecoder(in []byte) *decoder { - return &decoder{in, 0, typeM} -} - -// -------------------------------------------------------------------------- -// Some helper functions. - -func corrupted() { - panic("Document is corrupted") -} - -// -------------------------------------------------------------------------- -// Unmarshaling of documents. - -const ( - setterUnknown = iota - setterNone - setterType - setterAddr -) - -var setterStyles map[reflect.Type]int -var setterIface reflect.Type -var setterMutex sync.RWMutex - -func init() { - var iface Setter - setterIface = reflect.TypeOf(&iface).Elem() - setterStyles = make(map[reflect.Type]int) -} - -func setterStyle(outt reflect.Type) int { - setterMutex.RLock() - style := setterStyles[outt] - setterMutex.RUnlock() - if style != setterUnknown { - return style - } - - setterMutex.Lock() - defer setterMutex.Unlock() - if outt.Implements(setterIface) { - style = setterType - } else if reflect.PtrTo(outt).Implements(setterIface) { - style = setterAddr - } else { - style = setterNone - } - setterStyles[outt] = style - return style -} - -func getSetter(outt reflect.Type, out reflect.Value) Setter { - style := setterStyle(outt) - if style == setterNone { - return nil - } - if style == setterAddr { - if !out.CanAddr() { - return nil - } - out = out.Addr() - } else if outt.Kind() == reflect.Ptr && out.IsNil() { - out.Set(reflect.New(outt.Elem())) - } - return out.Interface().(Setter) -} - -func clearMap(m reflect.Value) { - var none reflect.Value - for _, k := range m.MapKeys() { - m.SetMapIndex(k, none) - } -} - -func (d *decoder) readDocTo(out reflect.Value) { - var elemType reflect.Type - outt := out.Type() - outk := outt.Kind() - - for { - if outk == reflect.Ptr && out.IsNil() { - out.Set(reflect.New(outt.Elem())) - } - if setter := getSetter(outt, out); setter != nil { - raw := d.readRaw(ElementDocument) - err := setter.SetBSON(raw) - if _, ok := err.(*TypeError); err != nil && !ok { - panic(err) - } - return - } - if outk == reflect.Ptr { - out = out.Elem() - outt = out.Type() - outk = out.Kind() - continue - } - break - } - - var fieldsMap map[string]fieldInfo - var inlineMap reflect.Value - if outt == typeRaw { - out.Set(reflect.ValueOf(d.readRaw(ElementDocument))) - return - } - - origout := out - if outk == reflect.Interface { - if d.docType.Kind() == reflect.Map { - mv := reflect.MakeMap(d.docType) - out.Set(mv) - out = mv - } else { - dv := reflect.New(d.docType).Elem() - out.Set(dv) - out = dv - } - outt = out.Type() - outk = outt.Kind() - } - - docType := d.docType - keyType := typeString - convertKey := false - switch outk { - case reflect.Map: - keyType = outt.Key() - if keyType != typeString { - convertKey = true - } - elemType = outt.Elem() - if elemType == typeIface { - d.docType = outt - } - if out.IsNil() { - out.Set(reflect.MakeMap(out.Type())) - } else if out.Len() > 0 { - clearMap(out) - } - case reflect.Struct: - sinfo, err := getStructInfo(out.Type()) - if err != nil { - panic(err) - } - fieldsMap = sinfo.FieldsMap - out.Set(sinfo.Zero) - if sinfo.InlineMap != -1 { - inlineMap = out.Field(sinfo.InlineMap) - if !inlineMap.IsNil() && inlineMap.Len() > 0 { - clearMap(inlineMap) - } - elemType = inlineMap.Type().Elem() - if elemType == typeIface { - d.docType = inlineMap.Type() - } - } - case reflect.Slice: - switch outt.Elem() { - case typeDocElem: - origout.Set(d.readDocElems(outt)) - return - case typeRawDocElem: - origout.Set(d.readRawDocElems(outt)) - return - } - fallthrough - default: - panic("Unsupported document type for unmarshalling: " + out.Type().String()) - } - - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - name := d.readCStr() - if d.i >= end { - corrupted() - } - - switch outk { - case reflect.Map: - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - k := reflect.ValueOf(name) - if convertKey { - mapKeyType := out.Type().Key() - mapKeyKind := mapKeyType.Kind() - - switch mapKeyKind { - case reflect.Int: - fallthrough - case reflect.Int8: - fallthrough - case reflect.Int16: - fallthrough - case reflect.Int32: - fallthrough - case reflect.Int64: - fallthrough - case reflect.Uint: - fallthrough - case reflect.Uint8: - fallthrough - case reflect.Uint16: - fallthrough - case reflect.Uint32: - fallthrough - case reflect.Uint64: - fallthrough - case reflect.Float32: - fallthrough - case reflect.Float64: - parsed := d.parseMapKeyAsFloat(k, mapKeyKind) - k = reflect.ValueOf(parsed) - case reflect.String: - mapKeyType = keyType - default: - panic("BSON map must have string or decimal keys. Got: " + outt.String()) - } - - k = k.Convert(mapKeyType) - } - out.SetMapIndex(k, e) - } - case reflect.Struct: - if info, ok := fieldsMap[name]; ok { - if info.Inline == nil { - d.readElemTo(out.Field(info.Num), kind) - } else { - d.readElemTo(out.FieldByIndex(info.Inline), kind) - } - } else if inlineMap.IsValid() { - if inlineMap.IsNil() { - inlineMap.Set(reflect.MakeMap(inlineMap.Type())) - } - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - inlineMap.SetMapIndex(reflect.ValueOf(name), e) - } - } else { - d.dropElem(kind) - } - case reflect.Slice: - } - - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } - d.docType = docType -} - -func (decoder) parseMapKeyAsFloat(k reflect.Value, mapKeyKind reflect.Kind) float64 { - parsed, err := strconv.ParseFloat(k.String(), 64) - if err != nil { - panic("Map key is defined to be a decimal type (" + mapKeyKind.String() + ") but got error " + - err.Error()) - } - - return parsed -} - -func (d *decoder) readArrayDocTo(out reflect.Value) { - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - i := 0 - l := out.Len() - for d.in[d.i] != '\x00' { - if i >= l { - panic("Length mismatch on array field") - } - kind := d.readByte() - for d.i < end && d.in[d.i] != '\x00' { - d.i++ - } - if d.i >= end { - corrupted() - } - d.i++ - d.readElemTo(out.Index(i), kind) - if d.i >= end { - corrupted() - } - i++ - } - if i != l { - panic("Length mismatch on array field") - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } -} - -func (d *decoder) readSliceDoc(t reflect.Type) interface{} { - tmp := make([]reflect.Value, 0, 8) - elemType := t.Elem() - if elemType == typeRawDocElem { - d.dropElem(ElementArray) - return reflect.Zero(t).Interface() - } - if elemType == typeRaw { - return d.readSliceOfRaw() - } - - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - for d.i < end && d.in[d.i] != '\x00' { - d.i++ - } - if d.i >= end { - corrupted() - } - d.i++ - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - tmp = append(tmp, e) - } - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } - - n := len(tmp) - slice := reflect.MakeSlice(t, n, n) - for i := 0; i != n; i++ { - slice.Index(i).Set(tmp[i]) - } - return slice.Interface() -} - -func BSONElementSize(kind byte, offset int, buffer []byte) (int, error) { - switch kind { - case ElementFloat64: // Float64 - return 8, nil - case ElementJavaScriptWithoutScope: // JavaScript without scope - fallthrough - case ElementSymbol: // Symbol - fallthrough - case ElementString: // UTF-8 string - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 1 { - return 0, errors.New("String size can't be less then one byte") - } - size += 4 - if offset+size > len(buffer) { - return 0, io.ErrUnexpectedEOF - } - if buffer[offset+size-1] != 0 { - return 0, errors.New("Invalid string: non zero-terminated") - } - return size, nil - case ElementArray: // Array - fallthrough - case ElementDocument: // Document - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 5 { - return 0, errors.New("Declared document size is too small") - } - return size, nil - case ElementBinary: // Binary - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 0 { - return 0, errors.New("Binary data size can't be negative") - } - return size + 5, nil - case Element06: // Undefined (obsolete, but still seen in the wild) - return 0, nil - case ElementObjectId: // ObjectId - return 12, nil - case ElementBool: // Bool - return 1, nil - case ElementDatetime: // Timestamp - return 8, nil - case ElementNil: // Nil - return 0, nil - case ElementRegEx: // RegEx - end := offset - for i := 0; i < 2; i++ { - for end < len(buffer) && buffer[end] != '\x00' { - end++ - } - end++ - } - if end > len(buffer) { - return 0, io.ErrUnexpectedEOF - } - return end - offset, nil - case ElementDBPointer: // DBPointer - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 1 { - return 0, errors.New("String size can't be less then one byte") - } - return size + 12 + 4, nil - case ElementJavaScriptWithScope: // JavaScript with scope - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 4+5+5 { - return 0, errors.New("Declared document element is too small") - } - return size, nil - case ElementInt32: // Int32 - return 4, nil - case ElementTimestamp: // Mongo-specific timestamp - return 8, nil - case ElementInt64: // Int64 - return 8, nil - case ElementDecimal128: // Decimal128 - return 16, nil - case ElementMaxKey: // Max key - return 0, nil - case ElementMinKey: // Min key - return 0, nil - default: - return 0, errors.New(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) - } -} - -func (d *decoder) readRaw(kind byte) Raw { - size, err := BSONElementSize(kind, d.i, d.in) - if err != nil { - corrupted() - } - if d.i+size > len(d.in) { - corrupted() - } - d.i += size - return Raw{ - Kind: kind, - Data: d.in[d.i-size : d.i], - } -} - -func (d *decoder) readSliceOfRaw() interface{} { - tmp := make([]Raw, 0, 8) - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - for d.i < end && d.in[d.i] != '\x00' { - d.i++ - } - if d.i >= end { - corrupted() - } - d.i++ - e := d.readRaw(kind) - tmp = append(tmp, e) - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } - return tmp -} - -var typeSlice = reflect.TypeOf([]interface{}{}) -var typeIface = typeSlice.Elem() - -func (d *decoder) readDocElems(typ reflect.Type) reflect.Value { - docType := d.docType - d.docType = typ - slice := make([]DocElem, 0, 8) - d.readDocWith(func(kind byte, name string) { - e := DocElem{Name: name} - v := reflect.ValueOf(&e.Value) - if d.readElemTo(v.Elem(), kind) { - slice = append(slice, e) - } - }) - slicev := reflect.New(typ).Elem() - slicev.Set(reflect.ValueOf(slice)) - d.docType = docType - return slicev -} - -func (d *decoder) readRawDocElems(typ reflect.Type) reflect.Value { - docType := d.docType - d.docType = typ - slice := make([]RawDocElem, 0, 8) - d.readDocWith(func(kind byte, name string) { - e := RawDocElem{Name: name, Value: d.readRaw(kind)} - slice = append(slice, e) - }) - slicev := reflect.New(typ).Elem() - slicev.Set(reflect.ValueOf(slice)) - d.docType = docType - return slicev -} - -func (d *decoder) readDocWith(f func(kind byte, name string)) { - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - name := d.readCStr() - if d.i >= end { - corrupted() - } - f(kind, name) - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } -} - -// -------------------------------------------------------------------------- -// Unmarshaling of individual elements within a document. -func (d *decoder) dropElem(kind byte) { - size, err := BSONElementSize(kind, d.i, d.in) - if err != nil { - corrupted() - } - if d.i+size > len(d.in) { - corrupted() - } - d.i += size -} - -// Attempt to decode an element from the document and put it into out. -// If the types are not compatible, the returned ok value will be -// false and out will be unchanged. -func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { - outt := out.Type() - - if outt == typeRaw { - out.Set(reflect.ValueOf(d.readRaw(kind))) - return true - } - - if outt == typeRawPtr { - raw := d.readRaw(kind) - out.Set(reflect.ValueOf(&raw)) - return true - } - - if kind == ElementDocument { - // Delegate unmarshaling of documents. - outt := out.Type() - outk := out.Kind() - switch outk { - case reflect.Interface, reflect.Ptr, reflect.Struct, reflect.Map: - d.readDocTo(out) - return true - } - if setterStyle(outt) != setterNone { - d.readDocTo(out) - return true - } - if outk == reflect.Slice { - switch outt.Elem() { - case typeDocElem: - out.Set(d.readDocElems(outt)) - case typeRawDocElem: - out.Set(d.readRawDocElems(outt)) - default: - d.dropElem(kind) - } - return true - } - d.dropElem(kind) - return true - } - - if setter := getSetter(outt, out); setter != nil { - err := setter.SetBSON(d.readRaw(kind)) - if err == ErrSetZero { - out.Set(reflect.Zero(outt)) - return true - } - if err == nil { - return true - } - if _, ok := err.(*TypeError); !ok { - panic(err) - } - return false - } - - var in interface{} - - switch kind { - case ElementFloat64: - in = d.readFloat64() - case ElementString: - in = d.readStr() - case ElementDocument: - panic("Can't happen. Handled above.") - case ElementArray: - outt := out.Type() - if setterStyle(outt) != setterNone { - // Skip the value so its data is handed to the setter below. - d.dropElem(kind) - break - } - for outt.Kind() == reflect.Ptr { - outt = outt.Elem() - } - switch outt.Kind() { - case reflect.Array: - d.readArrayDocTo(out) - return true - case reflect.Slice: - in = d.readSliceDoc(outt) - default: - in = d.readSliceDoc(typeSlice) - } - case ElementBinary: - b := d.readBinary() - if b.Kind == BinaryGeneric || b.Kind == BinaryBinaryOld { - in = b.Data - } else { - in = b - } - case Element06: // Undefined (obsolete, but still seen in the wild) - in = Undefined - case ElementObjectId: - in = ObjectId(d.readBytes(12)) - case ElementBool: - in = d.readBool() - case ElementDatetime: // Timestamp - // MongoDB handles timestamps as milliseconds. - i := d.readInt64() - if i == -62135596800000 { - in = time.Time{} // In UTC for convenience. - } else { - in = time.Unix(i/1e3, i%1e3*1e6).UTC() - } - case ElementNil: - in = nil - case ElementRegEx: - in = d.readRegEx() - case ElementDBPointer: - in = DBPointer{Namespace: d.readStr(), Id: ObjectId(d.readBytes(12))} - case ElementJavaScriptWithoutScope: - in = JavaScript{Code: d.readStr()} - case ElementSymbol: - in = Symbol(d.readStr()) - case ElementJavaScriptWithScope: - start := d.i - l := int(d.readInt32()) - js := JavaScript{d.readStr(), make(M)} - d.readDocTo(reflect.ValueOf(js.Scope)) - if d.i != start+l { - corrupted() - } - in = js - case ElementInt32: - in = int(d.readInt32()) - case ElementTimestamp: // Mongo-specific timestamp - in = MongoTimestamp(d.readInt64()) - case ElementInt64: - switch out.Type() { - case typeTimeDuration: - in = time.Duration(time.Duration(d.readInt64()) * time.Millisecond) - default: - in = d.readInt64() - } - case ElementDecimal128: - in = Decimal128{ - l: uint64(d.readInt64()), - h: uint64(d.readInt64()), - } - case ElementMaxKey: - in = MaxKey - case ElementMinKey: - in = MinKey - default: - panic(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) - } - - if in == nil { - out.Set(reflect.Zero(outt)) - return true - } - - outk := outt.Kind() - - // Dereference and initialize pointer if necessary. - first := true - for outk == reflect.Ptr { - if !out.IsNil() { - out = out.Elem() - } else { - elem := reflect.New(outt.Elem()) - if first { - // Only set if value is compatible. - first = false - defer func(out, elem reflect.Value) { - if good { - out.Set(elem) - } - }(out, elem) - } else { - out.Set(elem) - } - out = elem - } - outt = out.Type() - outk = outt.Kind() - } - - inv := reflect.ValueOf(in) - if outt == inv.Type() { - out.Set(inv) - return true - } - - switch outk { - case reflect.Interface: - out.Set(inv) - return true - case reflect.String: - switch inv.Kind() { - case reflect.String: - out.SetString(inv.String()) - return true - case reflect.Slice: - if b, ok := in.([]byte); ok { - out.SetString(string(b)) - return true - } - case reflect.Int, reflect.Int64: - if outt == typeJSONNumber { - out.SetString(strconv.FormatInt(inv.Int(), 10)) - return true - } - case reflect.Float64: - if outt == typeJSONNumber { - out.SetString(strconv.FormatFloat(inv.Float(), 'f', -1, 64)) - return true - } - } - case reflect.Slice, reflect.Array: - // Remember, array (0x04) slices are built with the correct - // element type. If we are here, must be a cross BSON kind - // conversion (e.g. 0x05 unmarshalling on string). - if outt.Elem().Kind() != reflect.Uint8 { - break - } - switch inv.Kind() { - case reflect.String: - slice := []byte(inv.String()) - out.Set(reflect.ValueOf(slice)) - return true - case reflect.Slice: - switch outt.Kind() { - case reflect.Array: - reflect.Copy(out, inv) - case reflect.Slice: - out.SetBytes(inv.Bytes()) - } - return true - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch inv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetInt(inv.Int()) - return true - case reflect.Float32, reflect.Float64: - out.SetInt(int64(inv.Float())) - return true - case reflect.Bool: - if inv.Bool() { - out.SetInt(1) - } else { - out.SetInt(0) - } - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("can't happen: no uint types in BSON (!?)") - } - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - switch inv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetUint(uint64(inv.Int())) - return true - case reflect.Float32, reflect.Float64: - out.SetUint(uint64(inv.Float())) - return true - case reflect.Bool: - if inv.Bool() { - out.SetUint(1) - } else { - out.SetUint(0) - } - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("Can't happen. No uint types in BSON.") - } - case reflect.Float32, reflect.Float64: - switch inv.Kind() { - case reflect.Float32, reflect.Float64: - out.SetFloat(inv.Float()) - return true - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetFloat(float64(inv.Int())) - return true - case reflect.Bool: - if inv.Bool() { - out.SetFloat(1) - } else { - out.SetFloat(0) - } - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("Can't happen. No uint types in BSON?") - } - case reflect.Bool: - switch inv.Kind() { - case reflect.Bool: - out.SetBool(inv.Bool()) - return true - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetBool(inv.Int() != 0) - return true - case reflect.Float32, reflect.Float64: - out.SetBool(inv.Float() != 0) - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("Can't happen. No uint types in BSON?") - } - case reflect.Struct: - if outt == typeURL && inv.Kind() == reflect.String { - u, err := url.Parse(inv.String()) - if err != nil { - panic(err) - } - out.Set(reflect.ValueOf(u).Elem()) - return true - } - if outt == typeBinary { - if b, ok := in.([]byte); ok { - out.Set(reflect.ValueOf(Binary{Data: b})) - return true - } - } - } - - return false -} - -// -------------------------------------------------------------------------- -// Parsers of basic types. - -func (d *decoder) readRegEx() RegEx { - re := RegEx{} - re.Pattern = d.readCStr() - re.Options = d.readCStr() - return re -} - -func (d *decoder) readBinary() Binary { - l := d.readInt32() - b := Binary{} - b.Kind = d.readByte() - if b.Kind == BinaryBinaryOld && l > 4 { - // Weird obsolete format with redundant length. - rl := d.readInt32() - if rl != l-4 { - corrupted() - } - l = rl - } - b.Data = d.readBytes(l) - return b -} - -func (d *decoder) readStr() string { - l := d.readInt32() - b := d.readBytes(l - 1) - if d.readByte() != '\x00' { - corrupted() - } - return string(b) -} - -func (d *decoder) readCStr() string { - start := d.i - end := start - l := len(d.in) - for ; end != l; end++ { - if d.in[end] == '\x00' { - break - } - } - d.i = end + 1 - if d.i > l { - corrupted() - } - return string(d.in[start:end]) -} - -func (d *decoder) readBool() bool { - b := d.readByte() - if b == 0 { - return false - } - if b == 1 { - return true - } - panic(fmt.Sprintf("encoded boolean must be 1 or 0, found %d", b)) -} - -func (d *decoder) readFloat64() float64 { - return math.Float64frombits(uint64(d.readInt64())) -} - -func (d *decoder) readInt32() int32 { - b := d.readBytes(4) - return int32((uint32(b[0]) << 0) | - (uint32(b[1]) << 8) | - (uint32(b[2]) << 16) | - (uint32(b[3]) << 24)) -} - -func getSize(offset int, b []byte) (int, error) { - if offset+4 > len(b) { - return 0, io.ErrUnexpectedEOF - } - return int((uint32(b[offset]) << 0) | - (uint32(b[offset+1]) << 8) | - (uint32(b[offset+2]) << 16) | - (uint32(b[offset+3]) << 24)), nil -} - -func (d *decoder) readInt64() int64 { - b := d.readBytes(8) - return int64((uint64(b[0]) << 0) | - (uint64(b[1]) << 8) | - (uint64(b[2]) << 16) | - (uint64(b[3]) << 24) | - (uint64(b[4]) << 32) | - (uint64(b[5]) << 40) | - (uint64(b[6]) << 48) | - (uint64(b[7]) << 56)) -} - -func (d *decoder) readByte() byte { - i := d.i - d.i++ - if d.i > len(d.in) { - corrupted() - } - return d.in[i] -} - -func (d *decoder) readBytes(length int32) []byte { - if length < 0 { - corrupted() - } - start := d.i - d.i += int(length) - if d.i < start || d.i > len(d.in) { - corrupted() - } - return d.in[start : start+int(length)] -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/encode.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/encode.go deleted file mode 100644 index d0c6b2a855f8..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/encode.go +++ /dev/null @@ -1,645 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// gobson - BSON library for Go. - -package bson - -import ( - "encoding/json" - "fmt" - "math" - "net/url" - "reflect" - "sort" - "strconv" - "sync" - "time" -) - -// -------------------------------------------------------------------------- -// Some internal infrastructure. - -var ( - typeBinary = reflect.TypeOf(Binary{}) - typeObjectId = reflect.TypeOf(ObjectId("")) - typeDBPointer = reflect.TypeOf(DBPointer{"", ObjectId("")}) - typeSymbol = reflect.TypeOf(Symbol("")) - typeMongoTimestamp = reflect.TypeOf(MongoTimestamp(0)) - typeOrderKey = reflect.TypeOf(MinKey) - typeDocElem = reflect.TypeOf(DocElem{}) - typeRawDocElem = reflect.TypeOf(RawDocElem{}) - typeRaw = reflect.TypeOf(Raw{}) - typeRawPtr = reflect.PtrTo(reflect.TypeOf(Raw{})) - typeURL = reflect.TypeOf(url.URL{}) - typeTime = reflect.TypeOf(time.Time{}) - typeString = reflect.TypeOf("") - typeJSONNumber = reflect.TypeOf(json.Number("")) - typeTimeDuration = reflect.TypeOf(time.Duration(0)) -) - -var ( - // spec for []uint8 or []byte encoding - arrayOps = map[string]bool{ - "$in": true, - "$nin": true, - "$all": true, - } -) - -const itoaCacheSize = 32 - -const ( - getterUnknown = iota - getterNone - getterTypeVal - getterTypePtr - getterAddr -) - -var itoaCache []string - -var getterStyles map[reflect.Type]int -var getterIface reflect.Type -var getterMutex sync.RWMutex - -func init() { - itoaCache = make([]string, itoaCacheSize) - for i := 0; i != itoaCacheSize; i++ { - itoaCache[i] = strconv.Itoa(i) - } - var iface Getter - getterIface = reflect.TypeOf(&iface).Elem() - getterStyles = make(map[reflect.Type]int) -} - -func itoa(i int) string { - if i < itoaCacheSize { - return itoaCache[i] - } - return strconv.Itoa(i) -} - -func getterStyle(outt reflect.Type) int { - getterMutex.RLock() - style := getterStyles[outt] - getterMutex.RUnlock() - if style != getterUnknown { - return style - } - - getterMutex.Lock() - defer getterMutex.Unlock() - if outt.Implements(getterIface) { - vt := outt - for vt.Kind() == reflect.Ptr { - vt = vt.Elem() - } - if vt.Implements(getterIface) { - style = getterTypeVal - } else { - style = getterTypePtr - } - } else if reflect.PtrTo(outt).Implements(getterIface) { - style = getterAddr - } else { - style = getterNone - } - getterStyles[outt] = style - return style -} - -func getGetter(outt reflect.Type, out reflect.Value) Getter { - style := getterStyle(outt) - if style == getterNone { - return nil - } - if style == getterAddr { - if !out.CanAddr() { - return nil - } - return out.Addr().Interface().(Getter) - } - if style == getterTypeVal && out.Kind() == reflect.Ptr && out.IsNil() { - return nil - } - return out.Interface().(Getter) -} - -// -------------------------------------------------------------------------- -// Marshaling of the document value itself. - -type encoder struct { - out []byte -} - -func (e *encoder) addDoc(v reflect.Value) { - for { - if vi, ok := v.Interface().(Getter); ok { - getv, err := vi.GetBSON() - if err != nil { - panic(err) - } - v = reflect.ValueOf(getv) - continue - } - if v.Kind() == reflect.Ptr { - v = v.Elem() - continue - } - break - } - - if v.Type() == typeRaw { - raw := v.Interface().(Raw) - if raw.Kind != 0x03 && raw.Kind != 0x00 { - panic("Attempted to marshal Raw kind " + strconv.Itoa(int(raw.Kind)) + " as a document") - } - if len(raw.Data) == 0 { - panic("Attempted to marshal empty Raw document") - } - e.addBytes(raw.Data...) - return - } - - start := e.reserveInt32() - - switch v.Kind() { - case reflect.Map: - e.addMap(v) - case reflect.Struct: - e.addStruct(v) - case reflect.Array, reflect.Slice: - e.addSlice(v) - default: - panic("Can't marshal " + v.Type().String() + " as a BSON document") - } - - e.addBytes(0) - e.setInt32(start, int32(len(e.out)-start)) -} - -func (e *encoder) addMap(v reflect.Value) { - for _, k := range v.MapKeys() { - e.addElem(fmt.Sprint(k), v.MapIndex(k), false) - } -} - -func (e *encoder) addStruct(v reflect.Value) { - sinfo, err := getStructInfo(v.Type()) - if err != nil { - panic(err) - } - var value reflect.Value - if sinfo.InlineMap >= 0 { - m := v.Field(sinfo.InlineMap) - if m.Len() > 0 { - for _, k := range m.MapKeys() { - ks := k.String() - if _, found := sinfo.FieldsMap[ks]; found { - panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", ks)) - } - e.addElem(ks, m.MapIndex(k), false) - } - } - } - for _, info := range sinfo.FieldsList { - if info.Inline == nil { - value = v.Field(info.Num) - } else { - // as pointers to struct are allowed here, - // there is no guarantee that pointer won't be nil. - // - // It is expected allowed behaviour - // so info.Inline MAY consist index to a nil pointer - // and that is why we safely call v.FieldByIndex and just continue on panic - field, errField := safeFieldByIndex(v, info.Inline) - if errField != nil { - continue - } - - value = field - } - if info.OmitEmpty && isZero(value) { - continue - } - if useRespectNilValues && - (value.Kind() == reflect.Slice || value.Kind() == reflect.Map) && - value.IsNil() { - e.addElem(info.Key, reflect.ValueOf(nil), info.MinSize) - continue - } - e.addElem(info.Key, value, info.MinSize) - } -} - -func safeFieldByIndex(v reflect.Value, index []int) (result reflect.Value, err error) { - defer func() { - if recovered := recover(); recovered != nil { - switch r := recovered.(type) { - case string: - err = fmt.Errorf("%s", r) - case error: - err = r - } - } - }() - - result = v.FieldByIndex(index) - return -} - -func isZero(v reflect.Value) bool { - switch v.Kind() { - case reflect.String: - return len(v.String()) == 0 - case reflect.Ptr, reflect.Interface: - return v.IsNil() - case reflect.Slice: - return v.Len() == 0 - case reflect.Map: - return v.Len() == 0 - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return v.Int() == 0 - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return v.Uint() == 0 - case reflect.Float32, reflect.Float64: - return v.Float() == 0 - case reflect.Bool: - return !v.Bool() - case reflect.Struct: - vt := v.Type() - if vt == typeTime { - return v.Interface().(time.Time).IsZero() - } - for i := 0; i < v.NumField(); i++ { - if vt.Field(i).PkgPath != "" && !vt.Field(i).Anonymous { - continue // Private field - } - if !isZero(v.Field(i)) { - return false - } - } - return true - } - return false -} - -func (e *encoder) addSlice(v reflect.Value) { - vi := v.Interface() - if d, ok := vi.(D); ok { - for _, elem := range d { - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - if d, ok := vi.(RawD); ok { - for _, elem := range d { - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - l := v.Len() - et := v.Type().Elem() - if et == typeDocElem { - for i := 0; i < l; i++ { - elem := v.Index(i).Interface().(DocElem) - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - if et == typeRawDocElem { - for i := 0; i < l; i++ { - elem := v.Index(i).Interface().(RawDocElem) - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - for i := 0; i < l; i++ { - e.addElem(itoa(i), v.Index(i), false) - } -} - -// -------------------------------------------------------------------------- -// Marshaling of elements in a document. - -func (e *encoder) addElemName(kind byte, name string) { - e.addBytes(kind) - e.addBytes([]byte(name)...) - e.addBytes(0) -} - -func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { - - if !v.IsValid() { - e.addElemName(0x0A, name) - return - } - - if getter := getGetter(v.Type(), v); getter != nil { - getv, err := getter.GetBSON() - if err != nil { - panic(err) - } - e.addElem(name, reflect.ValueOf(getv), minSize) - return - } - - switch v.Kind() { - - case reflect.Interface: - e.addElem(name, v.Elem(), minSize) - - case reflect.Ptr: - e.addElem(name, v.Elem(), minSize) - - case reflect.String: - s := v.String() - switch v.Type() { - case typeObjectId: - if len(s) != 12 { - panic("ObjectIDs must be exactly 12 bytes long (got " + - strconv.Itoa(len(s)) + ")") - } - e.addElemName(0x07, name) - e.addBytes([]byte(s)...) - case typeSymbol: - e.addElemName(0x0E, name) - e.addStr(s) - case typeJSONNumber: - n := v.Interface().(json.Number) - if i, err := n.Int64(); err == nil { - e.addElemName(0x12, name) - e.addInt64(i) - } else if f, err := n.Float64(); err == nil { - e.addElemName(0x01, name) - e.addFloat64(f) - } else { - panic("failed to convert json.Number to a number: " + s) - } - default: - e.addElemName(0x02, name) - e.addStr(s) - } - - case reflect.Float32, reflect.Float64: - e.addElemName(0x01, name) - e.addFloat64(v.Float()) - - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - u := v.Uint() - if int64(u) < 0 { - panic("BSON has no uint64 type, and value is too large to fit correctly in an int64") - } else if u <= math.MaxInt32 && (minSize || v.Kind() <= reflect.Uint32) { - e.addElemName(0x10, name) - e.addInt32(int32(u)) - } else { - e.addElemName(0x12, name) - e.addInt64(int64(u)) - } - - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch v.Type() { - case typeMongoTimestamp: - e.addElemName(0x11, name) - e.addInt64(v.Int()) - - case typeOrderKey: - if v.Int() == int64(MaxKey) { - e.addElemName(0x7F, name) - } else { - e.addElemName(0xFF, name) - } - case typeTimeDuration: - // Stored as int64 - e.addElemName(0x12, name) - - e.addInt64(int64(v.Int() / 1e6)) - default: - i := v.Int() - if (minSize || v.Type().Kind() != reflect.Int64) && i >= math.MinInt32 && i <= math.MaxInt32 { - // It fits into an int32, encode as such. - e.addElemName(0x10, name) - e.addInt32(int32(i)) - } else { - e.addElemName(0x12, name) - e.addInt64(i) - } - } - - case reflect.Bool: - e.addElemName(0x08, name) - if v.Bool() { - e.addBytes(1) - } else { - e.addBytes(0) - } - - case reflect.Map: - e.addElemName(0x03, name) - e.addDoc(v) - - case reflect.Slice: - vt := v.Type() - et := vt.Elem() - if et.Kind() == reflect.Uint8 { - if arrayOps[name] { - e.addElemName(0x04, name) - e.addDoc(v) - } else { - e.addElemName(0x05, name) - e.addBinary(0x00, v.Bytes()) - } - } else if et == typeDocElem || et == typeRawDocElem { - e.addElemName(0x03, name) - e.addDoc(v) - } else { - e.addElemName(0x04, name) - e.addDoc(v) - } - - case reflect.Array: - et := v.Type().Elem() - if et.Kind() == reflect.Uint8 { - if arrayOps[name] { - e.addElemName(0x04, name) - e.addDoc(v) - } else { - e.addElemName(0x05, name) - if v.CanAddr() { - e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte)) - } else { - n := v.Len() - e.addInt32(int32(n)) - e.addBytes(0x00) - for i := 0; i < n; i++ { - el := v.Index(i) - e.addBytes(byte(el.Uint())) - } - } - } - } else { - e.addElemName(0x04, name) - e.addDoc(v) - } - - case reflect.Struct: - switch s := v.Interface().(type) { - - case Raw: - kind := s.Kind - if kind == 0x00 { - kind = 0x03 - } - if len(s.Data) == 0 && kind != 0x06 && kind != 0x0A && kind != 0xFF && kind != 0x7F { - panic("Attempted to marshal empty Raw document") - } - e.addElemName(kind, name) - e.addBytes(s.Data...) - - case Binary: - e.addElemName(0x05, name) - e.addBinary(s.Kind, s.Data) - - case Decimal128: - e.addElemName(0x13, name) - e.addInt64(int64(s.l)) - e.addInt64(int64(s.h)) - - case DBPointer: - e.addElemName(0x0C, name) - e.addStr(s.Namespace) - if len(s.Id) != 12 { - panic("ObjectIDs must be exactly 12 bytes long (got " + - strconv.Itoa(len(s.Id)) + ")") - } - e.addBytes([]byte(s.Id)...) - - case RegEx: - e.addElemName(0x0B, name) - e.addCStr(s.Pattern) - options := runes(s.Options) - sort.Sort(options) - e.addCStr(string(options)) - - case JavaScript: - if s.Scope == nil { - e.addElemName(0x0D, name) - e.addStr(s.Code) - } else { - e.addElemName(0x0F, name) - start := e.reserveInt32() - e.addStr(s.Code) - e.addDoc(reflect.ValueOf(s.Scope)) - e.setInt32(start, int32(len(e.out)-start)) - } - - case time.Time: - // MongoDB handles timestamps as milliseconds. - e.addElemName(0x09, name) - e.addInt64(s.Unix()*1000 + int64(s.Nanosecond()/1e6)) - - case url.URL: - e.addElemName(0x02, name) - e.addStr(s.String()) - - case undefined: - e.addElemName(0x06, name) - - default: - e.addElemName(0x03, name) - e.addDoc(v) - } - - default: - panic("Can't marshal " + v.Type().String() + " in a BSON document") - } -} - -// ------------- -// Helper method for sorting regex options -type runes []rune - -func (a runes) Len() int { return len(a) } -func (a runes) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a runes) Less(i, j int) bool { return a[i] < a[j] } - -// -------------------------------------------------------------------------- -// Marshaling of base types. - -func (e *encoder) addBinary(subtype byte, v []byte) { - if subtype == 0x02 { - // Wonder how that brilliant idea came to life. Obsolete, luckily. - e.addInt32(int32(len(v) + 4)) - e.addBytes(subtype) - e.addInt32(int32(len(v))) - } else { - e.addInt32(int32(len(v))) - e.addBytes(subtype) - } - e.addBytes(v...) -} - -func (e *encoder) addStr(v string) { - e.addInt32(int32(len(v) + 1)) - e.addCStr(v) -} - -func (e *encoder) addCStr(v string) { - e.addBytes([]byte(v)...) - e.addBytes(0) -} - -func (e *encoder) reserveInt32() (pos int) { - pos = len(e.out) - e.addBytes(0, 0, 0, 0) - return pos -} - -func (e *encoder) setInt32(pos int, v int32) { - e.out[pos+0] = byte(v) - e.out[pos+1] = byte(v >> 8) - e.out[pos+2] = byte(v >> 16) - e.out[pos+3] = byte(v >> 24) -} - -func (e *encoder) addInt32(v int32) { - u := uint32(v) - e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24)) -} - -func (e *encoder) addInt64(v int64) { - u := uint64(v) - e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24), - byte(u>>32), byte(u>>40), byte(u>>48), byte(u>>56)) -} - -func (e *encoder) addFloat64(v float64) { - e.addInt64(int64(math.Float64bits(v))) -} - -func (e *encoder) addBytes(v ...byte) { - e.out = append(e.out, v...) -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/json.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/json.go deleted file mode 100644 index 045c713012b9..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/json.go +++ /dev/null @@ -1,384 +0,0 @@ -package bson - -import ( - "bytes" - "encoding/base64" - "fmt" - "strconv" - "strings" - "time" - - "github.com/globalsign/mgo/internal/json" -) - -// UnmarshalJSON unmarshals a JSON value that may hold non-standard -// syntax as defined in BSON's extended JSON specification. -func UnmarshalJSON(data []byte, value interface{}) error { - d := json.NewDecoder(bytes.NewBuffer(data)) - d.Extend(&jsonExt) - return d.Decode(value) -} - -// MarshalJSON marshals a JSON value that may hold non-standard -// syntax as defined in BSON's extended JSON specification. -func MarshalJSON(value interface{}) ([]byte, error) { - var buf bytes.Buffer - e := json.NewEncoder(&buf) - e.Extend(&jsonExt) - err := e.Encode(value) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// jdec is used internally by the JSON decoding functions -// so they may unmarshal functions without getting into endless -// recursion due to keyed objects. -func jdec(data []byte, value interface{}) error { - d := json.NewDecoder(bytes.NewBuffer(data)) - d.Extend(&funcExt) - return d.Decode(value) -} - -var jsonExt json.Extension -var funcExt json.Extension - -// TODO -// - Shell regular expressions ("/regexp/opts") - -func init() { - jsonExt.DecodeUnquotedKeys(true) - jsonExt.DecodeTrailingCommas(true) - - funcExt.DecodeFunc("BinData", "$binaryFunc", "$type", "$binary") - jsonExt.DecodeKeyed("$binary", jdecBinary) - jsonExt.DecodeKeyed("$binaryFunc", jdecBinary) - jsonExt.EncodeType([]byte(nil), jencBinarySlice) - jsonExt.EncodeType(Binary{}, jencBinaryType) - - funcExt.DecodeFunc("ISODate", "$dateFunc", "S") - funcExt.DecodeFunc("new Date", "$dateFunc", "S") - jsonExt.DecodeKeyed("$date", jdecDate) - jsonExt.DecodeKeyed("$dateFunc", jdecDate) - jsonExt.EncodeType(time.Time{}, jencDate) - - funcExt.DecodeFunc("Timestamp", "$timestamp", "t", "i") - jsonExt.DecodeKeyed("$timestamp", jdecTimestamp) - jsonExt.EncodeType(MongoTimestamp(0), jencTimestamp) - - funcExt.DecodeConst("undefined", Undefined) - - jsonExt.DecodeKeyed("$regex", jdecRegEx) - jsonExt.EncodeType(RegEx{}, jencRegEx) - - funcExt.DecodeFunc("ObjectId", "$oidFunc", "Id") - jsonExt.DecodeKeyed("$oid", jdecObjectId) - jsonExt.DecodeKeyed("$oidFunc", jdecObjectId) - jsonExt.EncodeType(ObjectId(""), jencObjectId) - - funcExt.DecodeFunc("DBRef", "$dbrefFunc", "$ref", "$id") - jsonExt.DecodeKeyed("$dbrefFunc", jdecDBRef) - - funcExt.DecodeFunc("NumberLong", "$numberLongFunc", "N") - jsonExt.DecodeKeyed("$numberLong", jdecNumberLong) - jsonExt.DecodeKeyed("$numberLongFunc", jdecNumberLong) - jsonExt.EncodeType(int64(0), jencNumberLong) - jsonExt.EncodeType(int(0), jencInt) - - funcExt.DecodeConst("MinKey", MinKey) - funcExt.DecodeConst("MaxKey", MaxKey) - jsonExt.DecodeKeyed("$minKey", jdecMinKey) - jsonExt.DecodeKeyed("$maxKey", jdecMaxKey) - jsonExt.EncodeType(orderKey(0), jencMinMaxKey) - - jsonExt.DecodeKeyed("$undefined", jdecUndefined) - jsonExt.EncodeType(Undefined, jencUndefined) - - jsonExt.Extend(&funcExt) -} - -func fbytes(format string, args ...interface{}) []byte { - var buf bytes.Buffer - fmt.Fprintf(&buf, format, args...) - return buf.Bytes() -} - -func jdecBinary(data []byte) (interface{}, error) { - var v struct { - Binary []byte `json:"$binary"` - Type string `json:"$type"` - Func struct { - Binary []byte `json:"$binary"` - Type int64 `json:"$type"` - } `json:"$binaryFunc"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - - var binData []byte - var binKind int64 - if v.Type == "" && v.Binary == nil { - binData = v.Func.Binary - binKind = v.Func.Type - } else if v.Type == "" { - return v.Binary, nil - } else { - binData = v.Binary - binKind, err = strconv.ParseInt(v.Type, 0, 64) - if err != nil { - binKind = -1 - } - } - - if binKind == 0 { - return binData, nil - } - if binKind < 0 || binKind > 255 { - return nil, fmt.Errorf("invalid type in binary object: %s", data) - } - - return Binary{Kind: byte(binKind), Data: binData}, nil -} - -func jencBinarySlice(v interface{}) ([]byte, error) { - in := v.([]byte) - out := make([]byte, base64.StdEncoding.EncodedLen(len(in))) - base64.StdEncoding.Encode(out, in) - return fbytes(`{"$binary":"%s","$type":"0x0"}`, out), nil -} - -func jencBinaryType(v interface{}) ([]byte, error) { - in := v.(Binary) - out := make([]byte, base64.StdEncoding.EncodedLen(len(in.Data))) - base64.StdEncoding.Encode(out, in.Data) - return fbytes(`{"$binary":"%s","$type":"0x%x"}`, out, in.Kind), nil -} - -const jdateFormat = "2006-01-02T15:04:05.999Z07:00" - -func jdecDate(data []byte) (interface{}, error) { - var v struct { - S string `json:"$date"` - Func struct { - S string - } `json:"$dateFunc"` - } - _ = jdec(data, &v) - if v.S == "" { - v.S = v.Func.S - } - if v.S != "" { - var errs []string - for _, format := range []string{jdateFormat, "2006-01-02"} { - t, err := time.Parse(format, v.S) - if err == nil { - return t, nil - } - errs = append(errs, err.Error()) - } - return nil, fmt.Errorf("cannot parse date: %q [%s]", v.S, strings.Join(errs, ", ")) - } - - var vn struct { - Date struct { - N int64 `json:"$numberLong,string"` - } `json:"$date"` - Func struct { - S int64 - } `json:"$dateFunc"` - } - err := jdec(data, &vn) - if err != nil { - return nil, fmt.Errorf("cannot parse date: %q", data) - } - n := vn.Date.N - if n == 0 { - n = vn.Func.S - } - return time.Unix(n/1000, n%1000*1e6).UTC(), nil -} - -func jencDate(v interface{}) ([]byte, error) { - t := v.(time.Time) - return fbytes(`{"$date":%q}`, t.Format(jdateFormat)), nil -} - -func jdecTimestamp(data []byte) (interface{}, error) { - var v struct { - Func struct { - T int32 `json:"t"` - I int32 `json:"i"` - } `json:"$timestamp"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - return MongoTimestamp(uint64(v.Func.T)<<32 | uint64(uint32(v.Func.I))), nil -} - -func jencTimestamp(v interface{}) ([]byte, error) { - ts := uint64(v.(MongoTimestamp)) - return fbytes(`{"$timestamp":{"t":%d,"i":%d}}`, ts>>32, uint32(ts)), nil -} - -func jdecRegEx(data []byte) (interface{}, error) { - var v struct { - Regex string `json:"$regex"` - Options string `json:"$options"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - return RegEx{v.Regex, v.Options}, nil -} - -func jencRegEx(v interface{}) ([]byte, error) { - re := v.(RegEx) - type regex struct { - Regex string `json:"$regex"` - Options string `json:"$options"` - } - return json.Marshal(regex{re.Pattern, re.Options}) -} - -func jdecObjectId(data []byte) (interface{}, error) { - var v struct { - Id string `json:"$oid"` - Func struct { - Id string - } `json:"$oidFunc"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if v.Id == "" { - v.Id = v.Func.Id - } - return ObjectIdHex(v.Id), nil -} - -func jencObjectId(v interface{}) ([]byte, error) { - return fbytes(`{"$oid":"%s"}`, v.(ObjectId).Hex()), nil -} - -func jdecDBRef(data []byte) (interface{}, error) { - // TODO Support unmarshaling $ref and $id into the input value. - var v struct { - Obj map[string]interface{} `json:"$dbrefFunc"` - } - // TODO Fix this. Must not be required. - v.Obj = make(map[string]interface{}) - err := jdec(data, &v) - if err != nil { - return nil, err - } - return v.Obj, nil -} - -func jdecNumberLong(data []byte) (interface{}, error) { - var v struct { - N int64 `json:"$numberLong,string"` - Func struct { - N int64 `json:",string"` - } `json:"$numberLongFunc"` - } - var vn struct { - N int64 `json:"$numberLong"` - Func struct { - N int64 - } `json:"$numberLongFunc"` - } - err := jdec(data, &v) - if err != nil { - err = jdec(data, &vn) - v.N = vn.N - v.Func.N = vn.Func.N - } - if err != nil { - return nil, err - } - if v.N != 0 { - return v.N, nil - } - return v.Func.N, nil -} - -func jencNumberLong(v interface{}) ([]byte, error) { - n := v.(int64) - f := `{"$numberLong":"%d"}` - if n <= 1<<53 { - f = `{"$numberLong":%d}` - } - return fbytes(f, n), nil -} - -func jencInt(v interface{}) ([]byte, error) { - n := v.(int) - f := `{"$numberLong":"%d"}` - if int64(n) <= 1<<53 { - f = `%d` - } - return fbytes(f, n), nil -} - -func jdecMinKey(data []byte) (interface{}, error) { - var v struct { - N int64 `json:"$minKey"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if v.N != 1 { - return nil, fmt.Errorf("invalid $minKey object: %s", data) - } - return MinKey, nil -} - -func jdecMaxKey(data []byte) (interface{}, error) { - var v struct { - N int64 `json:"$maxKey"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if v.N != 1 { - return nil, fmt.Errorf("invalid $maxKey object: %s", data) - } - return MaxKey, nil -} - -func jencMinMaxKey(v interface{}) ([]byte, error) { - switch v.(orderKey) { - case MinKey: - return []byte(`{"$minKey":1}`), nil - case MaxKey: - return []byte(`{"$maxKey":1}`), nil - } - panic(fmt.Sprintf("invalid $minKey/$maxKey value: %d", v)) -} - -func jdecUndefined(data []byte) (interface{}, error) { - var v struct { - B bool `json:"$undefined"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if !v.B { - return nil, fmt.Errorf("invalid $undefined object: %s", data) - } - return Undefined, nil -} - -func jencUndefined(v interface{}) ([]byte, error) { - return []byte(`{"$undefined":true}`), nil -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/bson/stream.go b/src/runtime/vendor/github.com/globalsign/mgo/bson/stream.go deleted file mode 100644 index 466528457b5f..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/bson/stream.go +++ /dev/null @@ -1,90 +0,0 @@ -package bson - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" -) - -const ( - // MinDocumentSize is the size of the smallest possible valid BSON document: - // an int32 size header + 0x00 (end of document). - MinDocumentSize = 5 - - // MaxDocumentSize is the largest possible size for a BSON document allowed by MongoDB, - // that is, 16 MiB (see https://docs.mongodb.com/manual/reference/limits/). - MaxDocumentSize = 16777216 -) - -// ErrInvalidDocumentSize is an error returned when a BSON document's header -// contains a size smaller than MinDocumentSize or greater than MaxDocumentSize. -type ErrInvalidDocumentSize struct { - DocumentSize int32 -} - -func (e ErrInvalidDocumentSize) Error() string { - return fmt.Sprintf("invalid document size %d", e.DocumentSize) -} - -// A Decoder reads and decodes BSON values from an input stream. -type Decoder struct { - source io.Reader -} - -// NewDecoder returns a new Decoder that reads from source. -// It does not add any extra buffering, and may not read data from source beyond the BSON values requested. -func NewDecoder(source io.Reader) *Decoder { - return &Decoder{source: source} -} - -// Decode reads the next BSON-encoded value from its input and stores it in the value pointed to by v. -// See the documentation for Unmarshal for details about the conversion of BSON into a Go value. -func (dec *Decoder) Decode(v interface{}) (err error) { - // BSON documents start with their size as a *signed* int32. - var docSize int32 - if err = binary.Read(dec.source, binary.LittleEndian, &docSize); err != nil { - return - } - - if docSize < MinDocumentSize || docSize > MaxDocumentSize { - return ErrInvalidDocumentSize{DocumentSize: docSize} - } - - docBuffer := bytes.NewBuffer(make([]byte, 0, docSize)) - if err = binary.Write(docBuffer, binary.LittleEndian, docSize); err != nil { - return - } - - // docSize is the *full* document's size (including the 4-byte size header, - // which has already been read). - if _, err = io.CopyN(docBuffer, dec.source, int64(docSize-4)); err != nil { - return - } - - // Let Unmarshal handle the rest. - defer handleErr(&err) - return Unmarshal(docBuffer.Bytes(), v) -} - -// An Encoder encodes and writes BSON values to an output stream. -type Encoder struct { - target io.Writer -} - -// NewEncoder returns a new Encoder that writes to target. -func NewEncoder(target io.Writer) *Encoder { - return &Encoder{target: target} -} - -// Encode encodes v to BSON, and if successful writes it to the Encoder's output stream. -// See the documentation for Marshal for details about the conversion of Go values to BSON. -func (enc *Encoder) Encode(v interface{}) error { - data, err := Marshal(v) - if err != nil { - return err - } - - _, err = enc.target.Write(data) - return err -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/internal/json/LICENSE b/src/runtime/vendor/github.com/globalsign/mgo/internal/json/LICENSE deleted file mode 100644 index 74487567632c..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/internal/json/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/runtime/vendor/github.com/globalsign/mgo/internal/json/decode.go b/src/runtime/vendor/github.com/globalsign/mgo/internal/json/decode.go deleted file mode 100644 index d5ca1f9a851c..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/internal/json/decode.go +++ /dev/null @@ -1,1685 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Represents JSON data structure using native Go types: booleans, floats, -// strings, arrays, and maps. - -package json - -import ( - "bytes" - "encoding" - "encoding/base64" - "errors" - "fmt" - "reflect" - "runtime" - "strconv" - "unicode" - "unicode/utf16" - "unicode/utf8" -) - -// Unmarshal parses the JSON-encoded data and stores the result -// in the value pointed to by v. -// -// Unmarshal uses the inverse of the encodings that -// Marshal uses, allocating maps, slices, and pointers as necessary, -// with the following additional rules: -// -// To unmarshal JSON into a pointer, Unmarshal first handles the case of -// the JSON being the JSON literal null. In that case, Unmarshal sets -// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into -// the value pointed at by the pointer. If the pointer is nil, Unmarshal -// allocates a new value for it to point to. -// -// To unmarshal JSON into a struct, Unmarshal matches incoming object -// keys to the keys used by Marshal (either the struct field name or its tag), -// preferring an exact match but also accepting a case-insensitive match. -// Unmarshal will only set exported fields of the struct. -// -// To unmarshal JSON into an interface value, -// Unmarshal stores one of these in the interface value: -// -// bool, for JSON booleans -// float64, for JSON numbers -// string, for JSON strings -// []interface{}, for JSON arrays -// map[string]interface{}, for JSON objects -// nil for JSON null -// -// To unmarshal a JSON array into a slice, Unmarshal resets the slice length -// to zero and then appends each element to the slice. -// As a special case, to unmarshal an empty JSON array into a slice, -// Unmarshal replaces the slice with a new empty slice. -// -// To unmarshal a JSON array into a Go array, Unmarshal decodes -// JSON array elements into corresponding Go array elements. -// If the Go array is smaller than the JSON array, -// the additional JSON array elements are discarded. -// If the JSON array is smaller than the Go array, -// the additional Go array elements are set to zero values. -// -// To unmarshal a JSON object into a map, Unmarshal first establishes a map to -// use, If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal -// reuses the existing map, keeping existing entries. Unmarshal then stores key- -// value pairs from the JSON object into the map. The map's key type must -// either be a string or implement encoding.TextUnmarshaler. -// -// If a JSON value is not appropriate for a given target type, -// or if a JSON number overflows the target type, Unmarshal -// skips that field and completes the unmarshaling as best it can. -// If no more serious errors are encountered, Unmarshal returns -// an UnmarshalTypeError describing the earliest such error. -// -// The JSON null value unmarshals into an interface, map, pointer, or slice -// by setting that Go value to nil. Because null is often used in JSON to mean -// ``not present,'' unmarshaling a JSON null into any other Go type has no effect -// on the value and produces no error. -// -// When unmarshaling quoted strings, invalid UTF-8 or -// invalid UTF-16 surrogate pairs are not treated as an error. -// Instead, they are replaced by the Unicode replacement -// character U+FFFD. -// -func Unmarshal(data []byte, v interface{}) error { - // Check for well-formedness. - // Avoids filling out half a data structure - // before discovering a JSON syntax error. - var d decodeState - err := checkValid(data, &d.scan) - if err != nil { - return err - } - - d.init(data) - return d.unmarshal(v) -} - -// Unmarshaler is the interface implemented by types -// that can unmarshal a JSON description of themselves. -// The input can be assumed to be a valid encoding of -// a JSON value. UnmarshalJSON must copy the JSON data -// if it wishes to retain the data after returning. -type Unmarshaler interface { - UnmarshalJSON([]byte) error -} - -// An UnmarshalTypeError describes a JSON value that was -// not appropriate for a value of a specific Go type. -type UnmarshalTypeError struct { - Value string // description of JSON value - "bool", "array", "number -5" - Type reflect.Type // type of Go value it could not be assigned to - Offset int64 // error occurred after reading Offset bytes -} - -func (e *UnmarshalTypeError) Error() string { - return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() -} - -// An UnmarshalFieldError describes a JSON object key that -// led to an unexported (and therefore unwritable) struct field. -// (No longer used; kept for compatibility.) -type UnmarshalFieldError struct { - Key string - Type reflect.Type - Field reflect.StructField -} - -func (e *UnmarshalFieldError) Error() string { - return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() -} - -// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. -// (The argument to Unmarshal must be a non-nil pointer.) -type InvalidUnmarshalError struct { - Type reflect.Type -} - -func (e *InvalidUnmarshalError) Error() string { - if e.Type == nil { - return "json: Unmarshal(nil)" - } - - if e.Type.Kind() != reflect.Ptr { - return "json: Unmarshal(non-pointer " + e.Type.String() + ")" - } - return "json: Unmarshal(nil " + e.Type.String() + ")" -} - -func (d *decodeState) unmarshal(v interface{}) (err error) { - defer func() { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } - err = r.(error) - } - }() - - rv := reflect.ValueOf(v) - if rv.Kind() != reflect.Ptr || rv.IsNil() { - return &InvalidUnmarshalError{reflect.TypeOf(v)} - } - - d.scan.reset() - // We decode rv not rv.Elem because the Unmarshaler interface - // test must be applied at the top level of the value. - d.value(rv) - return d.savedError -} - -// A Number represents a JSON number literal. -type Number string - -// String returns the literal text of the number. -func (n Number) String() string { return string(n) } - -// Float64 returns the number as a float64. -func (n Number) Float64() (float64, error) { - return strconv.ParseFloat(string(n), 64) -} - -// Int64 returns the number as an int64. -func (n Number) Int64() (int64, error) { - return strconv.ParseInt(string(n), 10, 64) -} - -// isValidNumber reports whether s is a valid JSON number literal. -func isValidNumber(s string) bool { - // This function implements the JSON numbers grammar. - // See https://tools.ietf.org/html/rfc7159#section-6 - // and http://json.org/number.gif - - if s == "" { - return false - } - - // Optional - - if s[0] == '-' { - s = s[1:] - if s == "" { - return false - } - } - - // Digits - switch { - default: - return false - - case s[0] == '0': - s = s[1:] - - case '1' <= s[0] && s[0] <= '9': - s = s[1:] - for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { - s = s[1:] - } - } - - // . followed by 1 or more digits. - if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' { - s = s[2:] - for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { - s = s[1:] - } - } - - // e or E followed by an optional - or + and - // 1 or more digits. - if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') { - s = s[1:] - if s[0] == '+' || s[0] == '-' { - s = s[1:] - if s == "" { - return false - } - } - for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { - s = s[1:] - } - } - - // Make sure we are at the end. - return s == "" -} - -// decodeState represents the state while decoding a JSON value. -type decodeState struct { - data []byte - off int // read offset in data - scan scanner - nextscan scanner // for calls to nextValue - savedError error - useNumber bool - ext Extension -} - -// errPhase is used for errors that should not happen unless -// there is a bug in the JSON decoder or something is editing -// the data slice while the decoder executes. -var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") - -func (d *decodeState) init(data []byte) *decodeState { - d.data = data - d.off = 0 - d.savedError = nil - return d -} - -// error aborts the decoding by panicking with err. -func (d *decodeState) error(err error) { - panic(err) -} - -// saveError saves the first err it is called with, -// for reporting at the end of the unmarshal. -func (d *decodeState) saveError(err error) { - if d.savedError == nil { - d.savedError = err - } -} - -// next cuts off and returns the next full JSON value in d.data[d.off:]. -// The next value is known to be an object or array, not a literal. -func (d *decodeState) next() []byte { - c := d.data[d.off] - item, rest, err := nextValue(d.data[d.off:], &d.nextscan) - if err != nil { - d.error(err) - } - d.off = len(d.data) - len(rest) - - // Our scanner has seen the opening brace/bracket - // and thinks we're still in the middle of the object. - // invent a closing brace/bracket to get it out. - if c == '{' { - d.scan.step(&d.scan, '}') - } else if c == '[' { - d.scan.step(&d.scan, ']') - } else { - // Was inside a function name. Get out of it. - d.scan.step(&d.scan, '(') - d.scan.step(&d.scan, ')') - } - - return item -} - -// scanWhile processes bytes in d.data[d.off:] until it -// receives a scan code not equal to op. -// It updates d.off and returns the new scan code. -func (d *decodeState) scanWhile(op int) int { - var newOp int - for { - if d.off >= len(d.data) { - newOp = d.scan.eof() - d.off = len(d.data) + 1 // mark processed EOF with len+1 - } else { - c := d.data[d.off] - d.off++ - newOp = d.scan.step(&d.scan, c) - } - if newOp != op { - break - } - } - return newOp -} - -// value decodes a JSON value from d.data[d.off:] into the value. -// it updates d.off to point past the decoded value. -func (d *decodeState) value(v reflect.Value) { - if !v.IsValid() { - _, rest, err := nextValue(d.data[d.off:], &d.nextscan) - if err != nil { - d.error(err) - } - d.off = len(d.data) - len(rest) - - // d.scan thinks we're still at the beginning of the item. - // Feed in an empty string - the shortest, simplest value - - // so that it knows we got to the end of the value. - if d.scan.redo { - // rewind. - d.scan.redo = false - d.scan.step = stateBeginValue - } - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '"') - - n := len(d.scan.parseState) - if n > 0 && d.scan.parseState[n-1] == parseObjectKey { - // d.scan thinks we just read an object key; finish the object - d.scan.step(&d.scan, ':') - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '}') - } - - return - } - - switch op := d.scanWhile(scanSkipSpace); op { - default: - d.error(errPhase) - - case scanBeginArray: - d.array(v) - - case scanBeginObject: - d.object(v) - - case scanBeginLiteral: - d.literal(v) - - case scanBeginName: - d.name(v) - } -} - -type unquotedValue struct{} - -// valueQuoted is like value but decodes a -// quoted string literal or literal null into an interface value. -// If it finds anything other than a quoted string literal or null, -// valueQuoted returns unquotedValue{}. -func (d *decodeState) valueQuoted() interface{} { - switch op := d.scanWhile(scanSkipSpace); op { - default: - d.error(errPhase) - - case scanBeginArray: - d.array(reflect.Value{}) - - case scanBeginObject: - d.object(reflect.Value{}) - - case scanBeginName: - switch v := d.nameInterface().(type) { - case nil, string: - return v - } - - case scanBeginLiteral: - switch v := d.literalInterface().(type) { - case nil, string: - return v - } - } - return unquotedValue{} -} - -// indirect walks down v allocating pointers as needed, -// until it gets to a non-pointer. -// if it encounters an Unmarshaler, indirect stops and returns that. -// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. -func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { - // If v is a named type and is addressable, - // start with its address, so that if the type has pointer methods, - // we find them. - if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { - v = v.Addr() - } - for { - // Load value from interface, but only if the result will be - // usefully addressable. - if v.Kind() == reflect.Interface && !v.IsNil() { - e := v.Elem() - if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { - v = e - continue - } - } - - if v.Kind() != reflect.Ptr { - break - } - - if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { - break - } - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - if v.Type().NumMethod() > 0 { - if u, ok := v.Interface().(Unmarshaler); ok { - return u, nil, v - } - if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { - return nil, u, v - } - } - v = v.Elem() - } - return nil, nil, v -} - -// array consumes an array from d.data[d.off-1:], decoding into the value v. -// the first byte of the array ('[') has been read already. -func (d *decodeState) array(v reflect.Value) { - // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) - if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) - d.off-- - d.next() - return - } - - v = pv - - // Check type of target. - switch v.Kind() { - case reflect.Interface: - if v.NumMethod() == 0 { - // Decoding into nil interface? Switch to non-reflect code. - v.Set(reflect.ValueOf(d.arrayInterface())) - return - } - // Otherwise it's invalid. - fallthrough - default: - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) - d.off-- - d.next() - return - case reflect.Array: - case reflect.Slice: - break - } - - i := 0 - for { - // Look ahead for ] - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndArray { - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - // Get element of array, growing if necessary. - if v.Kind() == reflect.Slice { - // Grow slice if necessary - if i >= v.Cap() { - newcap := v.Cap() + v.Cap()/2 - if newcap < 4 { - newcap = 4 - } - newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) - reflect.Copy(newv, v) - v.Set(newv) - } - if i >= v.Len() { - v.SetLen(i + 1) - } - } - - if i < v.Len() { - // Decode into element. - d.value(v.Index(i)) - } else { - // Ran out of fixed array: skip. - d.value(reflect.Value{}) - } - i++ - - // Next token must be , or ]. - op = d.scanWhile(scanSkipSpace) - if op == scanEndArray { - break - } - if op != scanArrayValue { - d.error(errPhase) - } - } - - if i < v.Len() { - if v.Kind() == reflect.Array { - // Array. Zero the rest. - z := reflect.Zero(v.Type().Elem()) - for ; i < v.Len(); i++ { - v.Index(i).Set(z) - } - } else { - v.SetLen(i) - } - } - if i == 0 && v.Kind() == reflect.Slice { - v.Set(reflect.MakeSlice(v.Type(), 0, 0)) - } -} - -var nullLiteral = []byte("null") -var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem() - -// object consumes an object from d.data[d.off-1:], decoding into the value v. -// the first byte ('{') of the object has been read already. -func (d *decodeState) object(v reflect.Value) { - // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) - if d.storeKeyed(pv) { - return - } - if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - v = pv - - // Decoding into nil interface? Switch to non-reflect code. - if v.Kind() == reflect.Interface && v.NumMethod() == 0 { - v.Set(reflect.ValueOf(d.objectInterface())) - return - } - - // Check type of target: - // struct or - // map[string]T or map[encoding.TextUnmarshaler]T - switch v.Kind() { - case reflect.Map: - // Map key must either have string kind or be an encoding.TextUnmarshaler. - t := v.Type() - if t.Key().Kind() != reflect.String && - !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - if v.IsNil() { - v.Set(reflect.MakeMap(t)) - } - case reflect.Struct: - - default: - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - - var mapElem reflect.Value - - empty := true - for { - // Read opening " of string key or closing }. - op := d.scanWhile(scanSkipSpace) - if op == scanEndObject { - if !empty && !d.ext.trailingCommas { - d.syntaxError("beginning of object key string") - } - break - } - empty = false - if op == scanBeginName { - if !d.ext.unquotedKeys { - d.syntaxError("beginning of object key string") - } - } else if op != scanBeginLiteral { - d.error(errPhase) - } - unquotedKey := op == scanBeginName - - // Read key. - start := d.off - 1 - op = d.scanWhile(scanContinue) - item := d.data[start : d.off-1] - var key []byte - if unquotedKey { - key = item - // TODO Fix code below to quote item when necessary. - } else { - var ok bool - key, ok = unquoteBytes(item) - if !ok { - d.error(errPhase) - } - } - - // Figure out field corresponding to key. - var subv reflect.Value - destring := false // whether the value is wrapped in a string to be decoded first - - if v.Kind() == reflect.Map { - elemType := v.Type().Elem() - if !mapElem.IsValid() { - mapElem = reflect.New(elemType).Elem() - } else { - mapElem.Set(reflect.Zero(elemType)) - } - subv = mapElem - } else { - var f *field - fields := cachedTypeFields(v.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, key) { - f = ff - break - } - if f == nil && ff.equalFold(ff.nameBytes, key) { - f = ff - } - } - if f != nil { - subv = v - destring = f.quoted - for _, i := range f.index { - if subv.Kind() == reflect.Ptr { - if subv.IsNil() { - subv.Set(reflect.New(subv.Type().Elem())) - } - subv = subv.Elem() - } - subv = subv.Field(i) - } - } - } - - // Read : before value. - if op == scanSkipSpace { - op = d.scanWhile(scanSkipSpace) - } - if op != scanObjectKey { - d.error(errPhase) - } - - // Read value. - if destring { - switch qv := d.valueQuoted().(type) { - case nil: - d.literalStore(nullLiteral, subv, false) - case string: - d.literalStore([]byte(qv), subv, true) - default: - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) - } - } else { - d.value(subv) - } - - // Write value back to map; - // if using struct, subv points into struct already. - if v.Kind() == reflect.Map { - kt := v.Type().Key() - var kv reflect.Value - switch { - case kt.Kind() == reflect.String: - kv = reflect.ValueOf(key).Convert(v.Type().Key()) - case reflect.PtrTo(kt).Implements(textUnmarshalerType): - kv = reflect.New(v.Type().Key()) - d.literalStore(item, kv, true) - kv = kv.Elem() - default: - panic("json: Unexpected key type") // should never occur - } - v.SetMapIndex(kv, subv) - } - - // Next token must be , or }. - op = d.scanWhile(scanSkipSpace) - if op == scanEndObject { - break - } - if op != scanObjectValue { - d.error(errPhase) - } - } -} - -// isNull returns whether there's a null literal at the provided offset. -func (d *decodeState) isNull(off int) bool { - if off+4 >= len(d.data) || d.data[off] != 'n' || d.data[off+1] != 'u' || d.data[off+2] != 'l' || d.data[off+3] != 'l' { - return false - } - d.nextscan.reset() - for i, c := range d.data[off:] { - if i > 4 { - return false - } - switch d.nextscan.step(&d.nextscan, c) { - case scanContinue, scanBeginName: - continue - } - break - } - return true -} - -// name consumes a const or function from d.data[d.off-1:], decoding into the value v. -// the first byte of the function name has been read already. -func (d *decodeState) name(v reflect.Value) { - if d.isNull(d.off - 1) { - d.literal(v) - return - } - - // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) - if d.storeKeyed(pv) { - return - } - if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over function in input - return - } - v = pv - - // Decoding into nil interface? Switch to non-reflect code. - if v.Kind() == reflect.Interface && v.NumMethod() == 0 { - out := d.nameInterface() - if out == nil { - v.Set(reflect.Zero(v.Type())) - } else { - v.Set(reflect.ValueOf(out)) - } - return - } - - nameStart := d.off - 1 - - op := d.scanWhile(scanContinue) - - name := d.data[nameStart : d.off-1] - if op != scanParam { - // Back up so the byte just read is consumed next. - d.off-- - d.scan.undo(op) - if l, ok := d.convertLiteral(name); ok { - d.storeValue(v, l) - return - } - d.error(&SyntaxError{fmt.Sprintf("json: unknown constant %q", name), int64(d.off)}) - } - - funcName := string(name) - funcData := d.ext.funcs[funcName] - if funcData.key == "" { - d.error(fmt.Errorf("json: unknown function %q", funcName)) - } - - // Check type of target: - // struct or - // map[string]T or map[encoding.TextUnmarshaler]T - switch v.Kind() { - case reflect.Map: - // Map key must either have string kind or be an encoding.TextUnmarshaler. - t := v.Type() - if t.Key().Kind() != reflect.String && - !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - if v.IsNil() { - v.Set(reflect.MakeMap(t)) - } - case reflect.Struct: - - default: - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - - // TODO Fix case of func field as map. - //topv := v - - // Figure out field corresponding to function. - key := []byte(funcData.key) - if v.Kind() == reflect.Map { - elemType := v.Type().Elem() - v = reflect.New(elemType).Elem() - } else { - var f *field - fields := cachedTypeFields(v.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, key) { - f = ff - break - } - if f == nil && ff.equalFold(ff.nameBytes, key) { - f = ff - } - } - if f != nil { - for _, i := range f.index { - if v.Kind() == reflect.Ptr { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - v = v.Field(i) - } - if v.Kind() == reflect.Ptr { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - } - } - - // Check for unmarshaler on func field itself. - u, _, _ = d.indirect(v, false) - if u != nil { - d.off = nameStart - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - - var mapElem reflect.Value - - // Parse function arguments. - for i := 0; ; i++ { - // closing ) - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - if i >= len(funcData.args) { - d.error(fmt.Errorf("json: too many arguments for function %s", funcName)) - } - key := []byte(funcData.args[i]) - - // Figure out field corresponding to key. - var subv reflect.Value - destring := false // whether the value is wrapped in a string to be decoded first - - if v.Kind() == reflect.Map { - elemType := v.Type().Elem() - if !mapElem.IsValid() { - mapElem = reflect.New(elemType).Elem() - } else { - mapElem.Set(reflect.Zero(elemType)) - } - subv = mapElem - } else { - var f *field - fields := cachedTypeFields(v.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, key) { - f = ff - break - } - if f == nil && ff.equalFold(ff.nameBytes, key) { - f = ff - } - } - if f != nil { - subv = v - destring = f.quoted - for _, i := range f.index { - if subv.Kind() == reflect.Ptr { - if subv.IsNil() { - subv.Set(reflect.New(subv.Type().Elem())) - } - subv = subv.Elem() - } - subv = subv.Field(i) - } - } - } - - // Read value. - if destring { - switch qv := d.valueQuoted().(type) { - case nil: - d.literalStore(nullLiteral, subv, false) - case string: - d.literalStore([]byte(qv), subv, true) - default: - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) - } - } else { - d.value(subv) - } - - // Write value back to map; - // if using struct, subv points into struct already. - if v.Kind() == reflect.Map { - kt := v.Type().Key() - var kv reflect.Value - switch { - case kt.Kind() == reflect.String: - kv = reflect.ValueOf(key).Convert(v.Type().Key()) - case reflect.PtrTo(kt).Implements(textUnmarshalerType): - kv = reflect.New(v.Type().Key()) - d.literalStore(key, kv, true) - kv = kv.Elem() - default: - panic("json: Unexpected key type") // should never occur - } - v.SetMapIndex(kv, subv) - } - - // Next token must be , or ). - op = d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - if op != scanParam { - d.error(errPhase) - } - } -} - -// keyed attempts to decode an object or function using a keyed doc extension, -// and returns the value and true on success, or nil and false otherwise. -func (d *decodeState) keyed() (interface{}, bool) { - if len(d.ext.keyed) == 0 { - return nil, false - } - - unquote := false - - // Look-ahead first key to check for a keyed document extension. - d.nextscan.reset() - var start, end int - for i, c := range d.data[d.off-1:] { - switch op := d.nextscan.step(&d.nextscan, c); op { - case scanSkipSpace, scanContinue, scanBeginObject: - continue - case scanBeginLiteral, scanBeginName: - unquote = op == scanBeginLiteral - start = i - continue - } - end = i - break - } - - name := bytes.Trim(d.data[d.off-1+start:d.off-1+end], " \n\t") - - var key []byte - var ok bool - if unquote { - key, ok = unquoteBytes(name) - if !ok { - d.error(errPhase) - } - } else { - funcData, ok := d.ext.funcs[string(name)] - if !ok { - return nil, false - } - key = []byte(funcData.key) - } - - decode, ok := d.ext.keyed[string(key)] - if !ok { - return nil, false - } - - d.off-- - out, err := decode(d.next()) - if err != nil { - d.error(err) - } - return out, true -} - -func (d *decodeState) storeKeyed(v reflect.Value) bool { - keyed, ok := d.keyed() - if !ok { - return false - } - d.storeValue(v, keyed) - return true -} - -var ( - trueBytes = []byte("true") - falseBytes = []byte("false") - nullBytes = []byte("null") -) - -func (d *decodeState) storeValue(v reflect.Value, from interface{}) { - switch from { - case nil: - d.literalStore(nullBytes, v, false) - return - case true: - d.literalStore(trueBytes, v, false) - return - case false: - d.literalStore(falseBytes, v, false) - return - } - fromv := reflect.ValueOf(from) - for fromv.Kind() == reflect.Ptr && !fromv.IsNil() { - fromv = fromv.Elem() - } - fromt := fromv.Type() - for v.Kind() == reflect.Ptr && !v.IsNil() { - v = v.Elem() - } - vt := v.Type() - if fromt.AssignableTo(vt) { - v.Set(fromv) - } else if fromt.ConvertibleTo(vt) { - v.Set(fromv.Convert(vt)) - } else { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - } -} - -func (d *decodeState) convertLiteral(name []byte) (interface{}, bool) { - if len(name) == 0 { - return nil, false - } - switch name[0] { - case 't': - if bytes.Equal(name, trueBytes) { - return true, true - } - case 'f': - if bytes.Equal(name, falseBytes) { - return false, true - } - case 'n': - if bytes.Equal(name, nullBytes) { - return nil, true - } - } - if l, ok := d.ext.consts[string(name)]; ok { - return l, true - } - return nil, false -} - -// literal consumes a literal from d.data[d.off-1:], decoding into the value v. -// The first byte of the literal has been read already -// (that's how the caller knows it's a literal). -func (d *decodeState) literal(v reflect.Value) { - // All bytes inside literal return scanContinue op code. - start := d.off - 1 - op := d.scanWhile(scanContinue) - - // Scan read one byte too far; back up. - d.off-- - d.scan.undo(op) - - d.literalStore(d.data[start:d.off], v, false) -} - -// convertNumber converts the number literal s to a float64 or a Number -// depending on the setting of d.useNumber. -func (d *decodeState) convertNumber(s string) (interface{}, error) { - if d.useNumber { - return Number(s), nil - } - f, err := strconv.ParseFloat(s, 64) - if err != nil { - return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} - } - return f, nil -} - -var numberType = reflect.TypeOf(Number("")) - -// literalStore decodes a literal stored in item into v. -// -// fromQuoted indicates whether this literal came from unwrapping a -// string from the ",string" struct tag option. this is used only to -// produce more helpful error messages. -func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { - // Check for unmarshaler. - if len(item) == 0 { - //Empty string given - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - return - } - wantptr := item[0] == 'n' // null - u, ut, pv := d.indirect(v, wantptr) - if u != nil { - err := u.UnmarshalJSON(item) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - if item[0] != '"' { - if fromQuoted { - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - } - return - } - s, ok := unquoteBytes(item) - if !ok { - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) - } - } - err := ut.UnmarshalText(s) - if err != nil { - d.error(err) - } - return - } - - v = pv - - switch c := item[0]; c { - case 'n': // null - switch v.Kind() { - case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: - v.Set(reflect.Zero(v.Type())) - // otherwise, ignore null for primitives/string - } - case 't', 'f': // true, false - value := c == 't' - switch v.Kind() { - default: - if fromQuoted { - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) - } - case reflect.Bool: - v.SetBool(value) - case reflect.Interface: - if v.NumMethod() == 0 { - v.Set(reflect.ValueOf(value)) - } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) - } - } - - case '"': // string - s, ok := unquoteBytes(item) - if !ok { - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) - } - } - switch v.Kind() { - default: - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - case reflect.Slice: - if v.Type().Elem().Kind() != reflect.Uint8 { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - break - } - b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) - n, err := base64.StdEncoding.Decode(b, s) - if err != nil { - d.saveError(err) - break - } - v.SetBytes(b[:n]) - case reflect.String: - v.SetString(string(s)) - case reflect.Interface: - if v.NumMethod() == 0 { - v.Set(reflect.ValueOf(string(s))) - } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - } - } - - default: // number - if c != '-' && (c < '0' || c > '9') { - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) - } - } - s := string(item) - switch v.Kind() { - default: - if v.Kind() == reflect.String && v.Type() == numberType { - v.SetString(s) - if !isValidNumber(s) { - d.error(fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item)) - } - break - } - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) - } - case reflect.Interface: - n, err := d.convertNumber(s) - if err != nil { - d.saveError(err) - break - } - if v.NumMethod() != 0 { - d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) - break - } - v.Set(reflect.ValueOf(n)) - - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - n, err := strconv.ParseInt(s, 10, 64) - if err != nil || v.OverflowInt(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) - break - } - v.SetInt(n) - - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - n, err := strconv.ParseUint(s, 10, 64) - if err != nil || v.OverflowUint(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) - break - } - v.SetUint(n) - - case reflect.Float32, reflect.Float64: - n, err := strconv.ParseFloat(s, v.Type().Bits()) - if err != nil || v.OverflowFloat(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) - break - } - v.SetFloat(n) - } - } -} - -// The xxxInterface routines build up a value to be stored -// in an empty interface. They are not strictly necessary, -// but they avoid the weight of reflection in this common case. - -// valueInterface is like value but returns interface{} -func (d *decodeState) valueInterface() interface{} { - switch d.scanWhile(scanSkipSpace) { - default: - d.error(errPhase) - panic("unreachable") - case scanBeginArray: - return d.arrayInterface() - case scanBeginObject: - return d.objectInterface() - case scanBeginLiteral: - return d.literalInterface() - case scanBeginName: - return d.nameInterface() - } -} - -func (d *decodeState) syntaxError(expected string) { - msg := fmt.Sprintf("invalid character '%c' looking for %s", d.data[d.off-1], expected) - d.error(&SyntaxError{msg, int64(d.off)}) -} - -// arrayInterface is like array but returns []interface{}. -func (d *decodeState) arrayInterface() []interface{} { - var v = make([]interface{}, 0) - for { - // Look ahead for ] - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndArray { - if len(v) > 0 && !d.ext.trailingCommas { - d.syntaxError("beginning of value") - } - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - v = append(v, d.valueInterface()) - - // Next token must be , or ]. - op = d.scanWhile(scanSkipSpace) - if op == scanEndArray { - break - } - if op != scanArrayValue { - d.error(errPhase) - } - } - return v -} - -// objectInterface is like object but returns map[string]interface{}. -func (d *decodeState) objectInterface() interface{} { - v, ok := d.keyed() - if ok { - return v - } - - m := make(map[string]interface{}) - for { - // Read opening " of string key or closing }. - op := d.scanWhile(scanSkipSpace) - if op == scanEndObject { - if len(m) > 0 && !d.ext.trailingCommas { - d.syntaxError("beginning of object key string") - } - break - } - if op == scanBeginName { - if !d.ext.unquotedKeys { - d.syntaxError("beginning of object key string") - } - } else if op != scanBeginLiteral { - d.error(errPhase) - } - unquotedKey := op == scanBeginName - - // Read string key. - start := d.off - 1 - op = d.scanWhile(scanContinue) - item := d.data[start : d.off-1] - var key string - if unquotedKey { - key = string(item) - } else { - var ok bool - key, ok = unquote(item) - if !ok { - d.error(errPhase) - } - } - - // Read : before value. - if op == scanSkipSpace { - op = d.scanWhile(scanSkipSpace) - } - if op != scanObjectKey { - d.error(errPhase) - } - - // Read value. - m[key] = d.valueInterface() - - // Next token must be , or }. - op = d.scanWhile(scanSkipSpace) - if op == scanEndObject { - break - } - if op != scanObjectValue { - d.error(errPhase) - } - } - return m -} - -// literalInterface is like literal but returns an interface value. -func (d *decodeState) literalInterface() interface{} { - // All bytes inside literal return scanContinue op code. - start := d.off - 1 - op := d.scanWhile(scanContinue) - - // Scan read one byte too far; back up. - d.off-- - d.scan.undo(op) - item := d.data[start:d.off] - - switch c := item[0]; c { - case 'n': // null - return nil - - case 't', 'f': // true, false - return c == 't' - - case '"': // string - s, ok := unquote(item) - if !ok { - d.error(errPhase) - } - return s - - default: // number - if c != '-' && (c < '0' || c > '9') { - d.error(errPhase) - } - n, err := d.convertNumber(string(item)) - if err != nil { - d.saveError(err) - } - return n - } -} - -// nameInterface is like function but returns map[string]interface{}. -func (d *decodeState) nameInterface() interface{} { - v, ok := d.keyed() - if ok { - return v - } - - nameStart := d.off - 1 - - op := d.scanWhile(scanContinue) - - name := d.data[nameStart : d.off-1] - if op != scanParam { - // Back up so the byte just read is consumed next. - d.off-- - d.scan.undo(op) - if l, ok := d.convertLiteral(name); ok { - return l - } - d.error(&SyntaxError{fmt.Sprintf("json: unknown constant %q", name), int64(d.off)}) - } - - funcName := string(name) - funcData := d.ext.funcs[funcName] - if funcData.key == "" { - d.error(fmt.Errorf("json: unknown function %q", funcName)) - } - - m := make(map[string]interface{}) - for i := 0; ; i++ { - // Look ahead for ) - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - if i >= len(funcData.args) { - d.error(fmt.Errorf("json: too many arguments for function %s", funcName)) - } - m[funcData.args[i]] = d.valueInterface() - - // Next token must be , or ). - op = d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - if op != scanParam { - d.error(errPhase) - } - } - return map[string]interface{}{funcData.key: m} -} - -// getu4 decodes \uXXXX from the beginning of s, returning the hex value, -// or it returns -1. -func getu4(s []byte) rune { - if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { - return -1 - } - r, err := strconv.ParseUint(string(s[2:6]), 16, 64) - if err != nil { - return -1 - } - return rune(r) -} - -// unquote converts a quoted JSON string literal s into an actual string t. -// The rules are different than for Go, so cannot use strconv.Unquote. -func unquote(s []byte) (t string, ok bool) { - s, ok = unquoteBytes(s) - t = string(s) - return -} - -func unquoteBytes(s []byte) (t []byte, ok bool) { - if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { - return - } - s = s[1 : len(s)-1] - - // Check for unusual characters. If there are none, - // then no unquoting is needed, so return a slice of the - // original bytes. - r := 0 - for r < len(s) { - c := s[r] - if c == '\\' || c == '"' || c < ' ' { - break - } - if c < utf8.RuneSelf { - r++ - continue - } - rr, size := utf8.DecodeRune(s[r:]) - if rr == utf8.RuneError && size == 1 { - break - } - r += size - } - if r == len(s) { - return s, true - } - - b := make([]byte, len(s)+2*utf8.UTFMax) - w := copy(b, s[0:r]) - for r < len(s) { - // Out of room? Can only happen if s is full of - // malformed UTF-8 and we're replacing each - // byte with RuneError. - if w >= len(b)-2*utf8.UTFMax { - nb := make([]byte, (len(b)+utf8.UTFMax)*2) - copy(nb, b[0:w]) - b = nb - } - switch c := s[r]; { - case c == '\\': - r++ - if r >= len(s) { - return - } - switch s[r] { - default: - return - case '"', '\\', '/', '\'': - b[w] = s[r] - r++ - w++ - case 'b': - b[w] = '\b' - r++ - w++ - case 'f': - b[w] = '\f' - r++ - w++ - case 'n': - b[w] = '\n' - r++ - w++ - case 'r': - b[w] = '\r' - r++ - w++ - case 't': - b[w] = '\t' - r++ - w++ - case 'u': - r-- - rr := getu4(s[r:]) - if rr < 0 { - return - } - r += 6 - if utf16.IsSurrogate(rr) { - rr1 := getu4(s[r:]) - if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { - // A valid pair; consume. - r += 6 - w += utf8.EncodeRune(b[w:], dec) - break - } - // Invalid surrogate; fall back to replacement rune. - rr = unicode.ReplacementChar - } - w += utf8.EncodeRune(b[w:], rr) - } - - // Quote, control characters are invalid. - case c == '"', c < ' ': - return - - // ASCII - case c < utf8.RuneSelf: - b[w] = c - r++ - w++ - - // Coerce to well-formed UTF-8. - default: - rr, size := utf8.DecodeRune(s[r:]) - r += size - w += utf8.EncodeRune(b[w:], rr) - } - } - return b[0:w], true -} diff --git a/src/runtime/vendor/github.com/globalsign/mgo/internal/json/encode.go b/src/runtime/vendor/github.com/globalsign/mgo/internal/json/encode.go deleted file mode 100644 index e4b8f86487cd..000000000000 --- a/src/runtime/vendor/github.com/globalsign/mgo/internal/json/encode.go +++ /dev/null @@ -1,1260 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package json implements encoding and decoding of JSON as defined in -// RFC 4627. The mapping between JSON and Go values is described -// in the documentation for the Marshal and Unmarshal functions. -// -// See "JSON and Go" for an introduction to this package: -// https://golang.org/doc/articles/json_and_go.html -package json - -import ( - "bytes" - "encoding" - "encoding/base64" - "fmt" - "math" - "reflect" - "runtime" - "sort" - "strconv" - "strings" - "sync" - "unicode" - "unicode/utf8" -) - -// Marshal returns the JSON encoding of v. -// -// Marshal traverses the value v recursively. -// If an encountered value implements the Marshaler interface -// and is not a nil pointer, Marshal calls its MarshalJSON method -// to produce JSON. If no MarshalJSON method is present but the -// value implements encoding.TextMarshaler instead, Marshal calls -// its MarshalText method. -// The nil pointer exception is not strictly necessary -// but mimics a similar, necessary exception in the behavior of -// UnmarshalJSON. -// -// Otherwise, Marshal uses the following type-dependent default encodings: -// -// Boolean values encode as JSON booleans. -// -// Floating point, integer, and Number values encode as JSON numbers. -// -// String values encode as JSON strings coerced to valid UTF-8, -// replacing invalid bytes with the Unicode replacement rune. -// The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" -// to keep some browsers from misinterpreting JSON output as HTML. -// Ampersand "&" is also escaped to "\u0026" for the same reason. -// This escaping can be disabled using an Encoder with DisableHTMLEscaping. -// -// Array and slice values encode as JSON arrays, except that -// []byte encodes as a base64-encoded string, and a nil slice -// encodes as the null JSON value. -// -// Struct values encode as JSON objects. Each exported struct field -// becomes a member of the object unless -// - the field's tag is "-", or -// - the field is empty and its tag specifies the "omitempty" option. -// The empty values are false, 0, any -// nil pointer or interface value, and any array, slice, map, or string of -// length zero. The object's default key string is the struct field name -// but can be specified in the struct field's tag value. The "json" key in -// the struct field's tag value is the key name, followed by an optional comma -// and options. Examples: -// -// // Field is ignored by this package. -// Field int `json:"-"` -// -// // Field appears in JSON as key "myName". -// Field int `json:"myName"` -// -// // Field appears in JSON as key "myName" and -// // the field is omitted from the object if its value is empty, -// // as defined above. -// Field int `json:"myName,omitempty"` -// -// // Field appears in JSON as key "Field" (the default), but -// // the field is skipped if empty. -// // Note the leading comma. -// Field int `json:",omitempty"` -// -// The "string" option signals that a field is stored as JSON inside a -// JSON-encoded string. It applies only to fields of string, floating point, -// integer, or boolean types. This extra level of encoding is sometimes used -// when communicating with JavaScript programs: -// -// Int64String int64 `json:",string"` -// -// The key name will be used if it's a non-empty string consisting of -// only Unicode letters, digits, dollar signs, percent signs, hyphens, -// underscores and slashes. -// -// Anonymous struct fields are usually marshaled as if their inner exported fields -// were fields in the outer struct, subject to the usual Go visibility rules amended -// as described in the next paragraph. -// An anonymous struct field with a name given in its JSON tag is treated as -// having that name, rather than being anonymous. -// An anonymous struct field of interface type is treated the same as having -// that type as its name, rather than being anonymous. -// -// The Go visibility rules for struct fields are amended for JSON when -// deciding which field to marshal or unmarshal. If there are -// multiple fields at the same level, and that level is the least -// nested (and would therefore be the nesting level selected by the -// usual Go rules), the following extra rules apply: -// -// 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, -// even if there are multiple untagged fields that would otherwise conflict. -// 2) If there is exactly one field (tagged or not according to the first rule), that is selected. -// 3) Otherwise there are multiple fields, and all are ignored; no error occurs. -// -// Handling of anonymous struct fields is new in Go 1.1. -// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of -// an anonymous struct field in both current and earlier versions, give the field -// a JSON tag of "-". -// -// Map values encode as JSON objects. The map's key type must either be a string -// or implement encoding.TextMarshaler. The map keys are used as JSON object -// keys, subject to the UTF-8 coercion described for string values above. -// -// Pointer values encode as the value pointed to. -// A nil pointer encodes as the null JSON value. -// -// Interface values encode as the value contained in the interface. -// A nil interface value encodes as the null JSON value. -// -// Channel, complex, and function values cannot be encoded in JSON. -// Attempting to encode such a value causes Marshal to return -// an UnsupportedTypeError. -// -// JSON cannot represent cyclic data structures and Marshal does not -// handle them. Passing cyclic structures to Marshal will result in -// an infinite recursion. -// -func Marshal(v interface{}) ([]byte, error) { - e := &encodeState{} - err := e.marshal(v, encOpts{escapeHTML: true}) - if err != nil { - return nil, err - } - return e.Bytes(), nil -} - -// MarshalIndent is like Marshal but applies Indent to format the output. -func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { - b, err := Marshal(v) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = Indent(&buf, b, prefix, indent) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 -// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 -// so that the JSON will be safe to embed inside HTML

LpT`qc~hypC1k}wLe1vaBld8BC#V7e zKwpFygWVD<#`G0O$f+cQ?oujCvnDYgpe-?^oYtI>>}N%5G|hxVZb_?Yj#N2rEt>Rp z(V<3SIxXY!Ix|GoSBeUCnR*o;u$NFGYYakObIRayXQMfYt0huOB$5U0c2dQZcq&>A z0GUQ_C{Y(>G-1>6L9hyd7|>MAtPlEYIyejn0xech#B8rdt+36Vbw^k$ z7H2$Mx>(??{uIolfanEI&RS!fnRBEuA74l5Oe|Cl!j+uO4bUDh$H13GwSXjv!^UL^ zE3dN}!+zeEr9kxOW2`!z=CBxYJQ~-6QJbSeGC^5yffOjh&{?w~P}L=Jm1xCm^UDTf zUCoRZpPqO|3#AjlO6w)e0Et)W+f6%U{xH;CQ89-upU2OjoO`5QqG&boEKpgN-!gYS*g4F6`z?@VvYc4#*JvfU=SF6B3)&; z`cm>i4rF};uuo*vNHoZqqyC}?MM`2#a9If(hLK2x;Po!mAc?3el)=inte-NZ<+82< z+RlXpmnd^_r#luwWjEo;YgB@Wd8~QO!6aP%oB<t_>Wg5C7u9hg6jD)GwV7RH ztW*GkjThr}k${Y8R&y0haEf=P@mgKUaYC=l>I+B?Lnx;$r&je9cU^~*?mDV5U?jM~ zh8h642o8H%r`fBL;HcRVmJ5q6lQXIXNf9n$PCJ)P@uhTvlk5s+x93PN5}?h6G{zc` z6ckRH3pSUljQQ$)GHTG0vhs}1FDeO_UG}?@8tfyLIGHzVxezZ~G15}<<``N{`T-;3 zn!p-iwweigy(WK7Me_s}@@8-~YIa2MPy|iEbzsuN^~ODRs}sVqz#qo-h(HDPakb*8 zbGARF2N7Q!3Sb_aY9McCwGgGcd@#(xW`i*#9INWlkf>)0`V^Uiva-u>cC(O!R8s`0+Po>r8wX)k zsAP`%L{BnE7cwBxX?J6KUx>9QY61%q`zp{T6iJ|vV5M*ljMZqBk_?S*#0Z~2rPByM)uiVnV2+JjlbKlA zY;}iVqZX}GrXOZKL7l|8?13Uw)>&YbXY`a%(5yzrC70Qf#?(rD1xx6I0v!~>5iwCG zAG5)sc|BlRP+i3d8{o8RF+q4CLW6fwj`sMVI%(ug2@?c@Fcc43Q+AAY#l2RYJt>kA z15AWN;fS7zRgsig(=4)}K$dc%P^cC`j0!VRtD}Gb%*9Q-&mgAkyk>DY!nHJ1F2Nj{ z(3_KxMbud;V!&6YZLez2S`&yZs7vdV0OhJgV!(_%ltCZ&j11=&yx0+a@l6fR#GSzkW#C#Y<1Un{+w=*h=K&nYIRu3agZKIOqEL2 z7YVRM8cGqhJgQ=4#i>MaFnGM=;~*ka6g{*lhN+r>16ij_WI-ATG?^{ng;ItKx!7Vf=J93adKxtD zi6lHl7X+LdG>{77F}2`M#f?zXXfzc}Cc9a4SP?nSfHMi^lFwy`^0LwGOd_SEIq7y3 zK|F+)i-s647>zKYSecBzrl=ksclasrs-DXNkWwgnMSU$73HuSVR3B6|oQ2{7Q!>yd z8Of?nkQw(zgFx|;HCrf?5=F+DuZa>RvPIb>WSq8e0!$36N{9oX5vGAQ>SH(6L{BO2 zp^L$MxFn#EOcvbc5TGbDr$Y)@9I9(G352a7FqyFW(PEYe7+7;Or3Uz#udEjflrdL9 zErpOrD>!{PkptC@A;BLZLS9`?0x~u-YBbJxD;A%Mka3i(B!q;OhE!JKWE++8AXY|S z5OPpWGvsNL26ki4}_;^`UGw7T56!JSPZ@9>v78 zzvlD${kEWkcUX1c&zi_rnY1!TG!|?5~7jNndc7;^6QJRILMgEk}!KZ!|!ps~%48 z$E-PpfwNhgk){$jY%V(M>hA#AuLMbYb6K5*WT2p2EZ0EPfu}GiU-B{8cwOW!J_`&H z=riicIE;ZfMTG4k9|%AYDxfCPDvbK0m|bwN80Co2Acw=oQ;0D|C$flUvs8ly$_^30 zB5+1DTTE~m0H{D13^mGXF5-9@&jxFSlHZuob0u3OP({Ra!WjX3OTJPmY|!+97+rJL z4;i3fb0+0>)Z|2whk>TxupEZSH3Tnh^kRqyaTZ&v*rOtjqH#$qh3b;x zYp~rAHn2)56-NR!yubcuBDmWNnA$D{^-xJMAr5av zLR={iIN@OAv!<+KD1r$Ea{(&^HuXin<~7?(ECJPPJM1hAM5Y>Qgagi6ePP&FaMTge z0vx{oC(ip%WceRP`<=i36H$smdo--6kSjv*K_r!WNsclu@EO9bq$+ zvlf+i6z%?$Q=da4Wp6oMLuwdMwmN)d%`v^c>~?2C5QwD;1v)0C0uE~+8lyvr5N^-r zSt(v()4_5=WUcnJgwWQghjOaG)=4VN?MRBWg{=ttvKkESi8=f zR}2mV*uhEEOO~K4TsF|U6d9~x5Vl4G-`2rc@VoZtWqm@TEss>ESVgHi>_LEM=J z`yPd&0c0gIP9Ij_83SC9QhpkZB%^{DO$X&<9Ka6hb=s=D)mOq0Sxq`slaI&BIoj!< zs%b5orDHYCpH;X}J-k*OwWPi5_Xh0&a7ROtRF13%8Q`>o#UDc{BwydFOob6I`17YZ zL_rJ)rbax0IBNlXMgcxU5-t!00FY$eV5cjdwV^qz#)ADAM=Y$c2wg++C5(<(Tu}sM z;DSK^0Nvn(%|_NTep^f_k!28k)y>9Ua&w6=s-$$FPm5Yhx*@0059l5MwZ0lK$pVqY z@Kid*SSxB6^^y8a(V0MuAm>=3K@?q*A=2UG>pW5uoG#MM)dMLN#E#(f7}%EsHw6^T zLoC_yt|S7)Z8zc`)=b7TrdZJ~a1l6C1p8ePTiB~d!jZI)SL`J2C1ZfK(h3rHcwH(< zqt#>$*g1iO0zuJJptXE7#G3#|%5s6iSu{+6Jf$fV%~#NfVDKWk5atsBEa6dy1vqI{ zbETZf*0T+@yrg#+3|@l`oCeW(fCTb_i*}=URz%=ZHCW2?nJ89I(4maC7>6iR!Bs$f zMGrVK=>nvJr)!lQo#XW=RVJbg$GQ@@o40#iiMS1Ng3WF;S+JxPgI5bMWpD&AQ#QtE z6olZYJC%qyI0iG5u(2x2w9e1*b$o)Osk*Awph5to`~yl^l2UeG-kVHW?BMhwo{bhI zK@Ya-ExP)sk|@}M*2{Wa3q@I<0;TeyxY3`cloS_Kqb4l{_DaAB2e`x$Sqp+joH_&K zM|C;A?x*HVRK>!>$zYKIi4+|#lT;bVOcQeaSW{V##5E!pqZ1Hg19>1-4@t#(Bp)agikYCni3KH~K}jhbOoJnDG+J^+ z0%2dsmQEyTLUHF!6?Y;)i8^rRQjv9r0@&*Zyskv?pb`q{Jrx>I92}7$%_Mn@7YFSG zTjxrK<%|V0*fYn&Io?pw=sd@$ws=W3qa_&Z^gBQ%HCGQGk&43@DHr@Y1DimEbcC*_ zu~4!A$5OoC2*aoX68XNIkHhP38E{)1jL2kk;J}d4p|8URXd{z~F?twWfR7(i+y+^f zC;Yi+q-M6Edb4Pt6E17C=JE2N0O8~d`jRe`$ySRMJ$O)Od@JG3fpc$0Kn^IMG1rZv zY_Sxq^(8E#Rhf6Exw_QAEhoV~LsFk`8K|JeYViSF!Z9cXBN3k$L+Jn|!)3l6xX2)s zkHVRfj`2z+;E^&4u8C5_VKaXpXy3}VAD^ZculPBhf{PN@L=%SiM}5{hvVz-YTIe? z9XG5Q`P3V4wCnur^c#F*yT%@9WB2{@+tsSDRITu8WtKK;=$RcychdZddGj9J+^f9$ z?{gQtz9^nP-JibT@A>||DTCWS)^Wr07l_gOMuC5?9vFPnx#9oVb^ZC-kN*B(M9&-h z>mC_$Zp|;FR^NX5>55km)+1&ciWH$)>Nl=-95@Z;-A4^um0hh)}014w7B%^Z(RqB47FC|ryE+f z>pJqkzs8OmX<7SxL$6-FDD=-YJKgnYLyPd$Uo3C@bIp+hPPS~#m>Kice17K(O;bPJ_tBDf zp~>2oMW0Nea(&agdoMVLZJe-q%GZI7D>e_iFlUBmUTu62XLJhI&+y7dcVO}rgU>ha z-+j~6J=2d~;BJ$42QC%TYj^pY8?uidpF|Ga^Ty9jGl=e2zL@BSb{lh|armBhzTDlk z;`I4pZ_n7#*w}Kx2a(6GYR=3Ry3Idy+bi9#T=zXcXUv=l%7mYXnD-oN+OIDTnYr}A6EXontUwDeKqOs z$Ju|SjQearB%4&|}GK zc0HMOEXv<=qItu@mwxDII5cHvmGn(5R-*v4CS?K8Jwt{=B- zGk^8+`xecaJ#T1|Uf-fg+%o0DR{zGK+a7x8kui_IQd)aqC%=mwIZ^9<=l*4;Q7^7* z>smPiTDDX--}~9q3(4(0-~aH#RU;tj`&%xRn-iUWLyPf;80 zD;K2G4`Po;DP|C{W7We@ws#?T{g6KQ?pMhPjuo$VpE~vD=bDAjPX2BC=ci6=YdSI) zo7`u{TUQ=C%q$rOJ+xvi;o9D9Plpbl_14W>_vZZWLl*=ljNV@Sh@N-%8(U8;^xOc$yM1};GG^(qcOUP1 zK7J>;dpJkk(z18wO`LpUUf5n^N{Nfp6_=0>ghLKiWseXbsE=j@MiMhFL$kdX|Je&mUjlf=i4^y~ggkNw{r|E|$I(RuB-yVorG^-XI0&jUW|Cq3We z!ob6~jsJQ0n4MFke#^#fS~-lJDs|809(`-(q^0nR;qv0=AABis_Qh?YW$4X=S3P-n z(75djmNa@6zWUtv?|Sfi?NSweukUwz`uf{7i(S_5xclWUZx0Ke--a}eaT?#5J#WVB;a~4$RxMvP z;;W;zCb4wlgD(RC3)zo8`OTt!uT1^?i`OT$oWH*5iNvvM6YspWesB;v@m6Kqw-i)b zxBAr;E4O|5?V(fiyLWgv&~eA-!zLfMoJ;zD&3NC;AF1xTvC9;8VGWtU;UjXAC{5o=X z(^B~D$K7*2UeNUI@!=EiR*pY;&zu(H*B!X8F?sjP*G+Pb3=M8-85|Je6q@_YPd~{-aBFe>UN^?XymqZ$00z_T-z3mbKmf?4l32#dG_CAx zi^o=Hof|cNPy!cInT}X|d&_ruj$C$3MJc@u$6;U7xSt{@Sp;p;axyuO6M&=IYs(l5wZ>sKm%| z?S`~pGDUZy&**LUY!AG>>h4kg14~By!h4hLo6jv?5ok5vUfcEr_hI|y%s}sj+hSdQ zJk)dlJJ&uic}~K0t~1e(yuj~(H&0QY>b~Ro-i2X}pFVn9hxWgIJYuczz{)#5J!xLM z*KuX@p_>LSO|P|F^YMpG;_1`@ZrY%`_M6GGFO8U#8yWi7St0%be5o1Q*{-G*LFZ_SV2-jiI!_S=8eh>_z~To(TRoY=DW{a^QAe9wf&(dW-B zpE_#Kz4uSnwi_NCxLzP;?HP5}wP?Uyg{`+myZpFkDAS2wJm;M2+Ee3A??sx|-L+U) z!#~Vclik7)v7%uuKlJppj@if7p+9ZdzLl|x%iU$??}zq9h0 ze$CJeiJPDPTG_hy#I(>?d)>wl&z<#w1@8UHr0;s|NRCXN_5IouIQm@o;q3WIhb7*) z`_Pq-pHDt(t|i(x2X-~2bK!5AAGl%hx)ajTUPGsjdFkh8kM>y5az46qb&uOuU24YD zySIL=(~ukAYGa(!V#EHXBlo`0(5w8_FYb;L`gm`CQ5c!boq7jacV*oQKAUvYeS3cT z;?xU?vlAP~ywiB%o)NwNdFXh@i02w-Sl!@mvBl6NE>Fj3t=Ic47YV~R1XgT&_+rQX zr=9;UdMWhXaVNm;wI_8izxdjju`^!0y!oYdZvBNT=EvGF(AwVRlM~l{zNqV>Wiw`X zzje^sX;a)S#$Oj7I`hP-al+a*hdpCo?9LAA?S99y=aJv$HvXY0V=lFGQ=9fD_9CmP z0|VP-qv*u9w_NDa@9d#Q@|ELPG#@(dZo93ut+Mi&#?eb_zwTupK&OG@Ri4Rz6$APmpLCO2~n>SvoESWKH!YQuCLDTj=a&p$HX~U)szU$Ki zW3>nRe98-rSKXg`q5I^2#wS15@snd;emZOF(sLWPH??axyzV)uL!T2?-^OQFbzDYG zdhVt8b@LwLPc$#Ps>gx&`PqAYZ?7J-dEIxJKIs(;hYg(jOY(`cv&vx~^_?CFO^WqNz11{{hI2SeMpE16~jcQuCfBo%a zPkWbavvdit(|_nM(nl9JZae-_`7=wGW&Pgj{N(Tv%`?1fTlJc^dj**3@ns*cU(AMH zr1^W<>Nw5YYX|@I;fFm#4gEq_E}Ab4Bxk?0y0BsJmKJ+n-Fcxq2m~LWF?87Cw+?;t zb@xHNKRuHiwfo~8XTCgiXe++)vL$!*Kh~S|r6+IMaqRUO5AIof@X(FU;SYAc_o^zg zvD@9J=HC11)SD-d;`f~v&TO0W{P+!Xhpry>)v4I7)m?&9{&uPI{m=%4kg>hAZ|&vA z!?xfn)7v1cU5kEwMO^y%4P?%7=B^GWS9b3=wZ{hs{Hv68EoUKn_G}G&@J?gDd-r=u zg4E`|Jap^nF3Q%)KZ_S$cR#s##OasrUHF2s{HSr>V|`|?P40gPI`IJ8)aTauN1FFc zhFsE<8S1mQgm9NZFC4qJ!{vwn-gNl}aqVY&*%fn!{Ztt-30iwJPTty`?X;(V;Oxv5 z4-Rr|yceAH`Q+^E2kC7q3b$SALi?|~iy0ZZqU*qKm#pmGp|5!3#C1QOp7-t@OJ|Rl z+UL#&fNR~?-cPD{Zzhc`w-27%Ny?Wy4un!Sujn~mT5->W;-y+WV7k)UdEc6QUXfpG zyp*}SjvVlL8*$r^FI?^7l>4Wv{~|?j{OYc2J|mvb{?)sd*LPjhe#*p~FD1J=3jT0s zr%jVD`)STKU*B;FC|-*bP_U#E6UKw{yIcb@#L+n?*b+2zNFA9Reo?w~O8 zqf455AOa>N08Bd+H^T*~QyYz7E}6^*%b7p>8;^{zbE!S~Oa>bf@?IKMQZvj~zM>ErG_|LkQOTJ`UE z?$pQ&w;ey#V#Mu+iwU!l#~OCG_;&{FsjsVRlBT_F+zX!npj9pP&eVs$+~9e8$Jw4E zK6-55sA-p-AG&5l$J1v=uI_l_@AQT?Tp)vm`nbeK@1ySCUO^mkK-=IvRu;HAee zfvb!1D2{^Wi3c9Y6(1gQ;LyCWlLP(!Y6XdwPqRy}0;i2$oYAiNmU5>*xtKr&3fDj7 z9+|xL!_M!$-G9MbxBp2>-{13H^>R*UYz7*Arn*u7b3X&x-?PRkzI`MCcN;q7*c*SL z*T=4HSG>Gii|_-}?v6h9=jm9ls}JGte8=9ihkLodFsDT8Tkh@l^LA-%+gH}Fxns@W zKL5#%{nS^y<9fnsN3&E7Oz?t#2Fx zFYG;Z2;6h#fIrW*Yv-0opKi*x)t0^Y&A9fdKhMREcJ17vi>uw8$9nt8Ki6Ngd_aS% z-O33ax8KH49b5LEJ>}Xb{-P1QtD(jCrvVZeHe7cd_7}r!Z+#fO;qxncgBFHUe@n)Q*1z=s;oR-T z^BrsFFAy(HyYrDTzrf$`djq;~>nDnwAF)Fiyl8jh=x?7p+-Ec;47qwpa@VU<{vwb3 z!yN-gj(g_iqc@#g^Vv0H*ZzB@M?H1<=vu?Adgt6{E{bfx4{cg9=)Wder;Wl!UUyn} zuI(SBvZQm@HFK}KzUOa-?c3NAY0IqtCr}Oe?lym+pD%Q3m+9Dh-*Y|J6e;JZ{|M^p zF2C$P)!%gH@256ths~S$pP;Md%R|3G>~wyVqczjJ+x+Es!ma+W8t@ceyf1a+RB2wn zH>sWXWwyTY!cCU+jO$)rIQ=8fOZE8F-t7;wa)#^z{HjiqU_r{=qth=ZO?Ynmhj@;A8E*O8!qS4I*yseh~ zK?$+$KYsbk^NM?R+}M^m`C$Y{*i~(bZ0(HD@QP1Vq{{b8W{@!NQnRC5vefMm)&7i$K@AluDedo%h z^MdEQ5x36I9({Z1mQzX@d0rMWo1XFmDG#{&vbq$yZfg0$<<0IF8b`nS$oMyI`FDGr z;89|$-r22JpS^`mubzJ;v~9q>@17p^<@RA)j`cgS_eaa+&41g{>YE$t#Je{&dBjYkv|#JwXudlpp99*a$DjZ9~d8;FSi}PvQPet-H(|c z`MEmWa`=8TvtZQHt$YV*X=q@1>+;8-ow2XxYw?fXy7AlIy@tlWfBVZMw+){x&!C>X z^$Q~eOHlUpNxRm$SLwfau>YyQmCEn!|9BpC>&?*Y z{vSPi)9XiwsoxLZHn-eMTJ}|C(U8vvKl#D9cOP%=(y`0zi$-n_bp51ZE!^(E!My91 zmfO~^Ig|h9$@`X;mN~=Az8Ue%$@^2goYMwv{3&qdt)@+94}COL-qhmBBh#*!Gj;fJ zcu?)gxKDpN)ju&}kh*Wl+`+3K8TVlI{Qm2e(rO*HJfoj=44}yXvGt?d0rqrlF}@UN{|D>D`i%9AOfH^E!3Tot&dA}7 znXgw?Hm`hsC;jP>&9A;+{&vp3+O+cTc5=_srLDgle@Blo9TqP4?WEwP8*96Uop|oP z!{2N>`#JycO2e7t=7CEeo>_YMhF>1-d;dGVzgzTJL$58*>o0nQu{%3G-2l=SZ`N*0 z-}P^e^yx-z0_l`l_Z|9|ZONIN<{dn``ofkO&-XmIYR8dz!Q1cu_`P{udM$H2vMTlE z9fvpZLZ6AN*PQ+QTPig)G4r+&ea^Y=U$(QcHdShU;mikDXO^Oif0%QAS%+`8UAU_J z!1VaI;@%CD7fN668q+Xuh_}6^PB7o!>3gd2-11c0=FwjaJw0Uahv$}W3UqD}9+Q9Q zKP))b-_^Wk?wGlI=JwhA(`$=2?f;RQJ8+pilO|Rf0xPNe?mxZlvafn%mOrs}W}gKk zrk!8$`l(kwJ-UARxm#Y@wA+4k!|l&qaD1%Ho%n)z|5@+6Z+aD89pUZE4>>mWA{o35 zn4#G9)6VE+PhI!~ZiQq0=>tKJJO21A&adTz_#H#Pg;f3~=9 zYH{iltF7Oh{dWAXE1?C4c71iLarTBy5AFyZdL=oO+xg9@9!EFrSon^uLO!e4Vr z%xvSwd0w2o?c2!G%fjW4smVv?4<2!T^Zg&5{KrlG%TrDrJ-^}9H@S{G&ENHHIbXbQ zWa%Q+J0I@)Nk0Hc&hy7!{evtAyw&ERHq7C#cP!ob(V?dY-1Fj?!=>ceLEGNkvLd^G z+^L7RY@F#@vVX3$_^S6Nj=Ju!@cj0R%dv^7dpc&InoABowEiL*Uj&Lb9$pg~x%u$) z-9P{1?Csri(8;y>=&EhUZ_GXN!GlER_{T4eb= zK3{2by4CyzJu91Memxjfe{%rFE*sEo{Ez#hdXo2e&`7VDlPcD*5f+?e{O(41j*h)!(iTe(`9}4$}`=5B@S$EB!M0(}QgvI6HRf z&ccqj_Wbhm^4$7=p9N?&|IRT7U+X2Wcy!EfGwGTGVao%TZ)?@dJD~3$k=Y$Rd{4Fb z`6ZH8n2+Co(Xf2gy4-scFTd~n*lU{hh}#}<^*i?c?(~P(pI`3VG5@Ro!`fTMMcH-j z!*nVss30XGN{Z4*hzbJIDUAq73`5tDf?^N~(xn2@-HoCkEgeHi4c#-y4Ds$0FR%Ol zKhN{={owEChjaG1&$U+^Ypr8#1P&&R*qU-;^dF7kT+@=UOUF{IKXNYx1X@`nJh#5| z2dli#g;1q54+@GC%4pgQlfM{7&L>Eo2vovi))iyKC0YDGI4-%~RWs@T69h-+D~?Ff z+My{@I@j&D=7Qaa4)hUeY~F{owTG>p(WtJJTl#hI8pTX|f%-*xcd5)z5tz(|_MfpF z<7aM4d5pqHrGLBK--^*DtR_1ZWDw0Meuj(wFPGaZjjk^g&w9LUA_}MJB8rtv>ckZL zSpSwQXqXyrSMfjmrO1M;Nvi4{dd;slSiEnlIbf~I%P`iY9x&O5>rqZ)aoy>Yti+WV z!or7y(Qk50@i6VAX*!aqTB)L;LvPCRlP@@LE7w0Nu3<-TXFX_P;uUcJ|AqKU=zrrj zo6tzXrse2YX-oZOf+~&DH}Ff;v;5pfg+lGh3Pg8BwTim84>Ee)S2VMhN74e&0%h#W zZ_}yzL|H0_smlU>PKu$o8|`cCRM3dUI|X%LYIR?ml|OwaAhSyc!JCSj3eQ4fCr?b% z-GUKYlU4hgKQTL~nDXA`=Evj~qimcjp?@*|y}L7&_KpLAjnp=}V@hSRx|3Eb*6xn8 z8308t^_#}oPAoaw7L}bvs^d0uu)WGB?FJjblF*}sk;{vDjyWz7&viVd-Lf3(Frwj? zFfaBS`2B8wLeh?Znq;iV<^2Z$zvVggyV<0te)j{A%pUN99ZLU|IN(zrqDL^1T%&{%dI0XoHHWKB$DmS^qgI8VP)0 zmB9%?R=Gq5?+}&QH~z+_w`E|39yjAZS(cm<#eciAcAH3re^K#YZO0=Jp&+UuWyIGD zKTQ7Lr64kPjcB=1YI<%bt~E_a$&&d75xFTo)b_GiaV7GE}_Yn%kt8w>1>abo#MG z0yJp6H_lIHBFV($K9u6kJxC=Z0#p|W<`{GAH)B`JiuT%mcvdW}+V&vt0Pn0J6-2UH zRIso=q10J1_QB)4i|8Nu-ls9Y^a$ed$dTZ;zPrst_C8il1DLHok4_*O+59n4*5{f2 zi50V`PNym|_h5WATd6ZPcRJ(M%3ujOr{Q+paE+O;Vws;qE$ z%3Z8r%mP7>_dvQwr*e?Uf&HfC+67f+bnJ*Qj^FUjhKX*KW{Z|R&-y9qe_4z-_k+xn zRafv4_y#Uyg)pCaP_e)-YIE|BRkH7QuPZ+%EB3EnaMs1;TQm^eKYc zh1}wCZPj!ABo_B_CSlmhhgJwk2oN)NCuOOp`S(+n7^p32#g;-b-iac_9WN#KyZFuI zUg`?@X2E9hC=`BBKaEp;$c^}m1_~@XfmY10W4CEbQ@EqwtWsA!PdF-3?|pXQ5y^Tm zKDBp&(ej?7!PgH7+$T`S90SCmt&_Q&^BZH_=vvM z{-D8Avy4!x^<#TE?df7qz#4alIdn*a_=qnkT9bdbEn)b_A{%5Xt7s%sf*5oHW8yKB zm7VXfy*{hoG5DFk$nL4m#N}v<*#4A@rBx`H%N7JWt|1TBQpvt$tDH}CL9sQr&enkn zaLgn=90-!zQRi<8Hfo;qBo$HFpl7>-y)m?LoEi`C5z$?vGM=^xpH<7p&z@)48J?ho z_Vq!Iz=Bnd8Gr5vfi&3!34R+`nk8845&lp;Q#2n$k9fzJ%%zqFxcn359ssp}KDcN~R?T33 zX?ZxhpXS>8Q!6f#{gZ2e(>ZhdGB_Pp@EI@bfkU0IdrR`o0K6_tD($nPA*fdAnX8Z` z1Tid!#&xtA1qLbBWiHIo1VHD#kfyu5N;+%Sj6 zL-6t62T@aG^8)`>6S%QYwWv&_tP4IQ2gA1xr5&F*n$P4v-J`F#$DRVYOiH3p+>pK# z8T27ovxNxW(V5TC=4_mn$mH6|Y0P7c?c!u(`r;QLr3KqARs(MW7YPEcYv55LshF#f*Ov*pLH(g15eTX9 zzl5aE9>^8K`gs=$QQLkGY$XU1h%R_-IY63`$qzW1<3}p1G}k@@5BYe-6;9+rPj<7p zqd;NE_&cjXkJ5FTpa+yUwqCmJYkEUgyhG5{{Mz??f8Yg5@Y&9&fxQQmL=~j5uIHdi zX9>Sk|A?9WMsL&8wt}jD7xZe|FvrSSn;BX(YsN8@Y?Ol{!>7rBL76Eahq=jJRsO}Wz!F9`)Ou^{ zrCqNk9I{qnA^?Na*=+CM22P~4Y}(e{16cl;gf0S9W?9&^LNr1qeG_sYln_p#R|SQ{`!+5-1RV6>q)MqiN|HUf{!c|bpP$I! z5$0)_I&M48%>CN|YPCBQo2QntM~{DUjACmz#7HGdTE2)FhnrQ{jooa0+~+7s82NOc ztpdsg?4TN@(#uU16ZWJMSPnDMkWA~lJMCHm1YiCe>72oET#M<4`#udUtgR|KP8K(H zUmfO;vF?jF15DdX7zzqRF0}FcxZ5JIVc|FxUXzwH`Ss-2KY+Mi`9=D27-w)<<`bf4 zbYw2nd<6L)e8OD@C=Pd;PHfW&rEI63$&wd8-U$P$prkkeMeF90=Cy<}ciUc-p}9j= zyDGoBWEJk_j)VxxG8%J-Bw{VrtQ!$sgpn$=*PTpOhy_)DkIBVRtfAb}3QNq*ljf+O3TBg0Q%Eo)&p=J}d#VLoqHw zu>j6pUQkmJQsQI+QA+zF4+J-FoXT79)1Jn&pY)zzijLVC6@A&*Uh%N?T}_9x$t9p3 zJ8M=@L9F;i=ZQhFTyLYO{J!+92hokkh9`)CFh+jU{5;%LR-2iR62IQlLV9Y62PXnb z^HB`Ml?E2XDZWb_?bI^Ivnpo;HXC?6Cu~Al%D*cx>tPQAb{B28>^TQLXKp3#tXCz< zS|sl)G`DMKtzZ<{239SnN6g+F_a94b=J(WX76$K+HuR&X{Z1Nsstc>8yaJ3fdyB-6 z%7JrQEw{@QKy1bw`uC{IE^o(~4ig5G=JmOB+V)fS>gTLJdrpZyT;*(Vz3Pvtv2NI| zH57&mBz5?8wnTdE-^!}cH#0H8j2WR=1CnFm6XK70>^H;vw))aVeD@I8gMg;xKobx2 z%9L%DiT&D-cVJ(6>5BGW=X-_y`>zy~&%Pf2+n)fA2iMQITo5)wcyA5?1GB z+i`$~of*h{@KWNRFYFhn0!c4m+7CV3>7->{GDxVH3yT_)3X7VOel-aBgz{JZ9EOcQ z-G?Fb*0HN2UIXOO+^Zi^WvY<_i4R>(7i|35B6y}RI5@4qEjWQwacRcX1`)7~ zqo%ZPOpz$-M~N2yUL3M$A87SId_{8R;wKJx{VymF-e~-N8UE9}jWjKw5WknY?{2R? zRpjvnF+D<4uRAu=UasRL)82i=k_9Sp^PWbm!`~?zbEQ0BBWf)hb^37iWy0IDpbrYC^SK|r+9jEd}5JusL7N$TIp2YN4lhk?=7JlDu8DT_GZ7V z$3qzT(*42^)CaP=PU99BBT3n5*5CjckfFy@ku0Pj`Mw0&ZS#DH$7sAwUANdkHaR_S zIC{!&O|t@7t~wrdY*;_atTHC3ur8iwuV3*9Zc@~RNMnt!Z}rDd5MLQGQaGRD78N05 zell-U6V^^4UCDCX?6-@H0 zky;rWF)MOuv2m=sPr6q61|!>`=+cxkNOR4(PwbXWt|}@x+G@hR@3~|}=dY=>Ta)|G z=+%FA6S87@O7PmOA2i5#IFz2jogLRNs?T5S6x{T)4rXC~QZM0{tX8vJRy=8^D`>=}PQkYiVX6g)I?jmxqJTrw?{lpusReH~st0gqry8X;vK7uUuhf8D zofBw~_|W8tm=|i;>5-bs&ylW#ZTi6jFl>xeRX#pogg>7jw#KEG-wIE)>eQVz_ z_~kU80|xh|m6iGB)hbhMwlh8o=s1(%BSP8|x{F@-@>{Y6pnxSG9XGC{8jh^qhGwe;wI&y+v$* zNjb7Q9w=(^L4o?%H$TDZb&tuTdF_}D`?3Uw&st?yRqK1TNiPLE)>U+9-A+m>?Yp@Yivlt?KK?=nwTKZA8-Wr5r7z6|yOB6)BU*X6JIyF0H) z2Wyo^tPP`BC z1buIFwwp;@ZcAgRD{z;?J1Oob3~}5i$+gazu=uIf)};DPK)4r`DRy=FFhgBTM~rl7 zOc$)b=FOJ0?1m|6)3M9%F1DMp2%WcC{+i@Kknd)p@wv81#J?URy&g6%tsU?t?)TT& zb5y?WUOV+8IO*(^%BftfBN#@1wE?3<;*DXxkL!Ls`+Gf7w*1R6saIJ=|8pacsR=WG z&D977=I-%hYWh@f$%0{m0(D$XY~v`$&GUK3E&kh2n*z%%BiAd3c>OjDv}%mlymjIxC%F#} z_Tdd58InX?|Hw#H@LSQj)_2>hu7=4Caz9mmmlY{3+=6fzYzGEP1kz;1$u4@=*SekO zDyEkig1W0(LTxKz^n1Ltk+B%*v*edwvAROFN>xys zOfMC=Csz*3X}YCEVer=dcj?*kF=hVpRj#<0C(M*X+1~?B3JU5j{eq28xt|;f&y2r@ zapY5BI-3=c9?>wwJD=OKJ7?V+X1>n1ZAx#) zS57^No=-CgYkw_N$*&_DGwuiQXMv(^qza%AktXtSxCO?4d|Ac)aBwP1uZu$Ys9S%< z3Xo6-XRd1pE|_6Pc9#lk4<=sY{aDch9l7H>!BI)yY$#IJR9_?Ar6tGK0F@O+5n#Q! z+OV5TSxhC1GHwuFfo;}rgtFwy#2F{^;SzBhBj_XSNUbY9Tp=I*oXZU#Uyt1Vsw$MG zT)z6Vutij|{@8`cAOhPRJ2RH_$*961fp5Fb%ztR0s4*9pcJY03AKXmFud`cs>S4pl zMd>JbW@VLFr2kHzVGq*7c7kwO=#*KlNCQ6SrD$i_qU1hh~Aud|62p@ zOX$kz(fj$c6b$B>0z;kW%mE?roAuHGiNRP198<}J{3-1WSjQx{Po4qD;~9=WOp8h_ z`Ur1`5p9-S{Q4%Ec`-nAkGP;t0ejKl3)^z&|86)>8CJWh z>-X}cXT2+Z=o_I&Lamh*=NBmewr_V0loKZYFVX}F&MEwO>+ZU@%Qj)Y_uTYIDSAcM zHea@R#l5Sqb~*V(PclpNR=jds$|P6#cjrYjfT6U``;`%WAI}hqc>>@qYe6 zid^jzObJrl!@XsMknnfLA3pS+W==8LxA0?;Q%OQL5e}Up-vp1=GQ~tUkdM>PR!rz2 zb(qaFdr~(&hhj?46CO~XsJ!u*vbGQRDqeej?^Vn9cs`M4o^PYWFu`wG!oXIS?F|hj z3GQq_i1{Nb1TW9U#!8B?6F{@+Y2Ha03tFEr3R7JYR$^|{3rLjh;VEz0XxXhqgfxb~U zZZ~cq%YOGR)$Z}6wTY^)wfRonH&x@XcCz9LY5F^H-^6b|wdhc6Za6KuN~(}e%HqAE zqog&@L`0hj9Y;|k&*ak!Hp#0u-0WI?jW#OEiA-h52YFa@Nf|8i&OE ztP=yYQ&AVaLaN}|7opHg9JFF>rhDF)Yg{34;d7j}jk@378au4MM>2+@=z6nqa&qna z;?-juBB^;FfsxoehlqBa?iAFnQ$_0J)c-knM;{LVOpxrFr47uk5mQuz$l{oElb1o} zu3y~n{O2-Y*W?@5isJ0;n9Zg44tH0aO~w`%cnhjOSx4|8)7jCZzngp7RmI>OfV<0b{aEs+=qk;5V}^Q*lKpJ_Chp+rNipy z^RxOCb3cBgCHQ+xLA?TzS8h`ZRjoygO_8b-X{?xBwd%E5Tm zD5hpV?4j$pdBHH&@7UMG^CVbuGh}XXNA_g9uxfg1431s@SWIB{5^(>>8J`{ zfjL#KDD!%!HQm}KMAvR#g$Kqhi$#8s;rhi3MEDyUa@r0MAO^KUy0Rh{@mskCm@Zy@X=DR_R4`xQY}oo z`WB1q2`0&Ky-(<%g7^-KB_LVSp)uk7x6yNJO^in|6E6qU$`kJIn|dTPo@^_ojG?c= zTR|R6g82AwHXggeYd2JC6@$c1Q@G9));--?uhNxllV`!Lo6HuAbtb$0;3!ocZc5!w z`CKn9e86Lc_+Gjm`BvONARvq>< zOGF*9R5;l-i;O(&;!a!YKx`E3HLTnGtg}%sI80+`xbxzy@r(L{Sb>Js4dp8D z-?jD?b9zR0;ktU;GF9Q-V(Y$6F^DnGS2udbvF&kYD@&BTQ%;uC$go1~x=V%bBL;UV z&-3@|4>W1-^A>^IKC4_YqmD*n%mm$^oLgRC(fiX@%n?|dk7M4_5t98$78O&8_CDwn zY~4j4=fP_x9-a%$xtmsTASPA~8Ro?`Tx_E{>2G}~4NIzT>T7C3wR{} z$*8L)iFB_&p4KHY@TT`6&e-^#Xt>fcVv@d!L+`ih9=DuOj&Q&mb2xkRCQI)yM^E|o zNQ{Hv$s3c@8SikM7GO78mrf~J>7LFutp&)d$o=x$~9~j|KK`2sQw?t z{pIo-NQ1+Q{nn>7BxlQy7?Zed?vJUa`W`S064_NrF zzwj7Y=hh)MZPz#ZecYkt?4a+#0&mK#-B7kCb80v)i^=!=%nly)M~^**C)oBdoRMCg zBmvm9;{nKFiYVNZ6zd*M_p? zrRAQeC}s6Pyv{pmr6E6|;v{Gkow=)*D(dzn-3{xE2qJ>(4Y@If;`0@^S5j@M8!3vu+Sw7Hn?j+%b7T z2sPa7Kylnlu9UVK{Z26&7v95s(imHyFPPNn_G{g9sgOH$u4y^|n6U7Gsjx5rCjrjI)gOco}+dq)- ziq9>vG~@XNCdi6`Ez*s=Ww%DbJ5XnIG)FjRp7_}WU_R%~v#~L~lye59?sFq)bx@hB7edjHx_6HAU0)pFVlcd`QP{pb7GcVKVf;Nxly2J`Z_kRBZRQAQO+s}}W`vf^| zp$%3}LyY`Sn7;)aO@_^1#|*DE#+l@XkK}k)>WLdF@f(8{gd4xIcjlG9)!<2MIER}g zwEOq|m!47*I6|Uu&Nv4WVLae106Ef?ayIu1r@|6OkyWMf(TxFVhZn=^sI;PV>d~f- zq%*dTO&Kn(h0kDOc_U&>7p z;PY|pt<3w9020EKpDI4t`(&s$wyMPh!8{#M^Pdd@d;S1g93)&SHDpBGG*1E->W9~Y z7Lwl+(7r4OPe1Tvi-9@>K!?uc|M28E`?3Tm+67M;)yVSG?w%sYAENR-L(o6TsEh1? zPVpT$#sAsC`NKHLNE70T1Qmlbw+SpVHBM#52Q;!YWLB6+{u=0Rhos2I$b()S*CDif zQ<TTY#z|&;{Q*ZYn$<(&iC+r(6;i|u$3mrf!Y{Xp@Zy)8jkD7neJrgw82pc zp=-nlYAFAU_0iQ3TtgS6#Q@GLS6ZbG`pr|a8-%<}NB8(?A3*pIby_DrrGV_ElG=sI zWdKdNms^+32W(rz1#FjVBvbgB0#IAa!7<^AC#S7yFLprvt2H*P)u}V~%~=_AZrt~6 zzkN-ZD5#hm@?r4lpGLXbiuQXoq7FmpGN*1!h$g;W{3c1mLwXfWu-WSv9+=DZzxwJB zO9Yx1pi_-L`YyII?Y%qnE8o=aXL(MiOcoyz4Vi{{U%fXI5+%StQ2t7P z%aUaPU22aG0=eh~MTN5IZ1-PKx1yX1w5-`SeIC4NO#NJG5-}JsY$Vug2wH_?=am2U zzC{cIUO^ z-*NakWit|p05-uq?H%^SQ1Q=j&}E7n5qy_rsuY@o z@Xh~vg)s07x$`L1anZ_1P^;K{JR zL~2|G>j5D?0&U=~*RP^!$ex3>HQWeeUIzY;E;wkcmcBj~b;@3hO#RTxZ}_wl>K5x3cct=0?Jv2xgVg!XOH@h&- zmLopRIxnb!>v=MO?Wg<8GGTQKteEe8j;)#2HrzuC7lp#q^F^c_dRsR#(e(p~FUdz0 zl5Vv|Y-8$6TL9^IvM$##{ROQ(`Dx36Cc8lHJpnlGiglA7MRqR`Prfj_k`hty|J!vZ zX$bilnHo>xm^OwaH*y@;>mD&xR5h&BIl8J!R!(sg;?iV5J}PIqeRz;4>sjcZgeh25 zOPz0k{iKRL+q$Cpp#%s+?h2&wrO0fzS3oFy?;uNmq+>qTzJf>oe)T`M7ac%)(OWdCe=CB|gL>X6p!J=^2WN5uUE;6x zA=40ZfBR4S|22d8_6P8}B^w<4GZ)uez%9H3aWXYU z6R>M{@_FJyA$R&xGC23tmf1B_ARd7I*g3Vx58x+c9!$ClctKH&5CH|Xa2jxwWJ7~Q zB7zjn%g=>D<`4*2Ohg7j^sd2nAl~eFK@0t&1M2EB)&CeooO;@G1{xqx6E7@UpQ