From 7ee9ff34a8003bc5038c881178aa097837a37536 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 13 Mar 2023 19:45:33 +1000 Subject: [PATCH] Initial public release --- .gitignore | 2 + LICENSE | 21 + README.md | 243 +++++- bin/.gitignore | 2 + build.bat | 1 + build.ps1 | 426 ++++++++++ cloud/aws/README.md | 115 +++ cloud/aws/cluster/.gitignore | 1 + cloud/aws/cluster/test-cluster.template | 24 + cloud/aws/deploy.bat | 1 + cloud/aws/deploy.ps1 | 305 +++++++ cloud/aws/node/.gitignore | 2 + .../aws/node/eks-worker-node.pkr.hcl.template | 79 ++ cloud/aws/node/generate-setup-script.py | 354 ++++++++ cloud/aws/node/scripts/cleanup.ps1 | 9 + cloud/aws/node/scripts/setup.ps1 | 130 +++ cloud/aws/node/scripts/startup.ps1 | 26 + deployments/default-daemonsets.yml | 57 ++ deployments/multitenancy-configmap.yml | 87 ++ deployments/multitenancy-inline.yml | 70 ++ .../cuda-devicequery-mcdm.yml | 25 + .../cuda-devicequery-wddm.yml | 25 + .../cuda-montecarlo/cuda-montecarlo-mcdm.yml | 25 + .../cuda-montecarlo/cuda-montecarlo-wddm.yml | 25 + .../device-discovery-mcdm.yml | 22 + .../device-discovery-wddm.yml | 22 + examples/directml/directml-mcdm.yml | 22 + examples/directml/directml-wddm.yml | 22 + examples/ffmpeg-amf/ffmpeg-amf.yml | 23 + .../ffmpeg-autodetect/autodetect-encoder.ps1 | 26 + .../ffmpeg-autodetect/ffmpeg-autodetect.yml | 32 + examples/ffmpeg-nvenc/ffmpeg-nvenc.yml | 23 + .../ffmpeg-quicksync/ffmpeg-quicksync.yml | 23 + examples/nvidia-smi/nvidia-smi-mcdm.yml | 26 + examples/nvidia-smi/nvidia-smi-wddm.yml | 26 + examples/opencl-enum/opencl-enum-mcdm.yml | 25 + examples/opencl-enum/opencl-enum-wddm.yml | 25 + examples/vulkaninfo/vulkaninfo.yml | 23 + external/.gitignore | 2 + library/CMakeLists.txt | 51 ++ library/include/DeviceDiscovery.h | 269 ++++++ library/include/DeviceFilter.h | 65 ++ library/src/Adapter.h | 37 + library/src/AdapterEnumeration.cpp | 174 ++++ library/src/AdapterEnumeration.h | 37 + library/src/D3DHelpers.cpp | 72 ++ library/src/D3DHelpers.h | 48 ++ library/src/Device.h | 61 ++ library/src/DeviceDiscovery.cpp | 115 +++ library/src/DeviceDiscoveryImp.cpp | 260 ++++++ library/src/DeviceDiscoveryImp.h | 51 ++ library/src/DllMain.cpp | 10 + library/src/ErrorHandling.cpp | 50 ++ library/src/ErrorHandling.h | 103 +++ library/src/ObjectHelpers.h | 31 + library/src/RegistryQuery.cpp | 198 +++++ library/src/RegistryQuery.h | 28 + library/src/SafeArray.cpp | 34 + library/src/SafeArray.h | 71 ++ library/src/WmiQuery.cpp | 355 ++++++++ library/src/WmiQuery.h | 34 + library/src/pch.h | 36 + library/test/test-device-discovery-cpp.cpp | 82 ++ library/vcpkg.json | 16 + plugins/cmd/device-plugin-mcdm/main.go | 12 + plugins/cmd/device-plugin-wddm/main.go | 12 + plugins/cmd/gen-device-mounts/main.go | 245 ++++++ plugins/cmd/query-hcs-capabilities/main.go | 150 ++++ plugins/cmd/test-device-discovery-go/main.go | 71 ++ plugins/go.mod | 37 + plugins/go.sum | 798 ++++++++++++++++++ plugins/internal/discovery/device.go | 85 ++ .../internal/discovery/device_discovery.go | 466 ++++++++++ plugins/internal/discovery/device_filter.go | 14 + plugins/internal/discovery/runtime_file.go | 14 + plugins/internal/mount/default_mounts.go | 31 + plugins/internal/mount/device_mounts.go | 91 ++ plugins/internal/mount/vendors.go | 6 + plugins/internal/plugin/common_main.go | 83 ++ plugins/internal/plugin/deletion_watcher.go | 77 ++ plugins/internal/plugin/device_plugin.go | 405 +++++++++ plugins/internal/plugin/device_watcher.go | 205 +++++ .../internal/plugin/plugin_configuration.go | 145 ++++ update-version.bat | 1 + update-version.ps1 | 51 ++ 85 files changed, 7682 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 bin/.gitignore create mode 100644 build.bat create mode 100644 build.ps1 create mode 100644 cloud/aws/README.md create mode 100644 cloud/aws/cluster/.gitignore create mode 100644 cloud/aws/cluster/test-cluster.template create mode 100644 cloud/aws/deploy.bat create mode 100644 cloud/aws/deploy.ps1 create mode 100644 cloud/aws/node/.gitignore create mode 100644 cloud/aws/node/eks-worker-node.pkr.hcl.template create mode 100755 cloud/aws/node/generate-setup-script.py create mode 100644 cloud/aws/node/scripts/cleanup.ps1 create mode 100644 cloud/aws/node/scripts/setup.ps1 create mode 100644 cloud/aws/node/scripts/startup.ps1 create mode 100644 deployments/default-daemonsets.yml create mode 100644 deployments/multitenancy-configmap.yml create mode 100644 deployments/multitenancy-inline.yml create mode 100644 examples/cuda-devicequery/cuda-devicequery-mcdm.yml create mode 100644 examples/cuda-devicequery/cuda-devicequery-wddm.yml create mode 100644 examples/cuda-montecarlo/cuda-montecarlo-mcdm.yml create mode 100644 examples/cuda-montecarlo/cuda-montecarlo-wddm.yml create mode 100644 examples/device-discovery/device-discovery-mcdm.yml create mode 100644 examples/device-discovery/device-discovery-wddm.yml create mode 100644 examples/directml/directml-mcdm.yml create mode 100644 examples/directml/directml-wddm.yml create mode 100644 examples/ffmpeg-amf/ffmpeg-amf.yml create mode 100644 examples/ffmpeg-autodetect/autodetect-encoder.ps1 create mode 100644 examples/ffmpeg-autodetect/ffmpeg-autodetect.yml create mode 100644 examples/ffmpeg-nvenc/ffmpeg-nvenc.yml create mode 100644 examples/ffmpeg-quicksync/ffmpeg-quicksync.yml create mode 100644 examples/nvidia-smi/nvidia-smi-mcdm.yml create mode 100644 examples/nvidia-smi/nvidia-smi-wddm.yml create mode 100644 examples/opencl-enum/opencl-enum-mcdm.yml create mode 100644 examples/opencl-enum/opencl-enum-wddm.yml create mode 100644 examples/vulkaninfo/vulkaninfo.yml create mode 100644 external/.gitignore create mode 100644 library/CMakeLists.txt create mode 100644 library/include/DeviceDiscovery.h create mode 100644 library/include/DeviceFilter.h create mode 100644 library/src/Adapter.h create mode 100644 library/src/AdapterEnumeration.cpp create mode 100644 library/src/AdapterEnumeration.h create mode 100644 library/src/D3DHelpers.cpp create mode 100644 library/src/D3DHelpers.h create mode 100644 library/src/Device.h create mode 100644 library/src/DeviceDiscovery.cpp create mode 100644 library/src/DeviceDiscoveryImp.cpp create mode 100644 library/src/DeviceDiscoveryImp.h create mode 100644 library/src/DllMain.cpp create mode 100644 library/src/ErrorHandling.cpp create mode 100644 library/src/ErrorHandling.h create mode 100644 library/src/ObjectHelpers.h create mode 100644 library/src/RegistryQuery.cpp create mode 100644 library/src/RegistryQuery.h create mode 100644 library/src/SafeArray.cpp create mode 100644 library/src/SafeArray.h create mode 100644 library/src/WmiQuery.cpp create mode 100644 library/src/WmiQuery.h create mode 100644 library/src/pch.h create mode 100644 library/test/test-device-discovery-cpp.cpp create mode 100644 library/vcpkg.json create mode 100644 plugins/cmd/device-plugin-mcdm/main.go create mode 100644 plugins/cmd/device-plugin-wddm/main.go create mode 100644 plugins/cmd/gen-device-mounts/main.go create mode 100644 plugins/cmd/query-hcs-capabilities/main.go create mode 100644 plugins/cmd/test-device-discovery-go/main.go create mode 100644 plugins/go.mod create mode 100644 plugins/go.sum create mode 100644 plugins/internal/discovery/device.go create mode 100644 plugins/internal/discovery/device_discovery.go create mode 100644 plugins/internal/discovery/device_filter.go create mode 100644 plugins/internal/discovery/runtime_file.go create mode 100644 plugins/internal/mount/default_mounts.go create mode 100644 plugins/internal/mount/device_mounts.go create mode 100644 plugins/internal/mount/vendors.go create mode 100644 plugins/internal/plugin/common_main.go create mode 100644 plugins/internal/plugin/deletion_watcher.go create mode 100644 plugins/internal/plugin/device_plugin.go create mode 100644 plugins/internal/plugin/device_watcher.go create mode 100644 plugins/internal/plugin/plugin_configuration.go create mode 100644 update-version.bat create mode 100644 update-version.ps1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fa551d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.vscode +build diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2940257 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022-2023 TensorWorks Pty Ltd + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index a9de6aa..c583fee 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,241 @@ -# DirectX-Device-Plugins -The Kubernetes Device Plugins for DirectX +# Kubernetes Device Plugins for DirectX + +This repository contains Kubernetes device plugins for exposing DirectX devices to Windows containers. The plugins include the following features: + +- **Individual devices** are mounted based on their unique [PCIe Location Path](https://learn.microsoft.com/en-us/windows-server/virtualization/hyper-v/plan/plan-for-deploying-devices-using-discrete-device-assignment#pcie-location-path), rather than mounting entire classes of devices based on [device interface class](https://learn.microsoft.com/en-us/windows-hardware/drivers/install/overview-of-device-interface-classes) GUIDs + +- Plugins are provided for both **graphics devices** and **compute-only devices** + +- Support for devices from **all hardware vendors** (e.g. AMD, Intel, NVIDIA, etc.) + +- Support for automatically mounting the runtime files required to **use non-DirectX APIs inside containers** (e.g. OpenGL, Vulkan, OpenCL, NVIDIA CUDA, etc.) + +- Support for **multitenancy** (i.e. sharing a single device between multiple containers) + + +## Contents + +- [Core components](#core-components) +- [Important notes on support for non-DirectX APIs](#important-notes-on-support-for-non-directx-apis) +- [Usage](#usage) + - [Prerequisites](#prerequisites) + - [Cluster setup](#cluster-setup) + - [Node setup](#node-setup) + - [Installation](#installation) + - [Testing the installation](#testing-the-installation) +- [Building from source](#building-from-source) + - [Build requirements](#build-requirements) + - [Building the core components](#building-the-core-components) + - [Building container images](#building-container-images) + - [Building the examples](#building-the-examples) +- [Known limitations](#known-limitations) +- [Legal](#legal) + + +## Core components + +The source code in this repository consists of three main components: + +- **Device Discovery Library:** a native C++/WinRT shared library that interacts with [DXCore](https://learn.microsoft.com/en-us/windows/win32/dxcore/dxcore) and other Windows APIs to enumerate DirectX adapters and retrieve information about with the underlying [Plug and Play (PnP)](https://learn.microsoft.com/en-us/windows-hardware/drivers/kernel/introduction-to-plug-and-play) hardware device for each adapter. The device discovery library is also responsible for querying Windows registry information for device drivers in order to determine which runtime files need to be mounted into containers to make use of non-DirectX APIs. + +- **Device plugin for MCDM:** a Kubernetes device plugin that provides access to DirectX compute-only devices (or "Core devices") that comply with the [Microsoft Compute Driver Model (MCDM)](https://learn.microsoft.com/en-us/windows/win32/direct3d12/core-feature-levels), under the resource name `directx.microsoft.com/compute`. Compute-only devices support the Direct3D 12 Core 1.0 Feature Level for running compute and machine learning workloads with the DirectML API. Compute-only devices may also support other compute APIs such as OpenCL and OpenMP, or vendor-specific compute APIs such as AMD HIP and NVIDIA CUDA. + +- **Device plugin for WDDM:** a Kubernetes device plugin that provides access to DirectX display devices that comply with the full [Windows Display Driver Model (WDDM)](https://learn.microsoft.com/en-us/windows-hardware/drivers/display/windows-vista-display-driver-model-design-guide), under the resource name `directx.microsoft.com/display`. Display devices support the full Direct3D 12 feature set for performing both graphics rendering and compute workloads with the Direct2D, Direct3D and DirectML APIs. Display devices may also support other graphics APIs such as OpenGL and Vulkan, compute APIs such as OpenCL and OpenMP, or various vendor-specific APIs for compute (AMD HIP/NVIDIA CUDA), low-level hardware access (AMD ADL/NVIDIA NVAPI), video encoding (AMD AMF/Intel Quick Sync/NVIDIA NVENC), etc. + + +## Important notes on support for non-DirectX APIs + +**DirectX APIs (DirectML for compute-only devices, and Direct2D/Direct3D/DirectML for display devices) are the only APIs guaranteed to be supported by all devices regardless of vendor and device driver version.** Support for other APIs is entirely dependent on vendor support at the device driver level, and may vary significantly between devices from different vendors or even between different versions of an individual device driver. TensorWorks strongly recommends testing the functionality of any non-DirectX APIs that your applications require in order to verify that they are supported by the hardware and device drivers present in your target execution environment. + +Kubernetes device plugins are unable to influence scheduling decisions based on factors other than requested resource name and [NUMA node topology](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#device-plugin-integration-with-the-topology-manager). As such, the device plugins for DirectX cannot select devices based on requirements such as a specific hardware vendor, model-specific details such as VRAM capacity, or support for a specific non-DirectX API. If heterogeneous devices are present across different worker nodes in a Kubernetes cluster and applications need to target a specific vendor, device model or API then the recommended mechanism for enforcing these requirements is to [add labels to your worker nodes and use a label selector to control which nodes any given application Pod is scheduled to](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). Some cloud platforms automatically apply node labels that are suitable as a starting point: + +- Worker nodes in Azure Kubernetes Service (AKS) node pools whose underlying VM series includes a GPU will be automatically labelled with the `kubernetes.azure.com/accelerator` label, with a value indicating the hardware vendor (e.g. `nvidia` for NVIDIA devices.) See [*Reserved system labels*](https://learn.microsoft.com/en-us/azure/aks/use-labels#reserved-system-labels) from the AKS documentation. + +Note that there is no way to target specific device characteristics when heterogeneous devices are present on a single worker node, since the Kubelet will treat all devices with a given NUMA affinity as equivalent and simply [select whichever devices are available](https://github.com/kubernetes/kubernetes/blob/v1.24.2/pkg/kubelet/cm/devicemanager/manager.go#L686) when allocating them to a container. There is also no guaranteed way to prevent an individual container from being allocated a set of heterogeneous devices when requesting multiple DirectX device resources, which would lead to clashes between the runtime file mounts for different device drivers and may result in some non-DirectX APIs becoming non-functional (e.g. if the device drivers for two heterogeneous devices both attempt to mount their own version of `OpenCL.dll` then only one will succeed and OpenCL may not function for the other device.) For these reasons, it is recommended that each worker node in a cluster include only homogeneous devices. + + +## Usage + +### Prerequisites + +The following prerequisites are necessary in order to use the Kubernetes device plugins for DirectX: + +- Kubernetes v1.22 or newer (v1.23 or newer recommended, see the [cluster setup notes](#cluster-setup) below) +- Windows Server 2022 or newer +- containerd v1.7.0 or newer +- Up-to-date device drivers for all DirectX devices + +### Cluster setup + +If your cluster is running Kubernetes v1.22 then you will need to enable support for Windows HostProcess containers by setting the `WindowsHostProcessContainers` [feature gate](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) to true. HostProcess containers are enabled by default in Kubernetes v1.23 and newer. + +### Node setup + +Windows nodes will need to be configured to use containerd v1.7.0 or newer as the container runtime, and GPU drivers will need to be installed for the DirectX devices that are present on each node. + +It is important to note that many managed Kubernetes offerings **do not** provide support for using custom worker node VM images, which precludes the use of the Kubernetes device plugins for DirectX on those platforms until official support is implemented. One managed Kubernetes offering that does support custom worker node VM images is [Amazon EKS](https://aws.amazon.com/eks/), and demo code for running the Kubernetes device plugins for DirectX on EKS is provided in the [cloud/aws](./cloud/aws) subdirectory of this repository. + +### Installation + +Once one or more compatible worker nodes have been added to a Kubernetes cluster then the device plugins can be installed. The recommended method of installing the Kubernetes device plugins is to use the [provided deployment YAML](./deployments/default-daemonsets.yml), which deploys each plugin as a Kubernetes [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) that runs on every Windows Server 2022 worker node as a [HostProcess](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/) container: + +```bash +kubectl apply -f "https://raw.githubusercontent.com/TensorWorks/directx-device-plugins/main/deployments/default-daemonsets.yml" +``` + +Configuration options can be passed to the device plugins by either setting environment variable values directly on the Pod spec or by creating and consuming a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/). Examples of both of these mechanisms are provided: + +- [multitenancy-inline.yml](./deployments/multitenancy-inline.yml) +- [multitenancy-configmap.yml](./deployments/multitenancy-configmap.yml) + +### Testing the installation + +A number of example workloads are provided in the [examples](./examples/) directory to test that the device plugins are allocating and mounting DirectX devices correctly. These examples include basic tests to verify device detection, simple workloads to test that DirectX APIs function correctly, and vendor-specific workloads that test non-DirectX APIs. + +The following examples should work when running on all DirectX devices: + +- [**device-discovery**](./examples/device-discovery/): runs a test program that uses the device discovery library enumerate DirectX devices and print their details. This is useful for verifying that a container can see the GPUs that were allocated to it. Versions of this example are available for requesting [compute-only devices](./examples/device-discovery/device-discovery-mcdm.yml) and [display devices](./examples/device-discovery/device-discovery-wddm.yml). + +- [**directml**](./examples/directml/): runs the [HelloDirectML](https://github.com/microsoft/DirectML/tree/master/Samples/HelloDirectML) "hello world" tensor data copy sample from the [official DirectML samples](https://github.com/microsoft/DirectML/tree/master/Samples). This is useful for verifying that the DirectML API is functioning correctly. Versions of this example are available for requesting [compute-only devices](./examples/directml/directml-mcdm.yml) and [display devices](./examples/directml/directml-wddm.yml). + +The following examples will only work when running on GPUs whose device drivers support using the OpenCL API inside a container: + +- [**opencl-enum**](./examples/opencl-enum/): runs the [enumopencl](https://github.com/KhronosGroup/OpenCL-SDK/tree/main/samples/core/enumopencl) device enumeration sample from the [official OpenCL samples](https://github.com/KhronosGroup/OpenCL-SDK/tree/main/samples). This is useful for verifying that the OpenCL API can see the devices that were allocated to a container. Versions of this example are available for requesting [compute-only devices](./examples/opencl-enum/opencl-enum-mcdm.yml) and [display devices](./examples/opencl-enum/opencl-enum-wddm.yml). + +The following examples will only work when running on GPUs whose device drivers support using the Vulkan API inside a container: + +- [**vulkaninfo**](./examples/vulkaninfo/vulkaninfo.yml): runs the [Vulkan Information](https://vulkan.lunarg.com/doc/view/latest/windows/vulkaninfo.html) tool from the [Vulkan SDK](https://vulkan.lunarg.com/), which enumerates Vulkan-compatible devices and prints their details. This is useful for verifying that the Vulkan API is functioning correctly. This example only supports display devices. + +The following examples will only work when running on AMD GPUs: + +- [**ffmpeg-amf**](./examples/ffmpeg-amf/ffmpeg-amf.yml): uses the [FFmpeg](https://ffmpeg.org/) video transcoding tool to encode a H.264 video stream with the [AMD AMF](https://gpuopen.com/advanced-media-framework/) hardware video encoder. This is useful for verifying that AMF is functioning correctly when running on AMD GPUs. This example only supports display devices. + +The following examples will only work when running on Intel GPUs: + +- [**ffmpeg-quicksync**](./examples/ffmpeg-quicksync/ffmpeg-quicksync.yml): uses the FFmpeg video transcoding tool to encode a H.264 video stream with the [Intel Quick Sync](https://www.intel.com/content/www/us/en/architecture-and-technology/quick-sync-video/quick-sync-video-general.html) hardware video encoder. This is useful for verifying that Quick Sync is functioning correctly when running on Intel GPUs. This example only supports display devices. + +The following examples will only work when running on NVIDIA GPUs: + +- [**cuda-devicequery**](./examples/cuda-devicequery/): runs the [deviceQuery](https://github.com/NVIDIA/cuda-samples/tree/master/Samples/1_Utilities/deviceQuery) device enumeration sample from the [official CUDA samples](https://github.com/NVIDIA/cuda-samples). This is useful for verifying that the CUDA API can see the NVIDIA GPUs that were allocated to a container. Versions of this example are available for requesting [compute-only devices](./examples/cuda-devicequery/cuda-devicequery-mcdm.yml) and [display devices](./examples/cuda-devicequery/cuda-devicequery-wddm.yml). + +- [**cuda-montecarlo**](./examples/cuda-montecarlo/): runs the [MC_EstimatePiP](https://github.com/NVIDIA/cuda-samples/tree/master/Samples/2_Concepts_and_Techniques/MC_EstimatePiP) Monte Carlo estimation of Pi sample from the [official CUDA samples](https://github.com/NVIDIA/cuda-samples). This is useful for verifying that the CUDA API can run compute kernels when running on NVIDIA GPUs. Versions of this example are available for requesting [compute-only devices](./examples/cuda-montecarlo/cuda-montecarlo-mcdm.yml) and [display devices](./examples/cuda-montecarlo/cuda-montecarlo-wddm.yml). + +- [**ffmpeg-nvenc**](./examples/ffmpeg-nvenc/ffmpeg-nvenc.yml): uses the FFmpeg video transcoding tool to encode a H.264 video stream with the [NVIDIA NVENC](https://developer.nvidia.com/nvidia-video-codec-sdk) hardware video encoder. This is useful for verifying that NVENC is functioning correctly when running on NVIDIA GPUs. This example only supports display devices. + +- [**nvidia-smi**](./examples/nvidia-smi/): runs the [NVIDIA System Management Interface](https://developer.nvidia.com/nvidia-system-management-interface) tool to enumerate NVIDIA GPUs and print their details. This is useful for verifying that the NVIDIA drivers can see the NVIDIA GPUs that were allocated to a container. Versions of this example are available for requesting [compute-only devices](./examples/nvidia-smi/nvidia-smi-mcdm.yml) and [display devices](./examples/nvidia-smi/nvidia-smi-wddm.yml). + +The following examples will only work when running on AMD, Intel or NVIDIA GPUs: + +- [**ffmpeg-autodetect**](./examples/ffmpeg-autodetect/ffmpeg-autodetect.yml): attempts to automatically detect the availability of AMD AMF, Intel Quick Sync or NVIDIA NVENC and uses the FFmpeg video transcoding tool to encode a H.264 video stream with the detected hardware video encoder. This is useful for verifying that hardware transcoding is functioning correctly when running on AMD, Intel or NVIDIA GPUs. This example only supports display devices. + + +## Building from source + +### Build requirements + +Building the device plugins from source requires the following: + +- Windows 10 version 2004 or newer / Windows Server 2022 or newer +- Visual Studio 2019 or newer (including at a minimum the [Desktop development with C++](https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2022&preserve-view=true#desktop-development-with-c) workload from the Build Tools or its equivalent for the full IDE) +- Windows 10 SDK version 10.0.18362.0 or newer (included by default with the *Desktop development with C++* workload) +- [CMake](https://cmake.org/) 3.22 or newer +- [Go](https://go.dev/) 1.18 or newer + +### Building the core components + +To build the device discovery library and the device plugins, simply run the following command from the root of the source tree: + +``` +build +``` + +The [build script](./build.ps1) will automatically download a number of external tools (such as the [NuGet CLI](https://docs.microsoft.com/en-us/nuget/reference/nuget-exe-cli-reference), [vcpkg](https://vcpkg.io/en/index.html) and [vswhere](https://github.com/microsoft/vswhere)) to the `external` subdirectory, use CMake and vcpkg to build the device discovery library (caching intermediate build files in the `build` subdirectory), and then use the Go compiler to build the Kubernetes device plugins. Once the build process is complete, you should see a number of binaries in the `bin` subdirectory: + +- `device-plugin-mcdm.exe`: the device plugin for MCDM + +- `device-plugin-wddm.exe`: the device plugin for WDDM + +- `directx-device-discovery.dll`: the device discovery library + +- `gen-device-mounts.exe`: a tool that generates flags for mounting devices and their accompanying DLL files into standalone containers without needing to deploy them to a Kubernetes cluster, for use when developing and testing container images + +- `query-hcs-capabilities.exe`: a utility to query the version of the Host Compute Service (HCS) schema supported by the operating system + +- `test-device-discovery-cpp.exe`: a test program that uses the device discovery library's C/C++ API to enumerate DirectX devices + +- `test-device-discovery-go.exe`: a test program that uses the device discovery library's Go API bindings to enumerate DirectX devices + +To clean all build artifacts, run the following command: + +``` +build -clean +``` + +This empties the `bin` subdirectory and removes the `build` subdirectory in its entirety. + +### Building container images + +The build script uses the [crane](https://github.com/google/go-containerregistry/tree/main/cmd/crane) image manipulation tool to build container images. Crane interacts directly with remote container registries, which means it does not require a container runtime like containerd or Docker to be installed locally. However, it does require the user to be authenticated with the remote container registry prior to building any images. To authenticate, run the following command (replacing the placeholders for username, password and container registry URL with the appropriate values): + +```bash +go run "github.com/google/go-containerregistry/cmd/crane@latest" auth login -u -p +``` + +For example, to authenticate to Docker Hub with the username "[contoso](https://devblogs.microsoft.com/oldnewthing/20061013-05/?p=29393)" and the password "example": + +```bash +go run "github.com/google/go-containerregistry/cmd/crane@latest" auth login -u contoso -p example index.docker.io +``` + +Container images can be built for both the Kubernetes device plugins and for the example workloads by specifying the `-images` flag or the `-examples` flag, respectively, when invoking the build script. Irrespective of the images being built, you will need to use the `-TagPrefix` flag to specify the tag prefix for the built container images and ensure they are pushed to the correct container registry. For most container registries, the tag prefix represents the registry URL and a username or organisation that acts as a namespace for images. For example, to build the images for the device plugins and push them to Docker Hub under the user "contoso", you would run the following command: + +``` +build -images -TagPrefix=index.docker.io/contoso +``` + +This will build and push the following container images: + +- `index.docker.io/contoso/device-plugin-mcdm:` +- `index.docker.io/contoso/device-plugin-wddm:` + +By default, `` is the tag name if you are building a release, or the Git commit hash when not building a release. You can override the version suffix by specifying the `-version` flag when invoking the build script: + +``` +build -images -TagPrefix=index.docker.io/contoso -version 0.0.0-dev +``` + +This will build and push the following container images: + +- `index.docker.io/contoso/device-plugin-mcdm:0.0.0-dev` +- `index.docker.io/contoso/device-plugin-wddm:0.0.0-dev` + +### Building the examples + +As stated in the section above, you can build the example applications and their container images by specifying the `-examples` flag when invoking the build script. Just as when building the container images for the device plugins, you will need to use the `-TagPrefix` flag to specify the tag prefix for the built container images, and the version suffix can optionally be controlled via the `-version` flag. For example, to push images to Docker Hub under the user "contoso" with the version suffix "0.0.0-dev": + +``` +build -examples -TagPrefix=index.docker.io/contoso -version 0.0.0-dev +``` + +This will build and push the following container images: + +- `index.docker.io/contoso/example-cuda-devicequery:0.0.0-dev` +- `index.docker.io/contoso/example-cuda-montecarlo:0.0.0-dev` +- `index.docker.io/contoso/example-device-discovery:0.0.0-dev` +- `index.docker.io/contoso/example-directml:0.0.0-dev` +- `index.docker.io/contoso/example-ffmpeg:0.0.0-dev` + +Note that you will need the [NVIDIA CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) version 11.6 installed in order to build the CUDA examples. If the build script detects that the CUDA Toolkit is not installed then it will skip building these examples. + +## Known limitations + +TensorWorks is aware of the following limitations that currently apply when using the Kubernetes device plugins for DirectX: + +- When running the device plugins on Kubernetes worker nodes with multiple DirectX devices, containers will sometimes see all host devices even when only a single device is mounted. We believe this is a bug in the underlying Host Compute Service (HCS) in Windows Server 2022 and investigation is ongoing. In the meantime, we recommend deploying the device plugins on worker nodes with only a single DirectX device, and limiting device requests to one per Kubernetes Pod. + +- The current version of the NVIDIA GPU driver does not appear to support Vulkan inside Windows containers. This is a limitation of the device driver itself, not the Kubernetes device plugins. + + +## Legal + +Copyright © 2022-2023, TensorWorks Pty Ltd. Licensed under the MIT License, see the file [LICENSE](./LICENSE) for details. diff --git a/bin/.gitignore b/bin/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/bin/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/build.bat b/build.bat new file mode 100644 index 0000000..f076167 --- /dev/null +++ b/build.bat @@ -0,0 +1 @@ +@powershell -ExecutionPolicy Bypass -File "%~dp0.\build.ps1" %* diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 0000000..465f3db --- /dev/null +++ b/build.ps1 @@ -0,0 +1,426 @@ +Param ( + [parameter(HelpMessage = "Clean existing build output")] + [switch] $Clean, + + [parameter(HelpMessage = "Build the device discovery library only, do not build the Kubernetes device plugins or container images")] + [switch] $LibraryOnly, + + [parameter(HelpMessage = "Build the Kubernetes device plugins only, do not build the device discovery library or container images")] + [switch] $PluginsOnly, + + [parameter(HelpMessage = "Build container images for the Kubernetes device plugins")] + [switch] $Images, + + [parameter(HelpMessage = "Build container images for example workloads")] + [switch] $Examples, + + [parameter(HelpMessage = "The prefix used for image tags when building container images")] + $TagPrefix = 'index.docker.io/tensorworks', + + [parameter(HelpMessage = "Override the repository version string for computing container image tag suffixes")] + $Version = '' +) + + +# Halt execution if we encounter an error +$ErrorActionPreference = 'Stop' + + +# Executes a command and throws an error if it returns a non-zero exit code +function Run-Command($command) { + + # Print the command and execute it + $formatted = If ($args.Count -gt 0) { "$command $args" } Else { "$command"} + Write-Host "[$formatted]" -ForegroundColor DarkYellow + & "$command" @args + + # If the command terminated with a non-zero exit code then throw an error + if ($LastExitCode -ne 0) { + throw "Command '$formatted' terminated with exit code $LastExitCode" + } +} + +# Invokes the Crane image management tool +function Invoke-Crane { + Run-Command go run 'github.com/google/go-containerregistry/cmd/crane@latest' @args +} + +# Builds a container image using Crane +function Build-ContainerImage { + Param ( + $BaseImage, + $Directory, + $Entrypoint, + $ImageTag, + $Files + ) + + # Create a tarball with the specified files + $tarball = "$env:Temp\layer-$([guid]::NewGuid()).tar" + Run-Command tar.exe --create --verbose --file="$tarball" --directory="$Directory" @Files + + # Append the filesystem layer to the base image and set the entrypoint + Invoke-Crane append --platform=windows/amd64 -b "$BaseImage" -f "$tarball" -t "$ImageTag" + Invoke-Crane mutate "$ImageTag" --entrypoint="$Entrypoint" + + # Perform cleanup + Remove-Item -Path "$tarball" -Force + + # Print the container image tag + Write-Host "Built image: $ImageTag" -ForegroundColor Cyan +} + + +# Resolve the path to the directories used during the build process +$buildDir = "$PSScriptRoot\build" +$binDir = "$PSScriptRoot\bin" +$examplesDir = "$PSScriptRoot\examples" +$externalDir = "$PSScriptRoot\external" +$vcpkgDir = "$externalDir\vcpkg" + +# Determine whether we are cleaning the build output +if ($Clean) +{ + # Remove the build directory if it exists + if ((Test-Path -Path $buildDir) -eq $true) + { + Write-Host 'Removing the build directory...' -ForegroundColor Green + Remove-Item -Path $buildDir -Recurse -Force + } + + # Remove all .dll and .exe files in the bin directory + Write-Host 'Removing binaries from the bin directory...' -ForegroundColor Green + Remove-Item -Path "$binDir\*.dll" -Force + Remove-Item -Path "$binDir\*.exe" -Force + Exit +} + +# Install vcpkg if we don't already have it +if ((Test-Path -Path $vcpkgDir) -eq $false) +{ + Write-Host "`nInstalling a local copy of vcpkg..." -ForegroundColor Green + Run-Command git clone 'https://github.com/Microsoft/vcpkg.git' "$vcpkgDir" + Run-Command "$vcpkgDir\bootstrap-vcpkg.bat" +} + +# Install vswhere if we don't already have it +$vsWhere = "$externalDir\vswhere.exe" +if ((Test-Path -Path $vsWhere) -eq $false) +{ + Write-Host "`nInstalling a local copy of vswhere..." -ForegroundColor Green + (New-Object System.Net.WebClient).DownloadFile( + 'https://github.com/microsoft/vswhere/releases/download/3.0.3/vswhere.exe', + "$vsWhere" + ) +} + +# Install the NuGet CLI tool if we don't already have it +$nuget = "$externalDir\nuget.exe" +$nugetConfig = "$externalDir\NuGet.Config" +if ((Test-Path -Path $nuget) -eq $false) +{ + # Install the CLI tool + Write-Host "`nInstalling a local copy of the NuGet CLI tool..." -ForegroundColor Green + (New-Object System.Net.WebClient).DownloadFile( + 'https://dist.nuget.org/win-x86-commandline/v6.2.1/nuget.exe', + "$nuget" + ) + + # Configure the CLI tool with the nuget.org source + Set-Content -Path "$nugetConfig" -Value '' -Force + Run-Command "$nuget" sources add -Name 'nuget.org' -Source 'https://api.nuget.org/v3/index.json' -ConfigFile "$nugetConfig" + Run-Command "$nuget" sources enable -Name 'nuget.org' -ConfigFile "$nugetConfig" +} + +# Use vswhere to determine the version of Visual Studio that is installed (2019, 2022, etc.) +$vsVersion = & "$vswhere" -latest -products * -requires Microsoft.Component.MSBuild -property 'catalog_productLineVersion' +if (!$vsVersion) { + throw "Failed to determine the version of Visual Studio" +} + +# Determine the platform toolset version that corresponds to the Visual Studio version +$toolsetVersions = @{ "2019" = "v142"; "2022" = "v143" } +$platformToolset = $toolsetVersions[$vsVersion] +if (!$platformToolset) { + throw "Failed to determine the platform toolset version that corresponds to the version of Visual Studio" +} + +# Use vswhere to locate MSBuild +$msBuild = & "$vswhere" -latest -products * -requires Microsoft.Component.MSBuild -find 'MSBuild\**\Bin\MSBuild.exe' +if (!$msBuild) { + throw "Failed to locate MSBuild" +} + +# Use vswhere to locate the MSBuild `BuildCustomizations` directory for the latest toolchain +$vsCustomisations = & "$vswhere" -latest -products * -requires Microsoft.Component.MSBuild -find 'MSBuild\Microsoft\VC\*\BuildCustomizations' +if (!$vsCustomisations) { + throw "Failed to locate the MSBuild BuildCustomizations directory" +} + +# If a version override was not specified then calculate it from the Git commit details +if (!$Version) +{ + # Set the version string to the commit hash + $Version = & git rev-parse HEAD + + # Determine whether the current commit is a tagged release + $gitTag = & git name-rev --name-only --tags HEAD + if ($gitTag -ne 'undefined') { + $Version = $gitTag + } +} + +# Report the detected values +Write-Host "Repository version: $Version" -ForegroundColor Cyan +Write-Host "Detected Visual Studio $vsVersion (platform toolset $platformToolset)" -ForegroundColor Cyan +Write-Host "Found MSBuild: $msBuild" -ForegroundColor Cyan + +if (!$PluginsOnly) +{ + # Create the build directory if it doesn't already exist + if ((Test-Path -Path $buildDir) -eq $false) + { + Write-Host "`nCreating the build directory..." -ForegroundColor Green + New-Item -Path $buildDir -ItemType Directory -Force | Out-Null + } + + # Build the device discovery library (vcpkg will automatically install all required dependencies) + Write-Host "`nBuilding the device discovery library..." -ForegroundColor Green + Push-Location "$buildDir" + Run-Command cmake "$PSScriptRoot/library" ` + -A x64 -DVCPKG_TARGET_TRIPLET=x64-windows ` + "-DCMAKE_INSTALL_PREFIX=$PSScriptRoot" ` + "-DCMAKE_TOOLCHAIN_FILE=$vcpkgDir\scripts\buildsystems\vcpkg.cmake" + Run-Command cmake --build . --target install --config Release + Pop-Location +} + +if (!$LibraryOnly) +{ + # Build the Kubernetes device plugins and strip the debug symbols from the executables + Write-Host "`nBuilding the Kubernetes device plugins..." -ForegroundColor Green + Push-Location "$PSScriptRoot/plugins" + $env:CGO_ENABLED=0 + Run-Command go build -o "$PSScriptRoot\bin" -ldflags '-s -w' ./... + Pop-Location +} + +if (!$PluginsOnly -and !$LibraryOnly -and $Images) +{ + # Use Crane to build the container image for the MCDM device plugin + Write-Host "`nBuilding and pushing the container image for the MCDM device plugin..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/mcdm-device-plugin:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/nanoserver:ltsc2022' ` + -Files @('directx-device-discovery.dll', 'device-plugin-mcdm.exe') ` + -Entrypoint 'device-plugin-mcdm.exe' ` + -Directory $binDir + + # Use Crane to build the container image for the WDDM device plugin + Write-Host "`nBuilding and pushing the container image for the WDDM device plugin..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/wddm-device-plugin:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/nanoserver:ltsc2022' ` + -Files @('directx-device-discovery.dll', 'device-plugin-wddm.exe') ` + -Entrypoint 'device-plugin-wddm.exe' ` + -Directory $binDir +} + +if (!$PluginsOnly -and !$LibraryOnly -and $Examples) +{ + # Use Crane to build the container image for the device discovery example + Write-Host "`nBuilding and pushing the container image for the device discovery example..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/example-device-discovery:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/servercore:ltsc2022' ` + -Files @('directx-device-discovery.dll', 'test-device-discovery-cpp.exe') ` + -Entrypoint 'C:\test-device-discovery-cpp.exe,--verbose' ` + -Directory $binDir + + # Clone the DirectML repository if we don't already have a copy + $directMLDir = "$externalDir\DirectML" + $directMLCommit = '3c5a947e0e4f8115dd5dd2fea00ac545120052ac' + if ((Test-Path -Path $directMLDir) -eq $false) + { + Write-Host "`nDownloading the DirectML repository..." -ForegroundColor Green + New-Item -Path $directMLDir -ItemType Directory -Force | Out-Null + Push-Location "$directMLDir" + Run-Command git init + Run-Command git remote add origin 'https://github.com/microsoft/DirectML.git' + Run-Command git fetch --depth=1 origin "$directMLCommit" + Run-Command git checkout "$directMLCommit" + Pop-Location + } + + # Build the HelloDirectML sample if it hasn't already been built + $helloDirectMLDir = "$directMLDir\Samples\HelloDirectML" + $helloDirectMLBin = "$helloDirectMLDir\HelloDirectML\x64\Release" + $helloDirectMLExe = "$helloDirectMLBin\HelloDirectML.exe" + if ((Test-Path -Path $helloDirectMLExe) -eq $false) + { + # Build the sample + Write-Host "`nBuilding the HelloDirectML sample..." -ForegroundColor Green + $env:_CL_='/MT' + Push-Location "$helloDirectMLDir" + Run-Command "$nuget" restore -ConfigFile "$nugetConfig" + Run-Command "$msBuild" HelloDirectML.vcxproj ` + /property:Configuration=Release ` + /property:Platform=x64 ` + "/property:PlatformToolset=$platformToolset" ` + /property:WindowsTargetPlatformVersion=10.0 + Pop-Location + + # Copy DirectML.dll from the NuGet package to the sample bin directory + Copy-Item -Path "$helloDirectMLDir\packages\Microsoft.AI.DirectML.1.7.0\bin\x64-win\DirectML.dll" -Destination "$helloDirectMLBin" -Force + } + + # Use Crane to build the container image for the DirectML example + Write-Host "`nBuilding and pushing the container image for the DirectML example..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/example-directml:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/servercore:ltsc2022' ` + -Files @('DirectML.dll', 'HelloDirectML.exe') ` + -Entrypoint 'C:\HelloDirectML.exe' ` + -Directory $helloDirectMLBin + + # Clone the OpenCL SDK repository if we don't already have a copy + $openCLSDK = "$externalDir\OpenCL-SDK" + if ((Test-Path -Path $openCLSDK) -eq $false) + { + Write-Host "`nDownloading the OpenCL SDK repository..." -ForegroundColor Green + Run-Command git clone -b 'v2022.05.18' --depth=1 --recursive 'https://github.com/KhronosGroup/OpenCL-SDK.git' "$openCLSDK" + } + + # Build the enumopencl sample if it hasn't already been built + $openCLBuild = "$openCLSDK\build" + $openCLBin = "$openCLBuild\bin\Release" + $enumOpenCL = "$openCLBin\enumopencl.exe" + if ((Test-Path -Path $enumOpenCL) -eq $false) + { + # Create a vcpkg manifest file to install the dependencies for the OpenCL samples + Set-Content -Force -Path "$openCLSDK\vcpkg.json" -Value '{"name":"opencl","version":"0.0.0","dependencies":["glm","sfml","tclap"]}' + + # Build the samples + Write-Host "`nBuilding the enumopencl sample..." -ForegroundColor Green + New-Item -Path $openCLBuild -ItemType Directory -Force | Out-Null + Push-Location "$openCLBuild" + Run-Command cmake "$openCLSDK" ` + -A x64 -DVCPKG_TARGET_TRIPLET=x64-windows ` + '-DCMAKE_CXX_FLAGS_RELEASE=/MT /O2 /Ob2 /DNDEBUG' ` + '-DCMAKE_C_FLAGS_RELEASE=/MT /O2 /Ob2 /DNDEBUG' ` + "-DCMAKE_INSTALL_PREFIX=$openCLSDK" ` + "-DCMAKE_TOOLCHAIN_FILE=$vcpkgDir\scripts\buildsystems\vcpkg.cmake" ` + -DBUILD_TESTING=OFF ` + -DBUILD_DOCS=OFF ` + -DBUILD_EXAMPLES=OFF ` + -DBUILD_TESTS=OFF ` + -DOPENCL_SDK_BUILD_SAMPLES=ON ` + -DOPENCL_SDK_TEST_SAMPLES=OFF + Run-Command cmake --build . --target enumopencl --config Release + Pop-Location + } + + # Use Crane to build the container image for the enumopencl example + Write-Host "`nBuilding and pushing the container image for the enumopencl example..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/example-opencl-enum:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/servercore:ltsc2022' ` + -Files @('enumopencl.exe') ` + -Entrypoint 'C:\enumopencl.exe' ` + -Directory $openCLBin + + # Determine whether the CUDA Toolkit v11.6 is installed + if ($env:CUDA_PATH_V11_6) + { + # Copy the Visual Studio integration files from the CUDA Toolkit to the MSBuild `BuildCustomizations` directory + Copy-Item -Path "$env:CUDA_PATH\extras\visual_studio_integration\MSBuildExtensions\*" -Destination "$vsCustomisations" -Force + + # Clone the CUDA samples repository if we don't already have a copy + $cudaSamples = "$externalDir\cuda-samples" + $cudaSamplesBin = "$cudaSamples\bin\win64\Release" + if ((Test-Path -Path $cudaSamples) -eq $false) + { + Write-Host "`nDownloading the CUDA samples repository..." -ForegroundColor Green + Run-Command git clone -b 'v11.6' --depth=1 'https://github.com/NVIDIA/cuda-samples.git' "$cudaSamples" + } + + # Build the CUDA deviceQuery sample if it hasn't already been built + $deviceQuery = "$cudaSamplesBin\deviceQuery.exe" + if ((Test-Path -Path $deviceQuery) -eq $false) + { + Write-Host "`nBuilding the CUDA deviceQuery sample..." -ForegroundColor Green + Run-Command "$msBuild" "$cudaSamples\Samples\1_Utilities\deviceQuery\deviceQuery_vs${vsVersion}.sln" /property:Configuration=Release + } + + # Build the CUDA MC_EstimatePiP sample if it hasn't already been built + $monteCarlo = "$cudaSamplesBin\MC_EstimatePiP.exe" + if ((Test-Path -Path $monteCarlo) -eq $false) + { + # Build the sample + Write-Host "`nBuilding the CUDA MC_EstimatePiP sample..." -ForegroundColor Green + Run-Command "$msBuild" "$cudaSamples\Samples\2_Concepts_and_Techniques\MC_EstimatePiP\MC_EstimatePiP_vs${vsVersion}.sln" /property:Configuration=Release + + # Copy curand64_10.dll from the CUDA Toolkit bin directory to the sample bin directory + Copy-Item -Path "$env:CUDA_PATH_V11_6\bin\curand64_10.dll" -Destination "$cudaSamplesBin" -Force + } + + # Use Crane to build the container image for the CUDA deviceQuery example + Write-Host "`nBuilding and pushing the container image for the CUDA deviceQuery example..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/example-cuda-devicequery:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/servercore:ltsc2022' ` + -Files @('deviceQuery.exe') ` + -Entrypoint 'C:\deviceQuery.exe' ` + -Directory $cudaSamplesBin + + # Use Crane to build the container image for the CUDA MC_EstimatePiP example + Write-Host "`nBuilding and pushing the container image for the CUDA MC_EstimatePiP example..." -ForegroundColor Green + Build-ContainerImage ` + -ImageTag "$TagPrefix/example-cuda-montecarlo:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/servercore:ltsc2022' ` + -Files @('curand64_10.dll', 'MC_EstimatePiP.exe') ` + -Entrypoint 'C:\MC_EstimatePiP.exe' ` + -Directory $cudaSamplesBin + } + else { + Write-Host "`nCUDA Toolkit v11.6 was not detected, not building CUDA examples" -ForegroundColor Cyan + } + + # Install FFmpeg if we don't already have it + $ffmpegDir = "$externalDir\ffmpeg" + if ((Test-Path -Path $ffmpegDir) -eq $false) + { + # Download the FFmpeg release archive + Write-Host "`nInstalling a local copy of FFmpeg..." -ForegroundColor Green + $ffmpegZip = "$externalDir\ffmpeg.zip" + (New-Object System.Net.WebClient).DownloadFile( + 'https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-5.1.2-essentials_build.zip', + "$ffmpegZip" + ) + + # Extract the archive + Expand-Archive -Path "$ffmpegZip" -DestinationPath "$ffmpegDir" -Force + } + + # Download a sample video for use with FFmpeg if we don't already have it + $ffmpegBin = "$ffmpegDir\ffmpeg-5.1.2-essentials_build\bin" + $sampleVideo = "$ffmpegBin\sample-video.mp4" + if ((Test-Path -Path $sampleVideo) -eq $false) + { + Write-Host "`nDownloading a sample video for use with FFmpeg..." -ForegroundColor Green + (New-Object System.Net.WebClient).DownloadFile( + 'https://raw.githubusercontent.com/SPBTV/video_av1_samples/master/spbtv_sample_bipbop_av1_960x540_25fps.mp4', + "$sampleVideo" + ) + } + + # Use Crane to build the container image for the FFmpeg examples + Write-Host "`nBuilding and pushing the container image for the FFmpeg examples..." -ForegroundColor Green + Copy-Item -Path "$examplesDir\ffmpeg-autodetect\autodetect-encoder.ps1" -Destination "$ffmpegBin" -Force + Build-ContainerImage ` + -ImageTag "$TagPrefix/example-ffmpeg:$Version" ` + -BaseImage 'mcr.microsoft.com/windows/server:ltsc2022' ` + -Files @('ffmpeg.exe', 'sample-video.mp4', 'autodetect-encoder.ps1') ` + -Entrypoint 'C:\ffmpeg.exe' ` + -Directory "$ffmpegBin" +} diff --git a/cloud/aws/README.md b/cloud/aws/README.md new file mode 100644 index 0000000..126ac41 --- /dev/null +++ b/cloud/aws/README.md @@ -0,0 +1,115 @@ +# Amazon EKS demo deployment + +This directory contains scripts that can be used to deploy the Kubernetes device plugins for DirectX to an [Amazon EKS](https://aws.amazon.com/eks/) Kubernetes cluster for demonstration purposes. Note that the deployment created by these scripts **is not intended for production use**, and lacks important functionality such as auto-scaling the Windows node group based on requests for DirectX devices. + +The [main deployment script](./deploy.ps1) performs the following steps: + +- Builds a custom [Amazon Machine Image (AMI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) based on Windows Server 2022 for use by Kubernetes worker nodes, with the NVIDIA GPU drivers and containerd v1.7.0 installed. The supporting scripts for building the AMI are located in the [node](./node) subdirectory. + +- Creates a EKS cluster with a Windows node group of `g4dn.xlarge` instances that is configured to use the custom AMI. The supporting configuration files for creating the cluster are located in the [cluster](./cluster) subdirectory. + +- Deploys the Kubernetes device plugins for DirectX to the EKS cluster using the [default HostProcess DaemonSets for the MCDM device plugin and the WDDM device plugin](../../deployments/default-daemonsets.yml). + + +## Contents + +- [Requirements](#requirements) +- [Running the deployment script](#running-the-deployment-script) +- [Testing the cluster](#testing-the-cluster) +- [Cleaning up](#cleaning-up) + + +## Requirements + +To use the deployment scripts, the following requirements must be met: + +- The AWS region that you are using needs to have sufficient quota to run at least one `g4dn.xlarge` EC2 instance. To view or change the relevant limit, login to the AWS web console and navigate to the [*Running On-Demand G and VT instances*](https://console.aws.amazon.com/servicequotas/home/services/ec2/quotas/L-DB2E81BA) service quota page. The minimum required value is 4 vCPUs. + +- The AWS region that you are using needs to have a default VPC configured with at least one subnet. If you have deleted the default VPC for the target region then you will need to [create a new one](https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html#create-default-vpc). + +- [Microsoft PowerShell](https://github.com/PowerShell/PowerShell) needs to be installed when running the deployment scripts under Linux or macOS systems. (Under Windows, the built-in Windows PowerShell is used instead.) + +- The [AWS CLI](https://docs.aws.amazon.com/cli/) needs to be installed and configured with credentials that permit the creation of AMIs and EKS clusters. For details, see [*Configuring the AWS CLI*](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). + +- [eksctl](https://eksctl.io/) version [0.111.0](https://github.com/weaveworks/eksctl/blob/v0.111.0/docs/release_notes/0.110.0.md) or newer needs to be installed (older versions will refuse to create Windows node groups with GPUs). + +- [HashiCorp Packer](https://www.packer.io/) needs to be installed. + +- [kubectl](https://kubernetes.io/docs/reference/kubectl/) needs to be installed. + + +## Running the deployment script + +Under Windows, run the main deployment script using the following command: + +``` +deploy.bat +``` + +Under Linux and macOS, use this command instead: + +```bash +pwsh deploy.ps1 +``` + +The following optional flags can be used to control the deployment options: + +- `-Region`: specifies the AWS region into which resources will be deployed. The default region is `us-east-1`. + +- `-AmiName`: specifies the name to use for the custom worker node AMI. The default name is `eks-worker-node`. + +- `-ClusterName`: specifies the name to use for the EKS cluster. The default name is `demo-cluster`. + +An example usage of these flags is shown below: + +```bash +# Deploys to the Sydney (ap-southeast-2) AWS region and uses custom names for both the AMI and the EKS cluster +pwsh deploy.ps1 -Region "ap-southeast-2" -AmiName "my-custom-ami" -ClusterName "my-test-cluster" +``` + + +## Testing the cluster + +Once the EKS cluster has been created, eksctl will configure kubectl to communicate with that cluster by default. This means you can start using kubectl to deploy examples from the top-level [examples](../../examples) directory without the need for any additional configuration steps: + +1. The first example you should deploy is the [**device-discovery**](../../examples/device-discovery/) test, which acts as a sanity check to verify that GPUs are being exposed to containers correctly: + + ```bash + kubectl apply -f '../../examples/device-discovery/device-discovery-wddm.yml' + ``` + + Once the Job has been created, wait for the Pod to be assigned to a Windows worker node and then run to completion. If the Job finishes with a status of "Succeeded" then you should check the Pod logs to verify that the NVIDIA Tesla T4 GPU is listed in the output. If the Job finishes with a status of "Failed" or if the log output lists zero devices then something has gone wrong. A failure here could indicate an issue with the Kubernetes device plugins for DirectX themselves, or with some aspect of the EKS cluster configuration. + +2. Since the EKS worker nodes are using NVIDIA GPUs, the second example you should deploy is the [**nvidia-smi**](../../examples/nvidia-smi/) test, which acts as a sanity check to verify that the NVIDIA GPU drivers are able to communicate with the GPU: + + ```bash + kubectl apply -f '../../examples/nvidia-smi/nvidia-smi-wddm.yml' + ``` + + If the Job finishes with a status of "Succeeded" then `nvidia-smi` was able to communicate with the GPU, and you can check the Pod logs to verify that the output is as expected. If the Job finishes with a status of "Failed" then this indicates either an issue with the Kubernetes device plugins for DirectX themselves or with the NVIDIA GPU drivers on the worker node. + +3. With these basic sanity checks out of the way, you can then try out any of the other examples that work on NVIDIA GPUs. Since the NVIDIA Tesla T4 GPUs provided by `g4dn.xlarge` EC2 instances support both compute and display, you will need to deploy examples that request a `directx.microsoft.com/display` resource. Note that some examples include YAML files for both compute-only (MCDM) and compute+display (WDDM) requests, so be sure to use the version with a filename suffix of `-wddm.yml`. + + Note that if you attempt to run two tests at once, one of them will wait for the other to complete before it can be scheduled. This is because the Windows node group will not automatically scale up in response to requests for DirectX devices, so only one node (and thus one GPU) will be available to allocate to containers at any given time. + + +## Cleaning up + +To delete all previously deployed AWS resources, run the main deployment script with the `-Clean` flag. Under Windows: + +``` +deploy.bat -Clean +``` + +Under Linux and macOS: + +```bash +pwsh deploy.ps1 -Clean +``` + +If you specified flags for the AWS region or custom resource names when deploying the resources then be sure to include these flags when deleting them as well. For example: + +```bash +# Deletes the AWS resources that were deployed in the example from the earlier section +pwsh deploy.ps1 -Region "ap-southeast-2" -AmiName "my-custom-ami" -ClusterName "my-test-cluster" -Clean +``` diff --git a/cloud/aws/cluster/.gitignore b/cloud/aws/cluster/.gitignore new file mode 100644 index 0000000..f082128 --- /dev/null +++ b/cloud/aws/cluster/.gitignore @@ -0,0 +1 @@ +test-cluster.yml diff --git a/cloud/aws/cluster/test-cluster.template b/cloud/aws/cluster/test-cluster.template new file mode 100644 index 0000000..463f028 --- /dev/null +++ b/cloud/aws/cluster/test-cluster.template @@ -0,0 +1,24 @@ +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: "__CLUSTER_NAME__" + region: "__AWS_REGION__" + version: "1.24" + +nodeGroups: + - name: windows + ami: "__AMI_ID__" + amiFamily: WindowsServer2022FullContainer + preBootstrapCommands: ["net user Administrator \"Passw0rd!\""] + instanceType: g4dn.xlarge + containerRuntime: containerd + volumeSize: 100 + minSize: 1 + maxSize: 3 + +managedNodeGroups: + - name: linux + instanceType: t2.large + minSize: 2 + maxSize: 3 diff --git a/cloud/aws/deploy.bat b/cloud/aws/deploy.bat new file mode 100644 index 0000000..3803b23 --- /dev/null +++ b/cloud/aws/deploy.bat @@ -0,0 +1 @@ +@powershell -ExecutionPolicy Bypass -File "%~dp0.\deploy.ps1" %* diff --git a/cloud/aws/deploy.ps1 b/cloud/aws/deploy.ps1 new file mode 100644 index 0000000..e87abf3 --- /dev/null +++ b/cloud/aws/deploy.ps1 @@ -0,0 +1,305 @@ +Param ( + [parameter(HelpMessage = "Remove existing resources created by a previous run")] + [switch] $Clean, + + [parameter(HelpMessage = "The AWS region in which to deploy resources")] + $Region = 'us-east-1', + + [parameter(HelpMessage = "The name to use for the custom worker node AMI")] + $AmiName = 'eks-worker-node', + + [parameter(HelpMessage = "The name to use for the EKS cluster")] + $ClusterName = 'demo-cluster' +) + + +# Halt execution if we encounter an error +$ErrorActionPreference = 'Stop' + + +# Replaces the placeholders in a template file with values and writes the output to a new file +function FillTemplate +{ + Param ( + $Template, + $Rendered, + $Values + ) + + $filled = Get-Content -Path $Template -Raw + $Values.GetEnumerator() | ForEach-Object { + $filled = $filled.Replace($_.Key, $_.Value) + } + Set-Content -Path $Rendered -Value $filled -NoNewline +} + +# Represents the output of a native process +class ProcessOutput +{ + ProcessOutput([string] $stdout, [string] $stderr) + { + $this.StandardOutput = $stdout + $this.StandardError = $stderr + } + + [string] $StandardOutput + [string] $StandardError +} + +# Helper functions for executing native commands +class ExecutionHelpers +{ + # Escapes command-line arguments for passing to a native command + static [string] EscapeArguments([string[]] $arguments) + { + $escaped = @() + + foreach ($arg in $arguments) + { + if ($arg.Contains(' ')) { + $escaped += @("`"$arg`"") + } + else { + $escaped += @($arg) + } + } + + return $escaped -join ' ' + } + + # Executes a command and throws an error if it returns a non-zero exit code + static [ProcessOutput] RunCommand([string] $command, [string[]] $arguments, [bool] $captureStdOut, [bool] $captureStdErr) + { + # Log the command + $escapedArgs = [ExecutionHelpers]::EscapeArguments($arguments) + $formatted = "[$command $escapedArgs]" + Write-Host "$formatted" -ForegroundColor DarkYellow + + # Execute the command and wait for it to complete, retrieving the exit code, stdout and stderr + $info = New-Object System.Diagnostics.ProcessStartInfo + $info.FileName = $command + $info.Arguments = $escapedArgs + $info.RedirectStandardError = $captureStdErr + $info.RedirectStandardOutput = $captureStdOut + $info.UseShellExecute = $false + $info.WorkingDirectory = (Get-Location).ToString() + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $info + $process.Start() + $process.WaitForExit() + $exitCode = $process.ExitCode + $stdout = if ($captureStdOut) { $process.StandardOutput.ReadToEnd() } else { '' } + $stderr = if ($captureStdErr) { $process.StandardError.ReadToEnd() } else { '' } + + # If the command terminated with a non-zero exit code then throw an error + if ($exitCode -ne 0) { + throw "Command $formatted terminated with exit code $exitCode, stdout $stdout and stderr $stderr" + } + + # Return the output + return [ProcessOutput]::new($stdout, $stderr) + } + + # Do not capture stdout and stderr of child processes unless the caller explicitly requests it + static [void] RunCommand([string] $command, [string[]] $arguments) { + [ExecutionHelpers]::RunCommand($command, $arguments, $false, $false) + } + + # Tests whether the specified command exists, by attempting to execute it with the supplied arguments + static [bool] CommandExists([string] $command, [string[]] $testArguments) + { + try + { + [ExecutionHelpers]::RunCommand($command, $testArguments, $true, $true) + return $true + } + catch { + return $false + } + } +} + +# Represents the Packer manifest data for our EKS worker node AMI +class PackerManifest +{ + PackerManifest([string] $path) { + $this.ManifestPath = $path + } + + [bool] Exists() { + return (Test-Path -Path $this.ManifestPath) + } + + [void] Parse() + { + # Parse the Packer manifest JSON and validate the AMI details + $manifestDetails = Get-Content -Path $this.ManifestPath -Raw | ConvertFrom-Json + $amiDetails = ($manifestDetails.builds[0].artifact_id -split ':') + if ($amiDetails.Length -lt 2) { + throw "Malformed 'artifact_id' field in Packer build manifest: '$amiDetails'" + } + + # Extract the region and AMI ID + $this.AmiRegion = $amiDetails[0] + $this.AmiID = $amiDetails[1] + + # If the manifest data doesn't contain the snapshot ID for the AMI then populate it + $this.SnapshotID = $manifestDetails.builds[0].custom_data.snapshot_id + if ($this.SnapshotID.Length -lt 1) + { + # Attempt to retrieve the snapshot ID from the AWS API + Write-Host 'Retrieving the snapshot ID for the AMI...' -ForegroundColor Green + $queryOutput = [ExecutionHelpers]::RunCommand('aws', @('ec2', 'describe-images', "--region=$($this.AmiRegion)", "--image-ids=$($this.AmiID)"), $true, $true) + $snapshotDetails = $queryOutput.StandardOutput | ConvertFrom-Json + $this.SnapshotID = $snapshotDetails.Images[0].BlockDeviceMappings[0].Ebs.SnapshotId + if ($amiDetails.Length -lt 1) { + throw "Failed to retrieve snapshot ID for AMI: '$this.AmiID'" + } + + # Inject the snapshot ID into the manifest data + $manifestDetails.builds[0].custom_data.snapshot_id = $this.SnapshotID + + # Write the updated manifest data back to the JSON file + $manifestJson = ConvertTo-Json $manifestDetails -Depth 32 + Set-Content -Path $this.ManifestPath -Value $manifestJson -NoNewline + } + } + + [void] Delete() + { + # De-register the AMI + [ExecutionHelpers]::RunCommand('aws', @('ec2', 'deregister-image', "--region=$($this.AmiRegion)", "--image-id=$($this.AmiID)")) + + # Remove the snapshot + [ExecutionHelpers]::RunCommand('aws', @('ec2', 'delete-snapshot', "--region=$($this.AmiRegion)", "--snapshot-id=$($this.SnapshotID)")) + + # Delete the manifest JSON file + Remove-Item -Force $this.ManifestPath + } + + [string] $ManifestPath + [string] $AmiID + [string] $AmiRegion + [string] $SnapshotID +} + +# Represents an EKS cluster managed by eksctl +class EksCluster +{ + EksCluster([string] $name) { + $this.Name = $name + } + + [bool] Exists() + { + try + { + [ExecutionHelpers]::RunCommand('eksctl', @('get', 'cluster', "--name=$($this.Name)", "--region=$($global:Region)"), $true, $true) + return $true + } + catch { + return $false + } + } + + [void] Create([string] $yamlFile) { + [ExecutionHelpers]::RunCommand('eksctl', @('create', 'cluster', '-f', $yamlFile.Replace('\', '/'))) + } + + [void] Delete() { + [ExecutionHelpers]::RunCommand('eksctl', @('delete', 'cluster', "--name=$($this.Name)", "--region=$($global:Region)")) + } + + [string] $Name +} + + +# Verify that all of the native commands we require are available +$requiredCommands = @{ + 'the AWS CLI' = [ExecutionHelpers]::CommandExists('aws', @('help')); + 'eksctl' = [ExecutionHelpers]::CommandExists('eksctl', @('version')); + 'kubectl' = [ExecutionHelpers]::CommandExists('kubectl', @('help')); + 'HashiCorp Packer' = [ExecutionHelpers]::CommandExists('packer', @('version')) +} +foreach ($command in $requiredCommands.GetEnumerator()) +{ + if ($command.Value -eq $false) { + throw "Error: $($command.Name) must be installed to run this script!" + } +} + +# Resolve the path to the Packer manifest file and create a helper object to represent the manifest data +$packerDir = "$PSScriptRoot\node" +$packerManifest = [PackerManifest]::new("$packerDir\manifest.json") + +# Create a helper object to represent our test EKS cluster +$eksCluster = [EksCluster]::new($global:ClusterName) + +# Determine whether we are removing existing resources created by a previous run +if ($Clean) +{ + # Remove the EKS cluster if it exists + if ($eksCluster.Exists()) + { + Write-Host 'Removing existing EKS cluster...' -ForegroundColor Green + $eksCluster.Delete() + } + + # Delete the AMI and its accompanying snapshot if they exist + if ($packerManifest.Exists()) + { + Write-Host 'Removing AMI and its accompanying snapshot...' -ForegroundColor Green + $packerManifest.Parse() + $packerManifest.Delete() + } + + Exit +} + +# Build the custom worker node AMI if it doesn't already exist +if ($packerManifest.Exists() -eq $false) +{ + # Populate the Packer template + $packerfile = "$packerDir\eks-worker-node.pkr.hcl" + FillTemplate ` + -Template "$packerDir\eks-worker-node.pkr.hcl.template" ` + -Rendered $packerfile ` + -Values @{ + '__AWS_REGION__' = $global:Region; + '__AMI_NAME__' = $global:AmiName + } + + # Build the AMI + Write-Host 'Building the EKS custom worker node AMI...' -ForegroundColor Green + Push-Location "$packerDir" + [ExecutionHelpers]::RunCommand('packer', @('init', 'eks-worker-node.pkr.hcl')) + [ExecutionHelpers]::RunCommand('packer', @('build', 'eks-worker-node.pkr.hcl')) + Pop-Location +} + +# Parse the Packer manifest JSON and validate the AMI details +$packerManifest.Parse() + +# Populate the cluster template YAML with the values for the AMI +$clusterDir = "$PSScriptRoot\cluster" +$configFile = "$clusterDir\test-cluster.yml" +FillTemplate ` + -Template "$clusterDir\test-cluster.template" ` + -Rendered $configFile ` + -Values @{ + '__CLUSTER_NAME__' = $global:ClusterName; + '__AWS_REGION__' = $packerManifest.AmiRegion; + '__AMI_ID__' = $packerManifest.AmiID + } + +# Deploy the test EKS cluster if it doesn't already exist +if ($eksCluster.Exists() -eq $false) +{ + Write-Host 'Deploying a test EKS cluster with a Windows worker node group using the custom AMI...' -ForegroundColor Green + $eksCluster.Create($configFile) +} + +# Deploy the device plugin DaemonSets to the test cluster +Write-Host 'Deploying the DirectX device plugin DaemonSets to the test EKS cluster...' -ForegroundColor Green +$deploymentsYaml = "$PSScriptRoot\..\..\deployments\default-daemonsets.yml" +[ExecutionHelpers]::RunCommand('kubectl', @('apply', '-f', $deploymentsYaml.Replace('\', '/'))) diff --git a/cloud/aws/node/.gitignore b/cloud/aws/node/.gitignore new file mode 100644 index 0000000..b3999a8 --- /dev/null +++ b/cloud/aws/node/.gitignore @@ -0,0 +1,2 @@ +eks-worker-node.pkr.hcl +manifest.json diff --git a/cloud/aws/node/eks-worker-node.pkr.hcl.template b/cloud/aws/node/eks-worker-node.pkr.hcl.template new file mode 100644 index 0000000..3603814 --- /dev/null +++ b/cloud/aws/node/eks-worker-node.pkr.hcl.template @@ -0,0 +1,79 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.0.9" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "eks-worker-node" { + ami_name = "__AMI_NAME__" + instance_type = "g4dn.xlarge" + region = "__AWS_REGION__" + + # Use the latest version of the official Windows Server 2022 base image + source_ami_filter { + filters = { + name = "Windows_Server-2022-English-Full-Base-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + + most_recent = true + owners = ["amazon"] + } + + # Expand the boot disk to 100GB + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 100 + volume_type = "gp3" + delete_on_termination = true + } + + # Allow S3 access for the VM + temporary_iam_instance_profile_policy_document { + Version = "2012-10-17" + Statement { + Action = ["s3:Get*", "s3:List*"] + Effect = "Allow" + Resource = ["*"] + } + } + + # Use our startup script to enable SSH access + user_data_file = "${path.root}/scripts/startup.ps1" + + # Use SSH for running commands in the VM + communicator = "ssh" + ssh_username = "Administrator" + ssh_timeout = "30m" + + # Don't automatically stop the instance, since sysprep will perform the shutdown + disable_stop_instance = true +} + +build { + name = "eks-worker-node" + sources = ["source.amazon-ebs.eks-worker-node"] + + # Run our EKS worker node setup script + provisioner "powershell" { + script = "${path.root}/scripts/setup.ps1" + } + + # Perform cleanup and shut down the VM + provisioner "powershell" { + script = "${path.root}/scripts/cleanup.ps1" + valid_exit_codes = [0, 2300218] + } + + # Store the AMI ID in a manifest file when the build completes + post-processor "manifest" { + output = "manifest.json" + custom_data = { + snapshot_id = "" + } + } +} diff --git a/cloud/aws/node/generate-setup-script.py b/cloud/aws/node/generate-setup-script.py new file mode 100755 index 0000000..76940b8 --- /dev/null +++ b/cloud/aws/node/generate-setup-script.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +# +# This script automates the generation of `setup.ps1` in the `scripts` subdirectory +# +import json, re, subprocess, yaml, sys +from pathlib import Path + + +class Utility: + + @staticmethod + def log(message): + """ + Logs a message to stderr + """ + print('[generate-setup-script.py]: {}'.format(message), flush=True, file=sys.stderr) + + @staticmethod + def capture(command, **kwargs): + """ + Executes the specified command and captures its output + """ + + # Log the command being executed + Utility.log(command) + + # Attempt to execute the specified command + result = subprocess.run( + command, + check = True, + capture_output = True, + universal_newlines = True, + **kwargs + ) + + # Return the contents of stdout + return result.stdout.strip() + + @staticmethod + def writeFile(filename, data): + """ + Writes data to the specified file + """ + return Path(filename).write_bytes(data.encode('utf-8')) + + @staticmethod + def commentForStep(name): + """ + Returns a descriptive comment for the build step with the specified name + """ + return { + + 'ConfigureDirectories': '# Create each of our directories', + 'DownloadKubernetes': '# Download the Kubernetes components', + 'DownloadEKSArtifacts': '# Download the EKS artifacts archive', + 'ExtractEKSArtifacts': '# Extract the EKS artifacts archive', + 'MoveEKSArtifacts': '# Move the EKS files into place', + 'ExecuteBuildScripts': '# Perform EKS worker node setup', + 'RemoveEKSArtifactDownloadDirectory': '# Perform cleanup', + + 'InstallContainers': '\n'.join([ + '# Install the Windows Containers feature', + '# (Note: this is actually a no-op here, since we install the feature beforehand in startup.ps1)' + ]) + + }.get(name, None) + + @staticmethod + def parseConstants(constants): + """ + Parses an EC2 ImageBuilder component's constants list + """ + parsed = {} + for entry in constants: + for key, values in entry.items(): + parsed[key] = values['value'] + return parsed + + @staticmethod + def replaceConstants(string, constants): + """ + Converts EC2 ImageBuilder constant references to PowerShell variable references + """ + + # If the value of a constant is used as a magic value rather than a reference, + # replace it with a reference to the variable representing the constant instead + transformed = string + for key, value in constants.items(): + transformed = transformed.replace(value, '${}'.format(key)) + + # Convert `{{ variable }}` syntax to PowerShell `$variable` syntax + # (Note that we don't bother to wrap the variable names in curly braces, since we know that none + # of the variable names contain special characters, and they're only ever interpolated as either + # part of a filesystem path surrounded by separators, or as a parameter surrounded by whitespace) + return re.sub('{{ (.+?) }}', '$\\1', transformed) + + @staticmethod + def replaceSystemPaths(path): + """ + Replaces hard-coded system paths with the equivalent environment variables + """ + replaced = path + replaced = replaced.replace('C:\\Program Files', '$env:ProgramFiles') + replaced = replaced.replace('C:\\ProgramData', '$env:ProgramData') + return replaced + + @staticmethod + def s3UriToHttpsUrl(s3Uri): + """ + Converts an `s3://` URI to an HTTPS URL + """ + url = s3Uri.replace('s3://', '') + components = url.split('/', 1) + return 'https://{}.s3.amazonaws.com/{}'.format(components[0], components[1]) + + +# Retrieve the contents of the "Amazon EKS Optimized Windows AMI" EC2 ImageBuilder component +componentData = json.loads(Utility.capture([ + 'aws', + 'imagebuilder', + 'get-component', + '--region=us-east-1', + '--component-build-version-arn', + 'arn:aws:imagebuilder:us-east-1:aws:component/eks-optimized-ami-windows/1.24.0' +])) + +# Parse the pipeline YAML data and extract the list of constants +pipelineData = yaml.load(componentData['component']['data'], Loader=yaml.Loader) +constants = Utility.parseConstants(pipelineData['constants']) + +# Extract the steps for the "build" phase +buildSteps = [p['steps'] for p in pipelineData['phases'] if p['name'] == 'build'][0] + +print('CONSTANTS:') +print(json.dumps(constants, indent=4)) + +print() +print('BUILD STEPS:') +print(json.dumps(buildSteps, indent=4)) + +# Prepend our header to the generated PowerShell code +generated = '''<# + THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT! + + This script is based on the logic from the "Amazon EKS Optimized Windows AMI" + EC2 ImageBuilder component, with modifications to use containerd 1.7.0. + + The original ImageBuilder component logic is Copyright Amazon.com, Inc. or + its affiliates, and is licensed under the MIT License. +#> + +# Halt execution if we encounter an error +$ErrorActionPreference = 'Stop' + + +# Applies in-place patches to a file +function PatchFile +{ + Param ( + $File, + $Patches + ) + + $patched = Get-Content -Path $File -Raw + $Patches.GetEnumerator() | ForEach-Object { + $patched = $patched.Replace($_.Key, $_.Value) + } + Set-Content -Path $File -Value $patched -NoNewline +} + + +''' + +# Inject an additional constant for the parent of the temp directory, immediately before the child directory +tempPath = {k:v for k,v in constants.items() if k == 'TempPath'} +otherConstants = {k:v for k,v in constants.items() if k != 'TempPath'} +constants = {**otherConstants, 'TempRoot': 'C:\\TempEKSArtifactDir', **tempPath} + +# Define variables for each of our constants +generated += '# Constants\n' +existingConstants = {} +for key, value in constants.items(): + transformed = Utility.replaceConstants(value, existingConstants) + transformed = Utility.replaceSystemPaths(transformed) + generated += '${} = "{}"\n'.format(key, transformed) + existingConstants[key] = value + +# Process each build step in turn +for step in buildSteps: + + # Determine whether we have custom preprocessing logic for the step + name = step['name'] + if name == 'ConfigureDirectories': + + # Add the temp directory to the list of directories to be created + step['loop']['forEach'] += [constants['TempRoot']] + + elif name == 'DownloadKubernetes': + + # Inject the driver installation step immediately prior to the Kubernetes download step + generated += '\n'.join([ + '', + '# Install the NVIDIA GPU drivers', + "$driverBucket = 'ec2-windows-nvidia-drivers'", + "$driver = Get-S3Object -BucketName $driverBucket -KeyPrefix 'latest' -Region 'us-east-1' | Where-Object {$_.Key.Contains('server2022')}", + 'Copy-S3Object -BucketName $driverBucket -Key $driver.Key -LocalFile "$TempRoot\driver.exe" -Region \'us-east-1\'', + "Start-Process -FilePath \"$TempRoot\driver.exe\" -ArgumentList @('-s', '-noreboot') -NoNewWindow -Wait", + '' + ]) + + elif name == 'ExtractEKSArtifacts': + + # Remove the redundant directory creation command + step['inputs']['commands'] = [ + c for c in step['inputs']['commands'] + if not c.startswith('New-Item') + ] + + # Use absolute file and directory paths rather than relative paths + step['inputs']['commands'] = [ + c.replace('EKS-Artifacts.zip', '"C:\\EKS-Artifacts.zip"').replace('TempEKSArtifactDir', 'C:\\TempEKSArtifactDir') + for c in step['inputs']['commands'] + ] + + elif name == 'InstallContainerRuntimes': + + # Inject the containerd 1.7.0 download step, along with our configuration patching steps, immediately prior to the containerd installation step + generated += '\n'.join([ + '', + '# -------', + '', + '# TEMPORARY UNTIL EKS ADDS SUPPORT FOR CONTAINERD v1.7.0:', + '# Download and extract the containerd 1.7.0 release build', + '$containerdTarball = "$TempPath\\containerd-1.7.0.tar.gz"', + '$containerdFiles = "$TempPath\\containerd-1.7.0"', + '$webClient.DownloadFile(\'https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-1.7.0-windows-amd64.tar.gz\', $containerdTarball)', + 'New-Item -Path "$containerdFiles" -ItemType Directory -Force | Out-Null', + 'tar.exe -xvzf "$containerdTarball" -C "$containerdFiles"', + '', + '# Move the containerd files into place', + 'Move-Item -Path "$containerdFiles\\bin\\containerd.exe" -Destination "$ContainerdPath\\containerd.exe" -Force', + 'Move-Item -Path "$containerdFiles\\bin\\containerd-shim-runhcs-v1.exe" -Destination "$ContainerdPath\\containerd-shim-runhcs-v1.exe" -Force', + 'Move-Item -Path "$containerdFiles\\bin\\ctr.exe" -Destination "$ContainerdPath\\ctr.exe" -Force', + '', + '# Clean up the containerd intermediate files', + 'Remove-Item -Path "$containerdFiles" -Recurse -Force', + 'Remove-Item -Path "$containerdTarball" -Force', + '', + '# -------', + '', + '# Patch the containerd setup script to configure a log file (rather than just discarding log output) and to use the upstream pause', + '# container image rather than the EKS version, since the latter appears to cause errors when attempting to create Windows Pods', + 'PatchFile -File "$TempPath\Add-ContainerdRuntime.ps1" -Patches @{', + ' "containerd --register-service" = "containerd --register-service --log-file \'C:\\ProgramData\\containerd\\root\\output.log\'";', + ' "amazonaws.com/eks/pause-windows:latest" = "registry.k8s.io/pause:3.9"', + '}', + '', + '# Add the full Windows Server 2022 base image and the pause image to the list of images to pre-pull', + '$baseLayersFile = "$TempPath\eks.baselayers.config"', + '$baseLayers = Get-Content -Path $baseLayersFile -Raw | ConvertFrom-Json', + '$baseLayers.2022 += "mcr.microsoft.com/windows/server:ltsc2022"', + '$baseLayers.2022 += "registry.k8s.io/pause:3.9"', + '$patchedJson = ConvertTo-Json -Depth 100 -InputObject $baseLayers', + 'Set-Content -Path $baseLayersFile -Value $patchedJson -NoNewline', + '', + ]) + + # Simplify the containerd installation command + step['inputs']['commands'] = [ + '', + '# Register containerd as the EKS container runtime', + 'Push-Location $TempPath', + '& .\Add-ContainerdRuntime.ps1 -Path "$ContainerdPath"', + 'Pop-Location' + ] + + elif name == 'ExecuteBuildScripts': + + # Prefix each script invocation with the call operator + step['loop']['forEach'] = [ + '& {}'.format(command) + for command in step['loop']['forEach'] + ] + + # Strip away the boilerplate code surrounding each script invocation + step['inputs']['commands'] = ['Push-Location $TempPath'] + step['loop']['forEach'] + ['Pop-Location'] + + # ------- + + # If we have a descriptive comment for the step then include it above its generated code + comment = Utility.commentForStep(name) + if comment != None: + generated += '\n{}\n'.format(comment) + + # ------- + + # Generate code for the step based on its action type + action = step['action'] + + if action == 'CreateFolder': + directories = [Utility.replaceConstants(d, constants) for d in step['loop']['forEach']] + generated += '\n'.join([ + 'foreach ($dir in @({})) {{'.format(', '.join(directories)), + '\tNew-Item -Path $dir -ItemType Directory -Force | Out-Null', + '}' + ]) + + elif action == 'DeleteFolder': + generated += '\n'.join([ + 'Remove-Item -Path "{}" -Recurse -Force'.format(Utility.replaceConstants(input['path'], constants)) + for input in step['inputs'] + ]) + + elif action == 'MoveFile': + generated += '\n'.join([ + 'Move-Item -Path "{}" -Destination "{}" -Force'.format( + Utility.replaceConstants(input['source'], constants), + Utility.replaceConstants(input['destination'], constants) + ) + for input in step['inputs'] + ]) + + elif action == 'S3Download': + generated += '\n'.join([ + '$webClient.DownloadFile("{}", "{}")'.format( + Utility.s3UriToHttpsUrl(input['source']), + Utility.replaceConstants(input['destination'], constants) + ) + for input in step['inputs'] + ]) + + elif action == 'ExecutePowerShell': + generated += '\n'.join([ + Utility.replaceConstants(c, constants).replace("'", '"') + for c in step['inputs']['commands'] + if not c.startswith('$ErrorActionPreference') + ]) + + elif action == 'Reboot': + Utility.log('Ignoring reboot step.') + continue + + else: + raise RuntimeError('Unknown build step action: {}'.format(action)) + + # ------- + + # Add a trailing newline after each non-ignored step + generated += '\n' + +# Write the generated code to the output script file +outfile = Path(__file__).parent / 'scripts' / 'setup.ps1' +Utility.writeFile(outfile, generated) +Utility.log('Wrote generated code to {}'.format(outfile)) diff --git a/cloud/aws/node/scripts/cleanup.ps1 b/cloud/aws/node/scripts/cleanup.ps1 new file mode 100644 index 0000000..c0a9a8b --- /dev/null +++ b/cloud/aws/node/scripts/cleanup.ps1 @@ -0,0 +1,9 @@ +# Perform cleanup +Set-Service -Name sshd -StartupType 'Manual' +Remove-Item -Path 'C:\ProgramData\ssh\administrators_authorized_keys' -Force + +# Remove the file for this script, since Packer won't have a chance to perform its own cleanup +Remove-Item -Path $PSCommandPath -Force + +# Perform sysprep and shut down the VM +& "$Env:ProgramFiles\Amazon\EC2Launch\EC2Launch.exe" sysprep --shutdown=true diff --git a/cloud/aws/node/scripts/setup.ps1 b/cloud/aws/node/scripts/setup.ps1 new file mode 100644 index 0000000..d156182 --- /dev/null +++ b/cloud/aws/node/scripts/setup.ps1 @@ -0,0 +1,130 @@ +<# + THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT! + + This script is based on the logic from the "Amazon EKS Optimized Windows AMI" + EC2 ImageBuilder component, with modifications to use containerd 1.7.0. + + The original ImageBuilder component logic is Copyright Amazon.com, Inc. or + its affiliates, and is licensed under the MIT License. +#> + +# Halt execution if we encounter an error +$ErrorActionPreference = 'Stop' + + +# Applies in-place patches to a file +function PatchFile +{ + Param ( + $File, + $Patches + ) + + $patched = Get-Content -Path $File -Raw + $Patches.GetEnumerator() | ForEach-Object { + $patched = $patched.Replace($_.Key, $_.Value) + } + Set-Content -Path $File -Value $patched -NoNewline +} + + +# Constants +$KubernetesPath = "$env:ProgramFiles\Kubernetes" +$KubernetesDownload = "https://amazon-eks.s3.amazonaws.com/1.24.7/2022-10-31/bin/windows/amd64" +$ContainerdPath = "$env:ProgramFiles\containerd" +$EKSPath = "$env:ProgramFiles\Amazon\EKS" +$CNIPath = "$EKSPath\cni" +$CSIProxyPath = "$EKSPath\bin" +$EKSLogsPath = "$env:ProgramData\Amazon\EKS\logs" +$TempRoot = "C:\TempEKSArtifactDir" +$TempPath = "$TempRoot\EKS-Artifacts" + +# Create each of our directories +foreach ($dir in @($ContainerdPath, $KubernetesPath, $EKSPath, $CNIPath, $CSIProxyPath, $EKSLogsPath, $TempRoot)) { + New-Item -Path $dir -ItemType Directory -Force | Out-Null +} + +# Install the NVIDIA GPU drivers +$driverBucket = 'ec2-windows-nvidia-drivers' +$driver = Get-S3Object -BucketName $driverBucket -KeyPrefix 'latest' -Region 'us-east-1' | Where-Object {$_.Key.Contains('server2022')} +Copy-S3Object -BucketName $driverBucket -Key $driver.Key -LocalFile "$TempRoot\driver.exe" -Region 'us-east-1' +Start-Process -FilePath "$TempRoot\driver.exe" -ArgumentList @('-s', '-noreboot') -NoNewWindow -Wait + +# Download the Kubernetes components +$webClient = New-Object System.Net.WebClient +$webClient.DownloadFile("$KubernetesDownload/kubelet.exe", "$KubernetesPath\kubelet.exe") +$webClient.DownloadFile("$KubernetesDownload/kube-proxy.exe", "$KubernetesPath\kube-proxy.exe") +$webClient.DownloadFile("$KubernetesDownload/aws-iam-authenticator.exe", "$EKSPath\aws-iam-authenticator.exe") + +# Download the EKS artifacts archive +$webClient.DownloadFile("https://ec2imagebuilder-managed-resources-us-east-1-prod.s3.amazonaws.com/components/eks-optimized-ami-windows/1.24.0/EKS-Artifacts.zip", "C:\EKS-Artifacts.zip") + +# Extract the EKS artifacts archive +Expand-Archive -Path "C:\EKS-Artifacts.zip" -DestinationPath $TempRoot +Remove-Item -Path "C:\EKS-Artifacts.zip" -Force + +# Move the EKS files into place +Move-Item -Path "$TempPath\ctr.exe" -Destination "$ContainerdPath\ctr.exe" -Force +Move-Item -Path "$TempPath\containerd.exe" -Destination "$ContainerdPath\containerd.exe" -Force +Move-Item -Path "$TempPath\containerd-shim-runhcs-v1.exe" -Destination "$ContainerdPath\containerd-shim-runhcs-v1.exe" -Force +Move-Item -Path "$TempPath\Start-EKSBootstrap.ps1" -Destination "$EKSPath\Start-EKSBootstrap.ps1" -Force +Move-Item -Path "$TempPath\EKS-StartupTask.ps1" -Destination "$EKSPath\EKS-StartupTask.ps1" -Force +Move-Item -Path "$TempPath\vpc-shared-eni.exe" -Destination "$CNIPath\vpc-shared-eni.exe" -Force +Move-Item -Path "$TempPath\csi-proxy.exe" -Destination "$CSIProxyPath\csi-proxy.exe" -Force + +# Install the Windows Containers feature +# (Note: this is actually a no-op here, since we install the feature beforehand in startup.ps1) +Install-WindowsFeature -Name Containers + +# ------- + +# TEMPORARY UNTIL EKS ADDS SUPPORT FOR CONTAINERD v1.7.0: +# Download and extract the containerd 1.7.0 release build +$containerdTarball = "$TempPath\containerd-1.7.0.tar.gz" +$containerdFiles = "$TempPath\containerd-1.7.0" +$webClient.DownloadFile('https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-1.7.0-windows-amd64.tar.gz', $containerdTarball) +New-Item -Path "$containerdFiles" -ItemType Directory -Force | Out-Null +tar.exe -xvzf "$containerdTarball" -C "$containerdFiles" + +# Move the containerd files into place +Move-Item -Path "$containerdFiles\bin\containerd.exe" -Destination "$ContainerdPath\containerd.exe" -Force +Move-Item -Path "$containerdFiles\bin\containerd-shim-runhcs-v1.exe" -Destination "$ContainerdPath\containerd-shim-runhcs-v1.exe" -Force +Move-Item -Path "$containerdFiles\bin\ctr.exe" -Destination "$ContainerdPath\ctr.exe" -Force + +# Clean up the containerd intermediate files +Remove-Item -Path "$containerdFiles" -Recurse -Force +Remove-Item -Path "$containerdTarball" -Force + +# ------- + +# Patch the containerd setup script to configure a log file (rather than just discarding log output) and to use the upstream pause +# container image rather than the EKS version, since the latter appears to cause errors when attempting to create Windows Pods +PatchFile -File "$TempPath\Add-ContainerdRuntime.ps1" -Patches @{ + "containerd --register-service" = "containerd --register-service --log-file 'C:\ProgramData\containerd\root\output.log'"; + "amazonaws.com/eks/pause-windows:latest" = "registry.k8s.io/pause:3.9" +} + +# Add the full Windows Server 2022 base image and the pause image to the list of images to pre-pull +$baseLayersFile = "$TempPath\eks.baselayers.config" +$baseLayers = Get-Content -Path $baseLayersFile -Raw | ConvertFrom-Json +$baseLayers.2022 += "mcr.microsoft.com/windows/server:ltsc2022" +$baseLayers.2022 += "registry.k8s.io/pause:3.9" +$patchedJson = ConvertTo-Json -Depth 100 -InputObject $baseLayers +Set-Content -Path $baseLayersFile -Value $patchedJson -NoNewline + +# Register containerd as the EKS container runtime +Push-Location $TempPath +& .\Add-ContainerdRuntime.ps1 -Path "$ContainerdPath" +Pop-Location + +# Perform EKS worker node setup +Push-Location $TempPath +& .\create-windows-pause-image.ps1 -ContainerRuntime containerd +& .\Get-EKSBaseLayers.ps1 -ConfigFile eks.baselayers.config -ContainerRuntime containerd +& .\Add-CSIProxy.ps1 -Path "$CSIProxyPath" -LogPath "$EKSLogsPath" +& .\EKS-WindowsServiceHost.ps1 +& .\Install-EKSWorkerNode.ps1 +Pop-Location + +# Perform cleanup +Remove-Item -Path "$TempRoot" -Recurse -Force diff --git a/cloud/aws/node/scripts/startup.ps1 b/cloud/aws/node/scripts/startup.ps1 new file mode 100644 index 0000000..a021e9a --- /dev/null +++ b/cloud/aws/node/scripts/startup.ps1 @@ -0,0 +1,26 @@ + + +# Install the OpenSSH server and set the sshd service to start automatically at system startup +Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 +Set-Service -Name sshd -StartupType 'Automatic' + +# Create the OpenSSH configuration directory if it doesn't already exist +$sshDir = 'C:\ProgramData\ssh' +if ((Test-Path -Path $sshDir) -eq $false) { + New-Item -Path $sshDir -ItemType Directory -Force | Out-Null +} + +# Retrieve the SHH public key from the EC2 metadata service +$authorisedKeys = "$sshDir\administrators_authorized_keys" +curl.exe 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' -o "$authorisedKeys" + +# Set the required ACLs for the authorised keys file +icacls.exe "$authorisedKeys" /inheritance:r /grant "Administrators:F" /grant "SYSTEM:F" + +# Install the Windows feature for containers, which will require a reboot +Install-WindowsFeature -Name Containers -IncludeAllSubFeature + +# Restart the VM +Restart-Computer + + \ No newline at end of file diff --git a/deployments/default-daemonsets.yml b/deployments/default-daemonsets.yml new file mode 100644 index 0000000..5234788 --- /dev/null +++ b/deployments/default-daemonsets.yml @@ -0,0 +1,57 @@ +# HostProcess DaemonSets for the MCDM device plugin and the WDDM device plugin, using default settings + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: device-plugin-mcdm +spec: + selector: + matchLabels: + app: device-plugin-mcdm + template: + metadata: + labels: + app: device-plugin-mcdm + spec: + nodeSelector: + kubernetes.io/os: 'windows' + kubernetes.io/arch: 'amd64' + node.kubernetes.io/windows-build: '10.0.20348' + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: device-plugin-mcdm + image: "index.docker.io/tensorworks/mcdm-device-plugin:0.0.1" + imagePullPolicy: Always + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: device-plugin-wddm +spec: + selector: + matchLabels: + app: device-plugin-wddm + template: + metadata: + labels: + app: device-plugin-wddm + spec: + nodeSelector: + kubernetes.io/os: 'windows' + kubernetes.io/arch: 'amd64' + node.kubernetes.io/windows-build: '10.0.20348' + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: device-plugin-wddm + image: "index.docker.io/tensorworks/wddm-device-plugin:0.0.1" + imagePullPolicy: Always diff --git a/deployments/multitenancy-configmap.yml b/deployments/multitenancy-configmap.yml new file mode 100644 index 0000000..d3db645 --- /dev/null +++ b/deployments/multitenancy-configmap.yml @@ -0,0 +1,87 @@ +# Example HostProcess DaemonSets for the MCDM device plugin and the WDDM device plugin, using settings that enable multitenancy +# +# This version of the DaemonSets uses a ConfigMap to provide configuration values. For a version that sets environment variable +# values directly in the Pod spec, see the file `multitenancy-inline.yml` + +apiVersion: v1 +kind: ConfigMap +metadata: + name: device-plugin-config +data: + + # Configure the device plugins to allow 4 containers to mount each device simultaneously + multitenancy: '4' + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: device-plugin-mcdm +spec: + selector: + matchLabels: + app: device-plugin-mcdm + template: + metadata: + labels: + app: device-plugin-mcdm + spec: + nodeSelector: + kubernetes.io/os: 'windows' + kubernetes.io/arch: 'amd64' + node.kubernetes.io/windows-build: '10.0.20348' + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: device-plugin-mcdm + image: "index.docker.io/tensorworks/mcdm-device-plugin:0.0.1" + imagePullPolicy: Always + + # Use the configuration values from the ConfigMap + env: + - name: MCDM_DEVICE_PLUGIN_MULTITENANCY + valueFrom: + configMapKeyRef: + name: device-plugin-config + key: multitenancy + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: device-plugin-wddm +spec: + selector: + matchLabels: + app: device-plugin-wddm + template: + metadata: + labels: + app: device-plugin-wddm + spec: + nodeSelector: + kubernetes.io/os: 'windows' + kubernetes.io/arch: 'amd64' + node.kubernetes.io/windows-build: '10.0.20348' + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: device-plugin-wddm + image: "index.docker.io/tensorworks/wddm-device-plugin:0.0.1" + imagePullPolicy: Always + + # Use the configuration values from the ConfigMap + env: + - name: WDDM_DEVICE_PLUGIN_MULTITENANCY + valueFrom: + configMapKeyRef: + name: device-plugin-config + key: multitenancy diff --git a/deployments/multitenancy-inline.yml b/deployments/multitenancy-inline.yml new file mode 100644 index 0000000..92cf75d --- /dev/null +++ b/deployments/multitenancy-inline.yml @@ -0,0 +1,70 @@ +# Example HostProcess DaemonSets for the MCDM device plugin and the WDDM device plugin, using settings that enable multitenancy +# +# This version of the DaemonSets sets environment variable values directly in the Pod spec. For a version that uses a ConfigMap +# to provide configuration values, see the file `multitenancy-configmap.yml` + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: device-plugin-mcdm +spec: + selector: + matchLabels: + app: device-plugin-mcdm + template: + metadata: + labels: + app: device-plugin-mcdm + spec: + nodeSelector: + kubernetes.io/os: 'windows' + kubernetes.io/arch: 'amd64' + node.kubernetes.io/windows-build: '10.0.20348' + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: device-plugin-mcdm + image: "index.docker.io/tensorworks/mcdm-device-plugin:0.0.1" + imagePullPolicy: Always + + # Configure the MCDM device plugin to allow 4 containers to mount each compute-only device simultaneously + env: + - name: MCDM_DEVICE_PLUGIN_MULTITENANCY + value: "4" + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: device-plugin-wddm +spec: + selector: + matchLabels: + app: device-plugin-wddm + template: + metadata: + labels: + app: device-plugin-wddm + spec: + nodeSelector: + kubernetes.io/os: 'windows' + kubernetes.io/arch: 'amd64' + node.kubernetes.io/windows-build: '10.0.20348' + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: device-plugin-wddm + image: "index.docker.io/tensorworks/wddm-device-plugin:0.0.1" + imagePullPolicy: Always + + # Configure the WDDM device plugin to allow 4 containers to mount each display device simultaneously + env: + - name: WDDM_DEVICE_PLUGIN_MULTITENANCY + value: "4" diff --git a/examples/cuda-devicequery/cuda-devicequery-mcdm.yml b/examples/cuda-devicequery/cuda-devicequery-mcdm.yml new file mode 100644 index 0000000..a03140d --- /dev/null +++ b/examples/cuda-devicequery/cuda-devicequery-mcdm.yml @@ -0,0 +1,25 @@ +# Example Job for running the CUDA deviceQuery sample program inside a container +# +# This version of the Job requests a compute-only device from the MCDM device plugin. For a version that +# requests a display device from the WDDM device plugin, see the file `cuda-devicequery-wddm.yml` +# +# NOTE: this Job will only work when the device allocated by the MCDM device plugin is an NVIDIA GPU, +# otherwise the DLL files required by `deviceQuery.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-cuda-devicequery-mcdm +spec: + template: + spec: + containers: + - name: example-cuda-devicequery-mcdm + image: "index.docker.io/tensorworks/example-cuda-devicequery:0.0.1" + resources: + limits: + directx.microsoft.com/compute: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/cuda-devicequery/cuda-devicequery-wddm.yml b/examples/cuda-devicequery/cuda-devicequery-wddm.yml new file mode 100644 index 0000000..8f1dc3b --- /dev/null +++ b/examples/cuda-devicequery/cuda-devicequery-wddm.yml @@ -0,0 +1,25 @@ +# Example Job for running the CUDA deviceQuery sample program inside a container +# +# This version of the Job requests a display device from the WDDM device plugin. For a version that +# requests a compute-only device from the MCDM device plugin, see the file `cuda-devicequery-mcdm.yml` +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an NVIDIA GPU, +# otherwise the DLL files required by `deviceQuery.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-cuda-devicequery-wddm +spec: + template: + spec: + containers: + - name: example-cuda-devicequery-wddm + image: "index.docker.io/tensorworks/example-cuda-devicequery:0.0.1" + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/cuda-montecarlo/cuda-montecarlo-mcdm.yml b/examples/cuda-montecarlo/cuda-montecarlo-mcdm.yml new file mode 100644 index 0000000..2d43646 --- /dev/null +++ b/examples/cuda-montecarlo/cuda-montecarlo-mcdm.yml @@ -0,0 +1,25 @@ +# Example Job for running the CUDA MC_EstimatePiP sample program inside a container +# +# This version of the Job requests a compute-only device from the MCDM device plugin. For a version that +# requests a display device from the WDDM device plugin, see the file `cuda-montecarlo-wddm.yml` +# +# NOTE: this Job will only work when the device allocated by the MCDM device plugin is an NVIDIA GPU, +# otherwise the DLL files required by `MC_EstimatePiP.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-cuda-montecarlo-mcdm +spec: + template: + spec: + containers: + - name: example-cuda-montecarlo-mcdm + image: "index.docker.io/tensorworks/example-cuda-montecarlo:0.0.1" + resources: + limits: + directx.microsoft.com/compute: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/cuda-montecarlo/cuda-montecarlo-wddm.yml b/examples/cuda-montecarlo/cuda-montecarlo-wddm.yml new file mode 100644 index 0000000..cdfdadd --- /dev/null +++ b/examples/cuda-montecarlo/cuda-montecarlo-wddm.yml @@ -0,0 +1,25 @@ +# Example Job for running the CUDA MC_EstimatePiP sample program inside a container +# +# This version of the Job requests a display device from the WDDM device plugin. For a version that +# requests a compute-only device from the MCDM device plugin, see the file `cuda-montecarlo-mcdm.yml` +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an NVIDIA GPU, +# otherwise the DLL files required by `MC_EstimatePiP.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-cuda-montecarlo-wddm +spec: + template: + spec: + containers: + - name: example-cuda-montecarlo-wddm + image: "index.docker.io/tensorworks/example-cuda-montecarlo:0.0.1" + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/device-discovery/device-discovery-mcdm.yml b/examples/device-discovery/device-discovery-mcdm.yml new file mode 100644 index 0000000..a2fcfc9 --- /dev/null +++ b/examples/device-discovery/device-discovery-mcdm.yml @@ -0,0 +1,22 @@ +# Example Job for running the device discovery test program inside a container +# +# This version of the Job requests a compute-only device from the MCDM device plugin. For a version that +# requests a display device from the WDDM device plugin, see the file `device-discovery-wddm.yml` + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-device-discovery-mcdm +spec: + template: + spec: + containers: + - name: example-device-discovery-mcdm + image: "index.docker.io/tensorworks/example-device-discovery:0.0.1" + resources: + limits: + directx.microsoft.com/compute: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/device-discovery/device-discovery-wddm.yml b/examples/device-discovery/device-discovery-wddm.yml new file mode 100644 index 0000000..6afd38c --- /dev/null +++ b/examples/device-discovery/device-discovery-wddm.yml @@ -0,0 +1,22 @@ +# Example Job for running the device discovery test program inside a container +# +# This version of the Job requests a display device from the WDDM device plugin. For a version that +# requests a compute-only device from the MCDM device plugin, see the file `device-discovery-mcdm.yml` + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-device-discovery-wddm +spec: + template: + spec: + containers: + - name: example-device-discovery-wddm + image: "index.docker.io/tensorworks/example-device-discovery:0.0.1" + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/directml/directml-mcdm.yml b/examples/directml/directml-mcdm.yml new file mode 100644 index 0000000..a77226b --- /dev/null +++ b/examples/directml/directml-mcdm.yml @@ -0,0 +1,22 @@ +# Example Job for running a DirectML sample inside a container +# +# This version of the Job requests a compute-only device from the MCDM device plugin. For a version that +# requests a display device from the WDDM device plugin, see the file `directml-wddm.yml` + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-directml-mcdm +spec: + template: + spec: + containers: + - name: example-directml-mcdm + image: "index.docker.io/tensorworks/example-directml:0.0.1" + resources: + limits: + directx.microsoft.com/compute: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/directml/directml-wddm.yml b/examples/directml/directml-wddm.yml new file mode 100644 index 0000000..857aff7 --- /dev/null +++ b/examples/directml/directml-wddm.yml @@ -0,0 +1,22 @@ +# Example Job for running a DirectML sample inside a container +# +# This version of the Job requests a display device from the WDDM device plugin. For a version that +# requests a compute-only device from the MCDM device plugin, see the file `directml-mcdm.yml` + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-directml-wddm +spec: + template: + spec: + containers: + - name: example-directml-wddm + image: "index.docker.io/tensorworks/example-directml:0.0.1" + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/ffmpeg-amf/ffmpeg-amf.yml b/examples/ffmpeg-amf/ffmpeg-amf.yml new file mode 100644 index 0000000..acd4014 --- /dev/null +++ b/examples/ffmpeg-amf/ffmpeg-amf.yml @@ -0,0 +1,23 @@ +# Example Job for running an AMD AMF transcode operation with FFmpeg inside a container +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an AMD GPU, +# otherwise the DLL files for AMF won't exist and FFmpeg will fail when it tries to load them. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-ffmpeg-amf +spec: + template: + spec: + containers: + - name: example-ffmpeg-amf + image: "index.docker.io/tensorworks/example-ffmpeg:0.0.1" + args: ["-i", "C:\\sample-video.mp4", "-c:v", "h264_amf", "-preset", "default", "C:\\output.mp4"] + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/ffmpeg-autodetect/autodetect-encoder.ps1 b/examples/ffmpeg-autodetect/autodetect-encoder.ps1 new file mode 100644 index 0000000..ab68c53 --- /dev/null +++ b/examples/ffmpeg-autodetect/autodetect-encoder.ps1 @@ -0,0 +1,26 @@ +# Attempt to detect the availability of a hardware video encoder +$encoder = '' +if ((Get-ChildItem "C:\Windows\System32\amfrt64.dll" -ErrorAction SilentlyContinue)) +{ + Write-Host 'Detected an AMD GPU, using the AMF video encoder' + $encoder = 'h264_amf' +} +elseif ((Get-ChildItem "C:\Windows\System32\intel_gfx_api-x64.dll" -ErrorAction SilentlyContinue)) +{ + Write-Host 'Detected an Intel GPU, using the Quick Sync video encoder' + $encoder = 'h264_qsv' +} +elseif ((Get-ChildItem "C:\Windows\System32\nvEncodeAPI64.dll" -ErrorAction SilentlyContinue)) +{ + Write-Host 'Detected an NVIDIA GPU, using the NVENC video encoder' + $encoder = 'h264_nvenc' +} +else { + throw "Failed to detect the availability of a supported hardware video encoder" +} + +# Invoke FFmpeg with the detected hardware video encoder +& C:\ffmpeg.exe -i C:\sample-video.mp4 -c:v "$encoder" -preset default C:\output.mp4 +if ($LastExitCode -ne 0) { + throw "FFmpeg terminated with exit code $LastExitCode" +} diff --git a/examples/ffmpeg-autodetect/ffmpeg-autodetect.yml b/examples/ffmpeg-autodetect/ffmpeg-autodetect.yml new file mode 100644 index 0000000..81b8957 --- /dev/null +++ b/examples/ffmpeg-autodetect/ffmpeg-autodetect.yml @@ -0,0 +1,32 @@ +# Example Job for running a hardware accelerated transcode operation with FFmpeg inside a container +# +# The transcode script will attempt to detect the availability of the following encoders: +# +# - AMD AMF +# - Intel Quick Sync +# - NVIDIA NVENC +# +# If a hardware encoder is detected then it will be used, otherwise the script will fail. +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an AMD, Intel or NVIDIA GPU, +# otherwise the DLL files for the hardware encoders won't exist and the script will fail when no encoder is detected. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-ffmpeg-autodetect +spec: + template: + spec: + containers: + - name: example-ffmpeg-autodetect + image: "index.docker.io/tensorworks/example-ffmpeg:0.0.1" + command: ["powershell"] + args: ["-ExecutionPolicy", "Bypass", "-File", "C:\\autodetect-encoder.ps1"] + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/ffmpeg-nvenc/ffmpeg-nvenc.yml b/examples/ffmpeg-nvenc/ffmpeg-nvenc.yml new file mode 100644 index 0000000..5dcecf8 --- /dev/null +++ b/examples/ffmpeg-nvenc/ffmpeg-nvenc.yml @@ -0,0 +1,23 @@ +# Example Job for running an NVIDIA NVENC transcode operation with FFmpeg inside a container +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an NVIDIA GPU, +# otherwise the DLL files for CUDA and NVENC won't exist and FFmpeg will fail when it tries to load them. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-ffmpeg-nvenc +spec: + template: + spec: + containers: + - name: example-ffmpeg-nvenc + image: "index.docker.io/tensorworks/example-ffmpeg:0.0.1" + args: ["-i", "C:\\sample-video.mp4", "-c:v", "h264_nvenc", "-preset", "default", "C:\\output.mp4"] + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/ffmpeg-quicksync/ffmpeg-quicksync.yml b/examples/ffmpeg-quicksync/ffmpeg-quicksync.yml new file mode 100644 index 0000000..f45d4a7 --- /dev/null +++ b/examples/ffmpeg-quicksync/ffmpeg-quicksync.yml @@ -0,0 +1,23 @@ +# Example Job for running an Intel Quick Sync transcode operation with FFmpeg inside a container +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an Intel GPU, +# otherwise the DLL files for Quick Sync won't exist and FFmpeg will fail when it tries to load them. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-ffmpeg-quicksync +spec: + template: + spec: + containers: + - name: example-ffmpeg-quicksync + image: "index.docker.io/tensorworks/example-ffmpeg:0.0.1" + args: ["-i", "C:\\sample-video.mp4", "-c:v", "h264_qsv", "-preset", "default", "C:\\output.mp4"] + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/nvidia-smi/nvidia-smi-mcdm.yml b/examples/nvidia-smi/nvidia-smi-mcdm.yml new file mode 100644 index 0000000..b2a1e2c --- /dev/null +++ b/examples/nvidia-smi/nvidia-smi-mcdm.yml @@ -0,0 +1,26 @@ +# Example Job for running the NVIDIA SMI tool inside a container +# +# This version of the Job requests a compute-only device from the MCDM device plugin. For a version that +# requests a display device from the WDDM device plugin, see the file `nvidia-smi-wddm.yml` +# +# NOTE: this Job will only work when the device allocated by the MCDM device plugin is an NVIDIA GPU, +# otherwise the executable `nvidia-smi.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-nvidia-smi +spec: + template: + spec: + containers: + - name: example-nvidia-smi + image: "mcr.microsoft.com/windows/servercore:ltsc2022" + command: ["nvidia-smi.exe"] + resources: + limits: + directx.microsoft.com/compute: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/nvidia-smi/nvidia-smi-wddm.yml b/examples/nvidia-smi/nvidia-smi-wddm.yml new file mode 100644 index 0000000..81b467c --- /dev/null +++ b/examples/nvidia-smi/nvidia-smi-wddm.yml @@ -0,0 +1,26 @@ +# Example Job for running the NVIDIA SMI tool inside a container +# +# This version of the Job requests a display device from the WDDM device plugin. For a version that +# requests a compute-only device from the MCDM device plugin, see the file `nvidia-smi-mcdm.yml` +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is an NVIDIA GPU, +# otherwise the executable `nvidia-smi.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-nvidia-smi +spec: + template: + spec: + containers: + - name: example-nvidia-smi + image: "mcr.microsoft.com/windows/servercore:ltsc2022" + command: ["nvidia-smi.exe"] + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/opencl-enum/opencl-enum-mcdm.yml b/examples/opencl-enum/opencl-enum-mcdm.yml new file mode 100644 index 0000000..5a7ff9b --- /dev/null +++ b/examples/opencl-enum/opencl-enum-mcdm.yml @@ -0,0 +1,25 @@ +# Example Job for running the OpenCL enumopencl sample program inside a container +# +# This version of the Job requests a compute-only device from the MCDM device plugin. For a version that +# requests a display device from the WDDM device plugin, see the file `opencl-enum-wddm.yml` +# +# NOTE: this Job will only work when the device allocated by the MCDM device plugin is is a GPU that supports +# OpenCL, otherwise the DLL files required by `enumopencl.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-opencl-enum-mcdm +spec: + template: + spec: + containers: + - name: example-opencl-enum-mcdm + image: "index.docker.io/tensorworks/example-opencl-enum:0.0.1" + resources: + limits: + directx.microsoft.com/compute: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/opencl-enum/opencl-enum-wddm.yml b/examples/opencl-enum/opencl-enum-wddm.yml new file mode 100644 index 0000000..f6821d4 --- /dev/null +++ b/examples/opencl-enum/opencl-enum-wddm.yml @@ -0,0 +1,25 @@ +# Example Job for running the OpenCL enumopencl sample program inside a container +# +# This version of the Job requests a display device from the WDDM device plugin. For a version that +# requests a compute-only device from the MCDM device plugin, see the file `opencl-enum-mcdm.yml` +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is is a GPU that supports +# OpenCL, otherwise the DLL files required by `enumopencl.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-opencl-enum-wddm +spec: + template: + spec: + containers: + - name: example-opencl-enum-wddm + image: "index.docker.io/tensorworks/example-opencl-enum:0.0.1" + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/examples/vulkaninfo/vulkaninfo.yml b/examples/vulkaninfo/vulkaninfo.yml new file mode 100644 index 0000000..3ac5622 --- /dev/null +++ b/examples/vulkaninfo/vulkaninfo.yml @@ -0,0 +1,23 @@ +# Example Job for running the Vulkan information tool inside a container +# +# NOTE: this Job will only work when the device allocated by the WDDM device plugin is a GPU that supports +# Vulkan, otherwise the executable `vulkaninfo.exe` won't exist and the Pod will fail to start. + +apiVersion: batch/v1 +kind: Job +metadata: + name: example-vulkaninfo +spec: + template: + spec: + containers: + - name: example-vulkaninfo + image: "mcr.microsoft.com/windows/server:ltsc2022" + command: ["vulkaninfo.exe"] + resources: + limits: + directx.microsoft.com/display: 1 + nodeSelector: + "kubernetes.io/os": windows + restartPolicy: Never + backoffLimit: 0 diff --git a/external/.gitignore b/external/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/external/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt new file mode 100644 index 0000000..e5c9dd9 --- /dev/null +++ b/library/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.22) +project(directx-device-discovery) + +# Set the C++ standard to C++17 +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# Locate our dependencies (these will be provided by vcpkg) +find_package(cppwinrt CONFIG REQUIRED) +find_package(fmt CONFIG REQUIRED) +find_package(spdlog CONFIG REQUIRED) +find_package(wil CONFIG REQUIRED) + +# Build our shared library +add_library(directx-device-discovery SHARED + src/AdapterEnumeration.cpp + src/D3DHelpers.cpp + src/DeviceDiscovery.cpp + src/DeviceDiscoveryImp.cpp + src/DllMain.cpp + src/ErrorHandling.cpp + src/RegistryQuery.cpp + src/SafeArray.cpp + src/WmiQuery.cpp +) +target_link_libraries(directx-device-discovery PRIVATE + dxcore.lib + dxguid.lib + fmt::fmt-header-only + gdi32.lib + Microsoft::CppWinRT + spdlog::spdlog_header_only + wbemuuid.lib + WIL::WIL + WindowsApp.lib +) +set_property(TARGET directx-device-discovery PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded") +target_include_directories(directx-device-discovery PUBLIC include) +target_precompile_headers(directx-device-discovery PRIVATE src/pch.h) + +# Build our test executable +add_executable(test-device-discovery-cpp test/test-device-discovery-cpp.cpp) +set_property(TARGET test-device-discovery-cpp PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded") +target_link_libraries(test-device-discovery-cpp PRIVATE Microsoft::CppWinRT directx-device-discovery) + +# Install the shared library and the test executable to the top-level bin directory +install( + TARGETS directx-device-discovery test-device-discovery-cpp + RUNTIME DESTINATION bin +) diff --git a/library/include/DeviceDiscovery.h b/library/include/DeviceDiscovery.h new file mode 100644 index 0000000..eac73db --- /dev/null +++ b/library/include/DeviceDiscovery.h @@ -0,0 +1,269 @@ +#pragma once +#include "DeviceFilter.h" + +#define DLLEXPORT __declspec(dllexport) + +#ifdef __cplusplus +extern "C" { +#endif + +// Opaque pointer type for DeviceDiscovery instances +typedef void* DeviceDiscoveryInstance; + +// Returns the version string for the device discovery library +DLLEXPORT const wchar_t* GetDiscoveryLibraryVersion(); + +// Disables verbose logging for the device discovery library (this is the default) +DLLEXPORT void DisableDiscoveryLogging(); + +// Enables verbose logging for the device discovery library +DLLEXPORT void EnableDiscoveryLogging(); + +// Creates a new DeviceDiscovery instance +DLLEXPORT DeviceDiscoveryInstance CreateDeviceDiscoveryInstance(); + +// Frees the memory for a DeviceDiscovery instance +DLLEXPORT void DestroyDeviceDiscoveryInstance(DeviceDiscoveryInstance instance); + +// Retrieves the error message for the last operation performed by the DeviceDiscovery instance. +// If the last operation succeeded then an empty string will be returned. +DLLEXPORT const wchar_t* DeviceDiscovery_GetLastErrorMessage(DeviceDiscoveryInstance instance); + +// Determines whether the current device list is stale and needs to be refreshed by performing device discovery again +DLLEXPORT int DeviceDiscovery_IsRefreshRequired(DeviceDiscoveryInstance instance); + +// Performs device discovery. Returns 0 on success and -1 on failure. +// Call GetLastErrorMessage to retrieve the error details for a failure. +DLLEXPORT int DeviceDiscovery_DiscoverDevices(DeviceDiscoveryInstance instance, int filter, int includeIntegrated, int includeDetachable); + +// Returns the number of devices found by the last device discovery, or -1 if device discovery has not been performed +DLLEXPORT int DeviceDiscovery_GetNumDevices(DeviceDiscoveryInstance instance); + +DLLEXPORT long long DeviceDiscovery_GetDeviceAdapterLUID(DeviceDiscoveryInstance instance, unsigned int device); + +// Returns the unique ID of the device with the specified index, or a NULL pointer if the specified device index is invalid +DLLEXPORT const wchar_t* DeviceDiscovery_GetDeviceID(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetDeviceDescription(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetDeviceDriverRegistryKey(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetDeviceDriverStorePath(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetDeviceLocationPath(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetDeviceVendor(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT int DeviceDiscovery_GetNumRuntimeFiles(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetRuntimeFileSource(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetRuntimeFileDestination(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file); + +DLLEXPORT int DeviceDiscovery_GetNumRuntimeFilesWow64(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetRuntimeFileSourceWow64(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file); + +DLLEXPORT const wchar_t* DeviceDiscovery_GetRuntimeFileDestinationWow64(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file); + +DLLEXPORT int DeviceDiscovery_IsDeviceIntegrated(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT int DeviceDiscovery_IsDeviceDetachable(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT int DeviceDiscovery_DoesDeviceSupportDisplay(DeviceDiscoveryInstance instance, unsigned int device); + +DLLEXPORT int DeviceDiscovery_DoesDeviceSupportCompute(DeviceDiscoveryInstance instance, unsigned int device); + +#ifdef __cplusplus +} // extern "C" + + +#include +#include + +// API wrapper classes for C++ clients + +class DeviceDiscoveryException +{ + public: + DeviceDiscoveryException(const wchar_t* message) { + this->message = message; + } + + DeviceDiscoveryException(const DeviceDiscoveryException& other) = default; + DeviceDiscoveryException(DeviceDiscoveryException&& other) = default; + DeviceDiscoveryException& operator=(const DeviceDiscoveryException& other) = default; + DeviceDiscoveryException& operator=(DeviceDiscoveryException&& other) = default; + + std::wstring what() const { + return this->message; + } + + private: + std::wstring message; +}; + +class DeviceDiscovery +{ + private: + DeviceDiscoveryInstance instance; + + public: + + inline DeviceDiscovery() { + this->instance = CreateDeviceDiscoveryInstance(); + } + + inline ~DeviceDiscovery() + { + DestroyDeviceDiscoveryInstance(this->instance); + this->instance = nullptr; + } + + inline const wchar_t* GetLastErrorMessage() { + return DeviceDiscovery_GetLastErrorMessage(this->instance); + } + + inline bool IsRefreshRequired() { + return DeviceDiscovery_IsRefreshRequired(this->instance); + } + + #define THROW_IF_ERROR(sentinel) if (result == sentinel) { throw DeviceDiscoveryException(DeviceDiscovery_GetLastErrorMessage(this->instance)); } + + inline bool DiscoverDevices(DeviceFilter filter, bool includeIntegrated, bool includeDetachable) + { + int result = DeviceDiscovery_DiscoverDevices(this->instance, static_cast(filter), includeIntegrated, includeDetachable); + THROW_IF_ERROR(-1); + return (result == 0); + } + + inline int GetNumDevices() + { + int result = DeviceDiscovery_GetNumDevices(this->instance); + THROW_IF_ERROR(-1); + return result; + } + + inline long long GetDeviceAdapterLUID(unsigned int device) + { + long long result = DeviceDiscovery_GetDeviceAdapterLUID(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + inline const wchar_t* GetDeviceID(unsigned int device) + { + const wchar_t* result = DeviceDiscovery_GetDeviceID(this->instance, device); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetDeviceDescription(unsigned int device) + { + const wchar_t* result = DeviceDiscovery_GetDeviceDescription(this->instance, device); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetDeviceDriverRegistryKey(unsigned int device) + { + const wchar_t* result = DeviceDiscovery_GetDeviceDriverRegistryKey(this->instance, device); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetDeviceDriverStorePath(unsigned int device) + { + const wchar_t* result = DeviceDiscovery_GetDeviceDriverStorePath(this->instance, device); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetDeviceLocationPath(unsigned int device) + { + const wchar_t* result = DeviceDiscovery_GetDeviceLocationPath(this->instance, device); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetDeviceVendor(unsigned int device) + { + const wchar_t* result = DeviceDiscovery_GetDeviceVendor(this->instance, device); + THROW_IF_ERROR(nullptr); + return result; + } + + inline int GetNumRuntimeFiles(unsigned int device) + { + int result = DeviceDiscovery_GetNumRuntimeFiles(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + inline const wchar_t* GetRuntimeFileSource(unsigned int device, unsigned int file) + { + const wchar_t* result = DeviceDiscovery_GetRuntimeFileSource(this->instance, device, file); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetRuntimeFileDestination(unsigned int device, unsigned int file) + { + const wchar_t* result = DeviceDiscovery_GetRuntimeFileDestination(this->instance, device, file); + THROW_IF_ERROR(nullptr); + return result; + } + + inline int GetNumRuntimeFilesWow64(unsigned int device) + { + int result = DeviceDiscovery_GetNumRuntimeFilesWow64(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + inline const wchar_t* GetRuntimeFileSourceWow64(unsigned int device, unsigned int file) + { + const wchar_t* result = DeviceDiscovery_GetRuntimeFileSourceWow64(this->instance, device, file); + THROW_IF_ERROR(nullptr); + return result; + } + + inline const wchar_t* GetRuntimeFileDestinationWow64(unsigned int device, unsigned int file) + { + const wchar_t* result = DeviceDiscovery_GetRuntimeFileDestinationWow64(this->instance, device, file); + THROW_IF_ERROR(nullptr); + return result; + } + + inline bool IsDeviceIntegrated(unsigned int device) + { + int result = DeviceDiscovery_IsDeviceIntegrated(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + inline bool IsDeviceDetachable(unsigned int device) + { + int result = DeviceDiscovery_IsDeviceDetachable(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + inline bool DoesDeviceSupportDisplay(unsigned int device) + { + int result = DeviceDiscovery_DoesDeviceSupportDisplay(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + inline bool DoesDeviceSupportCompute(unsigned int device) + { + int result = DeviceDiscovery_DoesDeviceSupportCompute(this->instance, device); + THROW_IF_ERROR(-1); + return result; + } + + #undef THROW_IF_ERROR +}; + +#endif diff --git a/library/include/DeviceFilter.h b/library/include/DeviceFilter.h new file mode 100644 index 0000000..bf12d05 --- /dev/null +++ b/library/include/DeviceFilter.h @@ -0,0 +1,65 @@ +#pragma once + +// Enumerate all devices +#define DEVICEFILTER_ALL 0 + +// Enumerate devices that support display, irrespective of whether they also support compute +#define DEVICEFILTER_DISPLAY_SUPPORTED 1 + +// Enumerate devices that support compute, irrespective of whether they also support display +#define DEVICEFILTER_COMPUTE_SUPPORTED 2 + +// Enumerate devices that support display and do not support compute (e.g. legacy DirectX 11 devices) +#define DEVICEFILTER_DISPLAY_ONLY 3 + +// Enumerate devices that support compute and do not support display (i.e. compute-only DirectX 12 devices) +#define DEVICEFILTER_COMPUTE_ONLY 4 + +// Enumerate devices that support both display and compute (i.e. fully-featured DirectX 12 devices) +#define DEVICEFILTER_DISPLAY_AND_COMPUTE 5 + + +#ifdef __cplusplus + +#include + +// Device filter enum for C++ clients +enum class DeviceFilter : int +{ + AllDevices = DEVICEFILTER_ALL, + DisplaySupported = DEVICEFILTER_DISPLAY_SUPPORTED, + ComputeSupported = DEVICEFILTER_COMPUTE_SUPPORTED, + DisplayOnly = DEVICEFILTER_DISPLAY_ONLY, + ComputeOnly = DEVICEFILTER_COMPUTE_ONLY, + DisplayAndCompute = DEVICEFILTER_DISPLAY_AND_COMPUTE +}; + +// Returns a string representation of a device filter +inline std::wstring DeviceFilterName(DeviceFilter filter) +{ + switch (filter) + { + case DeviceFilter::AllDevices: + return L"AllDevices"; + + case DeviceFilter::DisplaySupported: + return L"DisplaySupported"; + + case DeviceFilter::ComputeSupported: + return L"ComputeSupported"; + + case DeviceFilter::DisplayOnly: + return L"DisplayOnly"; + + case DeviceFilter::ComputeOnly: + return L"ComputeOnly"; + + case DeviceFilter::DisplayAndCompute: + return L"DisplayAndCompute"; + + default: + return L""; + } +} + +#endif diff --git a/library/src/Adapter.h b/library/src/Adapter.h new file mode 100644 index 0000000..8803373 --- /dev/null +++ b/library/src/Adapter.h @@ -0,0 +1,37 @@ +#pragma once + +// Represents a DirectX adapter as enumerated by DXCore +// (For additional details, see: ) +struct Adapter +{ + inline Adapter() : + InstanceLuid(0), + IsHardware(false), + IsIntegrated(false), + IsDetachable(false), + SupportsDisplay(false), + SupportsCompute(false) + {} + + // The locally unique identifier (LUID) for the adapter + int64_t InstanceLuid; + + // The PnP hardware ID information for the adapter + DXCoreHardwareID HardwareID; + + // Specifies whether the adapter is a hardware device (as opposed to a software device) + bool IsHardware; + + // Specifies whether the adapter is an integrated GPU (as opposed to a discrete GPU) + bool IsIntegrated; + + // Specifies whether the adapter is a detachable device (i.e. the device can be removed at runtime) + bool IsDetachable; + + // Specifies whether the adapter supports display + // (i.e. supports either the DXCORE_ADAPTER_ATTRIBUTE_D3D11_GRAPHICS or DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS attributes) + bool SupportsDisplay; + + // Specifies whether the adapter supports compute (i.e. supports the DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE attribute) + bool SupportsCompute; +}; diff --git a/library/src/AdapterEnumeration.cpp b/library/src/AdapterEnumeration.cpp new file mode 100644 index 0000000..48be429 --- /dev/null +++ b/library/src/AdapterEnumeration.cpp @@ -0,0 +1,174 @@ +#include "AdapterEnumeration.h" +#include "ErrorHandling.h" +#include "ObjectHelpers.h" + +#include + +AdapterEnumeration::AdapterEnumeration() +{ + // Create our DXCore adapter factory + auto error = CheckHresult(DXCoreCreateAdapterFactory(this->adapterFactory.put())); + if (error) { + throw error.Wrap(L"DXCoreCreateAdapterFactory failed"); + } +} + +void AdapterEnumeration::EnumerateAdapters(const DeviceFilter& filter, bool includeIntegrated, bool includeDetachable) +{ + // Log our enumeration parameters + LOG( + L"Enumerating DirectX adapters using parameters: {{ filter:{}, includeIntegrated:{}, includeDetachable:{} }}", + DeviceFilterName(filter), + includeIntegrated, + includeDetachable + ); + + // Clear our adapter lists and our set of unique adapters + this->adapterLists.clear(); + this->uniqueAdapters.clear(); + + #define ENUMERATE_ADAPTERS(attribute)\ + {\ + GUID attributes[]{ attribute };\ + this->adapterLists.push_back(nullptr); \ + auto error = CheckHresult(this->adapterFactory->CreateAdapterList(_countof(attributes), attributes, this->adapterLists.back().put()));\ + if (error) { \ + throw error.Wrap(L"IDXCoreAdapterFactory::CreateAdapterList() failed for attribute " + wstring(L#attribute));\ + }\ + } + + // Enumerate adapters that support Direct3D 11 + if (filter != DeviceFilter::ComputeOnly && filter != DeviceFilter::DisplayAndCompute) { + ENUMERATE_ADAPTERS(DXCORE_ADAPTER_ATTRIBUTE_D3D11_GRAPHICS); + } + + // Enumerate adapters that support Direct3D 12 + if (filter != DeviceFilter::ComputeOnly) { + ENUMERATE_ADAPTERS(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS); + } + + // Enumerate adapters that support Direct3D 12 Core + if (filter != DeviceFilter::DisplayOnly) { + ENUMERATE_ADAPTERS(DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE); + } + + #undef ENUMERATE_ADAPTERS + + // Process each of the enumerated adapters and apply our filtering criteria + for (auto const& adapters : this->adapterLists) + { + const uint32_t count = adapters->GetAdapterCount(); + for (uint32_t index = 0; index < count; ++index) + { + // Extract the details for the current adapter + com_ptr adapter; + auto error = CheckHresult(adapters->GetAdapter(index, adapter.put())); + if (error) { + throw error.Wrap(L"IDXCoreAdapterList::GetAdapter() failed for index " + std::to_wstring(index)); + } + Adapter details = this->ExtractAdapterDetails(adapter); + + // Ignore software devices + if (!details.IsHardware) { + continue; + } + + // If the adapter does not match our filter mode then ignore it + if ((filter == DeviceFilter::DisplayOnly && details.SupportsCompute) || + (filter == DeviceFilter::ComputeOnly && details.SupportsDisplay) || + (filter == DeviceFilter::DisplayAndCompute && (!details.SupportsDisplay || !details.SupportsCompute))) { + continue; + } + + // If the adapter is integrated and we are not including integrated devices then ignore it + if (details.IsIntegrated && !includeIntegrated) { + continue; + } + + // If the adapter is detachable and we are not including detachable devices then ignore it + if (details.IsDetachable && !includeDetachable) { + continue; + } + + // Add the adapter to our set of unique adapters + this->uniqueAdapters.insert(std::make_pair(details.InstanceLuid, details)); + } + } + + // Log the list of unique adapter LUIDs + LOG(L"Enumerated DirectX adapters with LUIDs: {}", FMT(ObjectHelpers::GetMappingKeys(this->uniqueAdapters))); +} + +const map& AdapterEnumeration::GetUniqueAdapters() const { + return this->uniqueAdapters; +} + +bool AdapterEnumeration::IsStale() const +{ + // If we have not yet performed enumeration then report that our data is stale + if (this->adapterLists.empty()) + { + LOG(L"No adapter lists yet, need to perform enumeration"); + return true; + } + + // If any of our adapter lists are stale then our data is stale + for (auto const& list : this->adapterLists) + { + if (list->IsStale()) + { + LOG(L"Found stale adapter list"); + return true; + } + } + + return false; +} + +Adapter AdapterEnumeration::ExtractAdapterDetails(const com_ptr& adapter) const +{ + Adapter details; + DeviceDiscoveryError error; + + // Extract the adapter LUID and convert it to an int64_t + LUID instanceLuid; + error = CheckHresult(adapter->GetProperty(DXCoreAdapterProperty::InstanceLuid, &instanceLuid)); + if (error) { + throw error.Wrap(L"IDXCoreAdapter::GetProperty() failed for property InstanceLuid"); + } + details.InstanceLuid = Int64FromLuid(instanceLuid); + + // Extract the PnP hardware ID information + error = CheckHresult(adapter->GetProperty(DXCoreAdapterProperty::HardwareID, &details.HardwareID)); + if (error) { + throw error.Wrap(L"IDXCoreAdapter::GetProperty() failed for property HardwareID"); + } + + // Extract the boolean specifying whether the adapter is a hardware device + error = CheckHresult(adapter->GetProperty(DXCoreAdapterProperty::IsHardware, &details.IsHardware)); + if (error) { + throw error.Wrap(L"IDXCoreAdapter::GetProperty() failed for property IsHardware"); + } + + // Extract the boolean specifying whether the adapter is an integrated GPU + error = CheckHresult(adapter->GetProperty(DXCoreAdapterProperty::IsIntegrated, &details.IsIntegrated)); + if (error) { + throw error.Wrap(L"IDXCoreAdapter::GetProperty() failed for property IsIntegrated"); + } + + // Extract the boolean specifying whether the adapter is detachable + error = CheckHresult(adapter->GetProperty(DXCoreAdapterProperty::IsDetachable, &details.IsDetachable)); + if (error) { + throw error.Wrap(L"IDXCoreAdapter::GetProperty() failed for property IsDetachable"); + } + + // Determine whether the adapter supports display + details.SupportsDisplay = + adapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D11_GRAPHICS) || + adapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS); + + // Determine whether the adapter supports compute + details.SupportsCompute = adapter->IsAttributeSupported(DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS); + + return details; +} diff --git a/library/src/AdapterEnumeration.h b/library/src/AdapterEnumeration.h new file mode 100644 index 0000000..97d5d48 --- /dev/null +++ b/library/src/AdapterEnumeration.h @@ -0,0 +1,37 @@ +#pragma once + +#include "Adapter.h" +#include "DeviceFilter.h" + +using std::map; +using std::vector; +using winrt::com_ptr; + +class AdapterEnumeration +{ + public: + AdapterEnumeration(); + + // Enumerates the DirectX adapters that meet the specified filtering criteria + void EnumerateAdapters(const DeviceFilter& filter, bool includeIntegrated, bool includeDetachable); + + // Retrieves the list of unique adapters retrieved during the last enumeration operation + const map& GetUniqueAdapters() const; + + // Determines whether the list of adapters is stale and needs to be refreshed by performing enumeration again + bool IsStale() const; + + private: + + // Extracts the details from a DXCore adapter object + Adapter ExtractAdapterDetails(const com_ptr& adapter) const; + + // Our DXCore adapter factory + com_ptr adapterFactory; + + // Our collection of DXCore adapter lists, used for enumerating adapters with various capabilities + vector< com_ptr > adapterLists; + + // The list of unique adapters retrieved during the last enumeration operation, keyed by adapter LUID + map uniqueAdapters; +}; diff --git a/library/src/D3DHelpers.cpp b/library/src/D3DHelpers.cpp new file mode 100644 index 0000000..1ad67ec --- /dev/null +++ b/library/src/D3DHelpers.cpp @@ -0,0 +1,72 @@ +#include "D3DHelpers.h" +#include "ErrorHandling.h" +#include "ObjectHelpers.h" + +QueryD3DRegistryInfo::QueryD3DRegistryInfo() +{ + this->Resize(0); + this->RegistryInfo->PhysicalAdapterIndex = 0; +} + +void QueryD3DRegistryInfo::SetFilesystemQuery(D3DDDI_QUERYREGISTRY_TYPE queryType) +{ + ZeroMemory(this->RegistryInfo->ValueName, sizeof(wchar_t) * MAX_PATH); + this->RegistryInfo->QueryFlags.TranslatePath = 0; + this->RegistryInfo->QueryType = queryType; + this->RegistryInfo->ValueType = 0; +} + +void QueryD3DRegistryInfo::SetAdapterKeyQuery(wstring_view name, ULONG valueType, bool translatePaths) +{ + memcpy(this->RegistryInfo->ValueName, name.data(), sizeof(wchar_t) * name.size()); + this->RegistryInfo->QueryFlags.TranslatePath = (translatePaths ? 1 : 0); + this->RegistryInfo->QueryType = D3DDDI_QUERYREGISTRY_ADAPTERKEY; + this->RegistryInfo->ValueType = valueType; +} + +void QueryD3DRegistryInfo::Resize(size_t trailingBuffer) +{ + // Allocate memory for the new struct + buffer + this->PrivateDataSize = sizeof(D3DDDI_QUERYREGISTRY_INFO) + trailingBuffer; + auto newData = std::make_unique(this->PrivateDataSize); + + // If we have existing struct values then copy them over to the new struct + if (this->PrivateData) { + memcpy(newData.get(), this->PrivateData.get(), sizeof(D3DDDI_QUERYREGISTRY_INFO)); + } + + // Release the existing data (if any) and update our struct pointer + this->PrivateData = std::move(newData); + this->RegistryInfo = reinterpret_cast(this->PrivateData.get()); +} + +void QueryD3DRegistryInfo::PerformQuery(unique_adapter_handle& adapter) +{ + while (true) + { + // Attempt to perform the query + auto adapterQuery = this->CreateAdapterQuery(adapter); + auto error = CheckNtStatus(D3DKMTQueryAdapterInfo(&adapterQuery)); + if (error) { + throw error.Wrap(L"D3DKMTQueryAdapterInfo failed"); + } + + // Determine whether we need to resize the trailing buffer and try again + if (this->RegistryInfo->Status == D3DDDI_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW) { + this->Resize(this->RegistryInfo->OutputValueSize); + } + else { + return; + } + } +} + +D3DKMT_QUERYADAPTERINFO QueryD3DRegistryInfo::CreateAdapterQuery(unique_adapter_handle& adapter) +{ + auto adapterQuery = ObjectHelpers::GetZeroedStruct(); + adapterQuery.hAdapter = adapter.get(); + adapterQuery.Type = KMTQAITYPE_QUERYREGISTRY; + adapterQuery.pPrivateDriverData = this->PrivateData.get(); + adapterQuery.PrivateDriverDataSize = this->PrivateDataSize; + return adapterQuery; +} diff --git a/library/src/D3DHelpers.h b/library/src/D3DHelpers.h new file mode 100644 index 0000000..d2a3045 --- /dev/null +++ b/library/src/D3DHelpers.h @@ -0,0 +1,48 @@ +#pragma once + +using std::wstring_view; + + +// Closes the supplied DirectX adapter handle +inline NTSTATUS CloseAdapter(D3DKMT_HANDLE adapter) +{ + D3DKMT_CLOSEADAPTER close; + close.hAdapter = adapter; + return D3DKMTCloseAdapter(&close); +} + +// Auto-releasing resource wrapper type for DirectX adapter handles +typedef wil::unique_any unique_adapter_handle; + + +// Encapsulates a D3DDDI_QUERYREGISTRY_INFO struct, along with its trailing buffer for receiving output data +class QueryD3DRegistryInfo +{ + public: + + // Use this to access the struct's member fields + D3DDDI_QUERYREGISTRY_INFO* RegistryInfo; + + QueryD3DRegistryInfo(); + + // Populates the struct fields for querying a filesystem path + void SetFilesystemQuery(D3DDDI_QUERYREGISTRY_TYPE queryType); + + // Populates the struct fields for querying a registry value from the adapter key + void SetAdapterKeyQuery(wstring_view name, ULONG valueType, bool translatePaths); + + // Resizes the trailing buffer + void Resize(size_t trailingBuffer); + + // Performs a registry query against the specified adapter, resizing the trailing buffer to accommodate the output data size as needed + void PerformQuery(unique_adapter_handle& adapter); + + private: + + // The underlying data and size for the struct along with its trailing buffer + std::unique_ptr PrivateData; + size_t PrivateDataSize; + + // Creates a D3DKMT_QUERYADAPTERINFO struct that wraps struct and its trailing buffer + D3DKMT_QUERYADAPTERINFO CreateAdapterQuery(unique_adapter_handle& adapter); +}; diff --git a/library/src/Device.h b/library/src/Device.h new file mode 100644 index 0000000..be6cffc --- /dev/null +++ b/library/src/Device.h @@ -0,0 +1,61 @@ +#pragma once + +#include "Adapter.h" + +using std::vector; +using std::wstring; + + +// Represents an additional file that needs to be copied from the driver store to the system directory in order to use a device with non-DirectX runtimes +// (For details, see: ) +struct RuntimeFile +{ + RuntimeFile(wstring SourcePath, wstring DestinationFilename) + { + this->SourcePath = SourcePath; + this->DestinationFilename = DestinationFilename; + + // If no destination filename was specified then use the filename from the source path + if (this->DestinationFilename.empty()) { + this->DestinationFilename = std::filesystem::path(this->SourcePath).filename().wstring(); + } + } + + // The relative path to the file in the driver store + wstring SourcePath; + + // The filename that the file should be given when copied to the destination directory + wstring DestinationFilename; +}; + + +// Represents the underlying PnP device associated with a DirectX adapter +struct Device +{ + // The DirectX adapter associated with the PnP device + Adapter DeviceAdapter; + + // The unique PNP hardware identifier for the device + wstring ID; + + // A human-readable description of the device (e.g. the model name) + wstring Description; + + // The registry key that contains the driver details for the device + wstring DriverRegistryKey; + + // The absolute path to the directory in the driver store that contains the driver files for the device + wstring DriverStorePath; + + // The path to the physical location of the device in the system + wstring LocationPath; + + // The list of additional files that need to be copied from the driver store to the System32 directory in order to use the device with non-DirectX runtimes + vector RuntimeFiles; + + // The list of additional files that need to be copied from the driver store to the SysWOW64 directory in order to use the device with non-DirectX runtimes + vector RuntimeFilesWow64; + + // The vendor of the device (e.g. AMD, Intel, NVIDIA) + wstring Vendor; +}; diff --git a/library/src/DeviceDiscovery.cpp b/library/src/DeviceDiscovery.cpp new file mode 100644 index 0000000..a466f64 --- /dev/null +++ b/library/src/DeviceDiscovery.cpp @@ -0,0 +1,115 @@ +#include "DeviceDiscovery.h" +#include "DeviceDiscoveryImp.h" + +#define LIBRARY_VERSION L"0.0.1" + +#define INSTANCE (reinterpret_cast(instance)) + +const wchar_t* GetDiscoveryLibraryVersion() { + return LIBRARY_VERSION; +} + +void DisableDiscoveryLogging() { + spdlog::set_level(spdlog::level::off); +} + +void EnableDiscoveryLogging() +{ + spdlog::set_pattern("%^[directx-device-discovery.dll %Y-%m-%dT%T%z]%$ [%s:%# %!] %v", spdlog::pattern_time_type::local); + spdlog::set_level(spdlog::level::info); + spdlog::flush_on(spdlog::level::info); +} + +DeviceDiscoveryInstance CreateDeviceDiscoveryInstance() { + return new DeviceDiscoveryImp(); +} + +void DestroyDeviceDiscoveryInstance(DeviceDiscoveryInstance instance) { + delete INSTANCE; +} + +const wchar_t* DeviceDiscovery_GetLastErrorMessage(DeviceDiscoveryInstance instance) { + return INSTANCE->GetLastErrorMessage(); +} + +int DeviceDiscovery_IsRefreshRequired(DeviceDiscoveryInstance instance) { + return INSTANCE->IsRefreshRequired(); +} + +int DeviceDiscovery_DiscoverDevices(DeviceDiscoveryInstance instance, int filter, int includeIntegrated, int includeDetachable) +{ + bool success = INSTANCE->DiscoverDevices(static_cast(filter), includeIntegrated, includeDetachable); + return (success ? 0 : -1); +} + +int DeviceDiscovery_GetNumDevices(DeviceDiscoveryInstance instance) { + return INSTANCE->GetNumDevices(); +} + +long long DeviceDiscovery_GetDeviceAdapterLUID(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceAdapterLUID(device); +} + +const wchar_t* DeviceDiscovery_GetDeviceID(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceID(device); +} + +const wchar_t* DeviceDiscovery_GetDeviceDescription(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceDescription(device); +} + +const wchar_t* DeviceDiscovery_GetDeviceDriverRegistryKey(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceDriverRegistryKey(device); +} + +const wchar_t* DeviceDiscovery_GetDeviceDriverStorePath(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceDriverStorePath(device); +} + +const wchar_t* DeviceDiscovery_GetDeviceLocationPath(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceLocationPath(device); +} + +const wchar_t* DeviceDiscovery_GetDeviceVendor(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetDeviceVendor(device); +} + +int DeviceDiscovery_GetNumRuntimeFiles(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetNumRuntimeFiles(device); +} + +const wchar_t* DeviceDiscovery_GetRuntimeFileSource(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file) { + return INSTANCE->GetRuntimeFileSource(device, file); +} + +const wchar_t* DeviceDiscovery_GetRuntimeFileDestination(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file) { + return INSTANCE->GetRuntimeFileDestination(device, file); +} + +int DeviceDiscovery_GetNumRuntimeFilesWow64(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->GetNumRuntimeFilesWow64(device); +} + +const wchar_t* DeviceDiscovery_GetRuntimeFileSourceWow64(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file) { + return INSTANCE->GetRuntimeFileSourceWow64(device, file); +} + +const wchar_t* DeviceDiscovery_GetRuntimeFileDestinationWow64(DeviceDiscoveryInstance instance, unsigned int device, unsigned int file) { + return INSTANCE->GetRuntimeFileDestinationWow64(device, file); +} + +int DeviceDiscovery_IsDeviceIntegrated(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->IsDeviceIntegrated(device); +} + +int DeviceDiscovery_IsDeviceDetachable(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->IsDeviceDetachable(device); +} + +int DeviceDiscovery_DoesDeviceSupportDisplay(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->DoesDeviceSupportDisplay(device); +} + +int DeviceDiscovery_DoesDeviceSupportCompute(DeviceDiscoveryInstance instance, unsigned int device) { + return INSTANCE->DoesDeviceSupportCompute(device); +} diff --git a/library/src/DeviceDiscoveryImp.cpp b/library/src/DeviceDiscoveryImp.cpp new file mode 100644 index 0000000..c9b4b62 --- /dev/null +++ b/library/src/DeviceDiscoveryImp.cpp @@ -0,0 +1,260 @@ +#include "DeviceDiscoveryImp.h" +#include "ErrorHandling.h" +#include "RegistryQuery.h" + +#include +#include + +#define RETURN_ERROR(sentinel, message) this->SetLastErrorMessage(message); return sentinel +#define RETURN_SUCCESS(value) this->SetLastErrorMessage(L""); return value + +#define VERIFY_DEVICE(sentinel) try { this->ValidateRequestedDevice(device); } catch (const DeviceDiscoveryError& err) { RETURN_ERROR(sentinel, err.message); } +#define VERIFY_FILE() if (file >= files.size()) { RETURN_ERROR(nullptr, L"requested runtime file index is invalid: " + std::to_wstring(file)); } + +const wchar_t* DeviceDiscoveryImp::GetLastErrorMessage() const { + return this->lastError.c_str(); +} + +bool DeviceDiscoveryImp::IsRefreshRequired() +{ + // Make sure WinRT is initialised for the calling thread + Windows::Foundation::Initialize(RO_INIT_MULTITHREADED); + + // We require a refresh if we have no data or we have stale data + return (this->HaveDevices()) ? this->enumeration->IsStale() : true; +} + +bool DeviceDiscoveryImp::DiscoverDevices(DeviceFilter filter, bool includeIntegrated, bool includeDetachable) +{ + // Make sure WinRT is initialised for the calling thread + Windows::Foundation::Initialize(RO_INIT_MULTITHREADED); + + try + { + // If this is the first time we're performing device discovery then create our helper objects + if (!this->HaveDevices()) + { + this->enumeration = std::make_unique(); + this->wmi = std::make_unique(); + } + + // Enumerate the DirectX adapters that meet the supplied filtering criteria + this->enumeration->EnumerateAdapters(filter, includeIntegrated, includeDetachable); + + // Retrieve the PnP device details from WMI for each of the enumerated adapters + this->devices = this->wmi->GetDevicesForAdapters(this->enumeration->GetUniqueAdapters()); + + // Retrieve the driver details from the registry for each of the devices + for (auto& device : this->devices) { + RegistryQuery::FillDriverDetails(device); + } + + RETURN_SUCCESS(true); + } + catch (const DeviceDiscoveryError& err) { + RETURN_ERROR(false, err.Pretty()); + } + catch (const std::runtime_error& err) { + RETURN_ERROR(false, winrt::to_hstring(err.what())); + } +} + +int DeviceDiscoveryImp::GetNumDevices() +{ + // Verify that we have a device list + if (!this->HaveDevices()) { + RETURN_ERROR(-1, L"attempted to retrieve device count before performing device discovery"); + } + + RETURN_SUCCESS(this->devices.size()); +} + +long long DeviceDiscoveryImp::GetDeviceAdapterLUID(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Retrieve the adapter LUID of the specified device + RETURN_SUCCESS(this->devices[device].DeviceAdapter.InstanceLuid); +} + +const wchar_t* DeviceDiscoveryImp::GetDeviceID(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Retrieve the ID of the specified device + RETURN_SUCCESS(this->devices[device].ID.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetDeviceDescription(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Retrieve the human-readable description of the specified device + RETURN_SUCCESS(this->devices[device].Description.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetDeviceDriverRegistryKey(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Retrieve the path of the registry key with the driver details for the specified device + RETURN_SUCCESS(this->devices[device].DriverRegistryKey.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetDeviceDriverStorePath(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Retrieve the absolute path to the driver store directory for the specified device + RETURN_SUCCESS(this->devices[device].DriverStorePath.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetDeviceLocationPath(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Retrieve the physical location path of the specified device + RETURN_SUCCESS(this->devices[device].LocationPath.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetDeviceVendor(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Retrieve the vendor of the specified device + RETURN_SUCCESS(this->devices[device].Vendor.c_str()); +} + +int DeviceDiscoveryImp::GetNumRuntimeFiles(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Retrieve the number of additional runtime files for the device + RETURN_SUCCESS(this->devices[device].RuntimeFiles.size()); +} + +const wchar_t* DeviceDiscoveryImp::GetRuntimeFileSource(unsigned int device, unsigned int file) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Verify that the requested file entry exists + const vector& files = this->devices[device].RuntimeFiles; + VERIFY_FILE(); + + // Retrieve the source path for the file + RETURN_SUCCESS(files[file].SourcePath.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetRuntimeFileDestination(unsigned int device, unsigned int file) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Verify that the requested file entry exists + const vector& files = this->devices[device].RuntimeFiles; + VERIFY_FILE(); + + // Retrieve the destination filename for the file + RETURN_SUCCESS(files[file].DestinationFilename.c_str()); +} + +int DeviceDiscoveryImp::GetNumRuntimeFilesWow64(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Retrieve the number of additional SysWOW64 runtime files for the device + RETURN_SUCCESS(this->devices[device].RuntimeFilesWow64.size()); +} + +const wchar_t* DeviceDiscoveryImp::GetRuntimeFileSourceWow64(unsigned int device, unsigned int file) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Verify that the requested file entry exists + const vector& files = this->devices[device].RuntimeFilesWow64; + VERIFY_FILE(); + + // Retrieve the source path for the file + RETURN_SUCCESS(files[file].SourcePath.c_str()); +} + +const wchar_t* DeviceDiscoveryImp::GetRuntimeFileDestinationWow64(unsigned int device, unsigned int file) +{ + // Verify that the requested device exists + VERIFY_DEVICE(nullptr); + + // Verify that the requested file entry exists + const vector& files = this->devices[device].RuntimeFilesWow64; + VERIFY_FILE(); + + // Retrieve the destination filename for the file + RETURN_SUCCESS(files[file].DestinationFilename.c_str()); +} + +int DeviceDiscoveryImp::IsDeviceIntegrated(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Determine whether the specified device is an integrated GPU + RETURN_SUCCESS(this->devices[device].DeviceAdapter.IsIntegrated); +} + +int DeviceDiscoveryImp::IsDeviceDetachable(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Determine whether the specified device is detachable + RETURN_SUCCESS(this->devices[device].DeviceAdapter.IsDetachable); +} + +int DeviceDiscoveryImp::DoesDeviceSupportDisplay(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Determine whether the specified device supports display + RETURN_SUCCESS(this->devices[device].DeviceAdapter.SupportsDisplay); +} + +int DeviceDiscoveryImp::DoesDeviceSupportCompute(unsigned int device) +{ + // Verify that the requested device exists + VERIFY_DEVICE(-1); + + // Determine whether the specified device supports compute + RETURN_SUCCESS(this->devices[device].DeviceAdapter.SupportsCompute); +} + +bool DeviceDiscoveryImp::HaveDevices() const { + return (this->enumeration && this->wmi); +} + +void DeviceDiscoveryImp::SetLastErrorMessage(std::wstring_view message) { + this->lastError = message; +} + +void DeviceDiscoveryImp::ValidateRequestedDevice(unsigned int device) +{ + // Verify that we have a device list + if (!this->HaveDevices()) { + throw CreateError(L"attempted to retrieve device details before performing device discovery"); + } + + // Verify that the specified device index is valid + if (device >= this->GetNumDevices()) { + throw CreateError(L"requested device index is invalid: " + std::to_wstring(device)); + } +} diff --git a/library/src/DeviceDiscoveryImp.h b/library/src/DeviceDiscoveryImp.h new file mode 100644 index 0000000..8fdf6eb --- /dev/null +++ b/library/src/DeviceDiscoveryImp.h @@ -0,0 +1,51 @@ +#pragma once + +#include "AdapterEnumeration.h" +#include "Device.h" +#include "DeviceFilter.h" +#include "WmiQuery.h" + +using std::wstring; +using std::wstring_view; +using std::unique_ptr; +using std::vector; + +class DeviceDiscoveryImp +{ + public: + + DeviceDiscoveryImp() {} + const wchar_t* GetLastErrorMessage() const; + bool IsRefreshRequired(); + bool DiscoverDevices(DeviceFilter filter, bool includeIntegrated, bool includeDetachable); + int GetNumDevices(); + long long GetDeviceAdapterLUID(unsigned int device); + const wchar_t* GetDeviceID(unsigned int device); + const wchar_t* GetDeviceDescription(unsigned int device); + const wchar_t* GetDeviceDriverRegistryKey(unsigned int device); + const wchar_t* GetDeviceDriverStorePath(unsigned int device); + const wchar_t* GetDeviceLocationPath(unsigned int device); + const wchar_t* GetDeviceVendor(unsigned int device); + int GetNumRuntimeFiles(unsigned int device); + const wchar_t* GetRuntimeFileSource(unsigned int device, unsigned int file); + const wchar_t* GetRuntimeFileDestination(unsigned int device, unsigned int file); + int GetNumRuntimeFilesWow64(unsigned int device); + const wchar_t* GetRuntimeFileSourceWow64(unsigned int device, unsigned int file); + const wchar_t* GetRuntimeFileDestinationWow64(unsigned int device, unsigned int file); + int IsDeviceIntegrated(unsigned int device); + int IsDeviceDetachable(unsigned int device); + int DoesDeviceSupportDisplay(unsigned int device); + int DoesDeviceSupportCompute(unsigned int device); + + private: + + bool HaveDevices() const; + void SetLastErrorMessage(wstring_view message); + void ValidateRequestedDevice(unsigned int device); + + vector devices; + wstring lastError; + + unique_ptr enumeration; + unique_ptr wmi; +}; diff --git a/library/src/DllMain.cpp b/library/src/DllMain.cpp new file mode 100644 index 0000000..1794a7b --- /dev/null +++ b/library/src/DllMain.cpp @@ -0,0 +1,10 @@ +BOOL APIENTRY DllMain(HINSTANCE hModule, DWORD dwReason, PVOID lpReserved) +{ + if (dwReason == DLL_PROCESS_ATTACH) + { + // Disable logging by default + spdlog::set_level(spdlog::level::off); + } + + return TRUE; +} diff --git a/library/src/ErrorHandling.cpp b/library/src/ErrorHandling.cpp new file mode 100644 index 0000000..8e976d5 --- /dev/null +++ b/library/src/ErrorHandling.cpp @@ -0,0 +1,50 @@ +#include "ErrorHandling.h" + +DeviceDiscoveryError ErrorHandling::ErrorForNtStatus(NTSTATUS status, wstring_view file, wstring_view function, size_t line) +{ + if (status < 0) + { + // Allocate a buffer to hold the error message + size_t bufSize = 1024; + auto buffer = std::make_unique(bufSize); + + // Attempt to retrieve the error message for the status code + DWORD length = FormatMessageW( + FORMAT_MESSAGE_FROM_HMODULE | FORMAT_MESSAGE_IGNORE_INSERTS, + GetModuleHandleW(L"ntdll.dll"), + status, + 0, + buffer.get(), + bufSize, + nullptr + ); + + if (length > 0) + { + // If the message has a trailing newline then remove it + wstring message(buffer.get(), length); + size_t newline = message.find_last_of(L"\r\n"); + if (newline != wstring::npos) { + message = message.substr(0, newline - 1); + } + + // Return an error with the retrieved message + return DeviceDiscoveryError(message, file, function, line); + } + else + { + // Return an error with the hexadecimal representation of the NTSTATUS code + return DeviceDiscoveryError( + fmt::format( + L"Unable to retrieve error message for NTSTATUS code 0x{:0>8X}", + static_cast(status) + ), + file, + function, + line + ); + } + } + + return DeviceDiscoveryError(L"", file, function, line); +} diff --git a/library/src/ErrorHandling.h b/library/src/ErrorHandling.h new file mode 100644 index 0000000..c4df6d0 --- /dev/null +++ b/library/src/ErrorHandling.h @@ -0,0 +1,103 @@ +#pragma once + +using std::wstring; +using std::wstring_view; + + +// Define Unicode versions of __FILE__ and __FUNCTION__ +#define __WIDE2(x) L##x +#define __WIDE1(x) __WIDE2(x) +#define __WFILE__ __WIDE1(__FILE__) +#define __WFUNCTION__ __WIDE1(__FUNCTION__) + + +// The exception type used to represent all errors inside the device discovery library +class DeviceDiscoveryError +{ + public: + inline DeviceDiscoveryError() : message(L""), file(L""), function(L""), line(0) {} + + inline DeviceDiscoveryError(wstring_view message, wstring_view file, wstring_view function, size_t line) : + message(message), file(file), function(function), line(line) + {} + + inline DeviceDiscoveryError(wstring_view message, const DeviceDiscoveryError& inner) : + file(inner.file), function(inner.function), line(inner.line) + { + this->message = wstring(message) + L": " + inner.message; + } + + DeviceDiscoveryError(const DeviceDiscoveryError& other) = default; + DeviceDiscoveryError(DeviceDiscoveryError&& other) = default; + DeviceDiscoveryError& operator=(const DeviceDiscoveryError& other) = default; + DeviceDiscoveryError& operator=(DeviceDiscoveryError&& other) = default; + + inline operator bool() const { + return (!this->message.empty()); + } + + // Wraps this error in a surrounding error message + inline DeviceDiscoveryError Wrap(wstring_view message) const { + return DeviceDiscoveryError(message, *this); + } + + // Formats the error details as a pretty string + inline wstring Pretty() const + { + // Extract the filename from the file path + wstring filename = std::filesystem::path(this->file).filename().wstring(); + + // Append the filename, line number and function name to the error message + wstring fileAndLine = filename + L":" + std::to_wstring(this->line); + return this->message + L" [" + fileAndLine + L" " + this->function + L"]"; + } + + wstring message; + wstring file; + wstring function; + size_t line; +}; + + +// Provides functionality related to managing errors +namespace ErrorHandling +{ + // Returns an error object representing the supplied NTSTATUS code + DeviceDiscoveryError ErrorForNtStatus(NTSTATUS status, wstring_view file, wstring_view function, size_t line); + + // Returns an error object representing the supplied HRESULT code + inline DeviceDiscoveryError ErrorForHresult(const winrt::hresult& result, wstring_view file, wstring_view function, size_t line) + { + try + { + winrt::check_hresult(result); + return DeviceDiscoveryError(L"", file, function, line); + } + catch (const winrt::hresult_error& err) { + return DeviceDiscoveryError(err.message(), file, function, line); + } + } + + // Returns an error object representing the supplied Win32 error code + template + inline DeviceDiscoveryError ErrorForWin32(T error, wstring_view file, wstring_view function, size_t line) + { + try + { + winrt::check_win32(error); + return DeviceDiscoveryError(L"", file, function, line); + } + catch (const winrt::hresult_error& err) { + return DeviceDiscoveryError(err.message(), file, function, line); + } + } + + // Convenience macros for automatically filling out error file, function and line details + #define CreateError(message) DeviceDiscoveryError(message, __WFILE__, __WFUNCTION__, __LINE__) + #define CheckNtStatus(status) ErrorHandling::ErrorForNtStatus(status, __WFILE__, __WFUNCTION__, __LINE__) + #define CheckHresult(status) ErrorHandling::ErrorForHresult(status, __WFILE__, __WFUNCTION__, __LINE__) + #define CheckWin32(status) ErrorHandling::ErrorForWin32(status, __WFILE__, __WFUNCTION__, __LINE__) + + // Catches a winrt::hresult_error object and converts it to a DeviceDiscoveryError object + #define CatchHresult(error, operation) try { operation; error = DeviceDiscoveryError(); } catch (const winrt::hresult_error & err) { error = CreateError(err.message()); } +} diff --git a/library/src/ObjectHelpers.h b/library/src/ObjectHelpers.h new file mode 100644 index 0000000..322e5da --- /dev/null +++ b/library/src/ObjectHelpers.h @@ -0,0 +1,31 @@ +#pragma once + +namespace ObjectHelpers { + + +// Retrieves the list of keys for an STL associative container type (maps, sets, etc.) +// For the full list of supported container types, see: +// - +// - +template +std::vector GetMappingKeys(const MappingType& mapping) +{ + std::vector keys; + + for (const auto& pair : mapping) { + keys.push_back(pair.first); + } + + return keys; +} + +// Returns a zeroed-out instance of the specified struct type +template T GetZeroedStruct() +{ + T instance; + ZeroMemory(&instance, sizeof(T)); + return instance; +} + + +} // namespace ObjectHelpers diff --git a/library/src/RegistryQuery.cpp b/library/src/RegistryQuery.cpp new file mode 100644 index 0000000..2d8b5ea --- /dev/null +++ b/library/src/RegistryQuery.cpp @@ -0,0 +1,198 @@ +#include "RegistryQuery.h" +#include "D3DHelpers.h" +#include "ErrorHandling.h" +#include "ObjectHelpers.h" + +#include +#include +#include + +map< wstring, vector > RegistryQuery::EnumerateMultiStringValues(unique_hkey& key) +{ + map< wstring, vector > values; + + LSTATUS result = ERROR_SUCCESS; + for (int i = 0; ; ++i) + { + // Receives the type of the enumerated value + DWORD valueType = 0; + + // Receives the name of the enumerated value + DWORD nameBufsize = 256; + auto valueName = std::make_unique(nameBufsize); + + // Receives the data of the enumerated value + DWORD dataBufsize = 1024; + auto valueData = std::make_unique(dataBufsize); + + // Retrieve the next value and check to see if we have processed all available values + result = RegEnumValueW(key.get(), i, valueName.get(), &nameBufsize, nullptr, &valueType, valueData.get(), &dataBufsize); + if (result == ERROR_NO_MORE_ITEMS) { + break; + } + + // Report any errors + auto error = CheckWin32(result); + if (error) { + throw error.Wrap(L"RegEnumValueW failed"); + } + + // Verify that the value data is of type REG_MULTI_SZ + wstring name = wstring(valueName.get(), nameBufsize); + if (valueType != REG_MULTI_SZ) { + throw CreateError(L"enumerated value was not of type REG_MULTI_SZ: " + name); + } + + // Parse the value data and add it to our mapping + auto strings = RegistryQuery::ExtractMultiStringValue((wchar_t*)(valueData.get()), dataBufsize); + values.insert(std::make_pair(name, strings)); + } + + return values; +} + +vector RegistryQuery::ExtractMultiStringValue(const wchar_t* data, size_t numBytes) +{ + vector strings; + + size_t offset = 0; + size_t upperBound = numBytes / sizeof(wchar_t); + while (offset < upperBound) + { + // Extract the next string and check that it's not empty + wstring nextString(data + offset); + if (nextString.size() == 0) { break; } + + // Add the string to our list and proceed to the next one + strings.push_back(nextString); + offset += strings.back().size() + 1; + } + + return strings; +} + +unique_hkey RegistryQuery::OpenKeyFromString(wstring_view key) +{ + // Our list of supported root keys + static map rootKeys = { + { L"HKEY_CLASSES_ROOT", HKEY_CLASSES_ROOT }, + { L"HKEY_CURRENT_CONFIG", HKEY_CURRENT_CONFIG }, + { L"HKEY_CURRENT_USER", HKEY_CURRENT_USER }, + { L"HKEY_LOCAL_MACHINE", HKEY_LOCAL_MACHINE }, + { L"HKEY_PERFORMANCE_DATA", HKEY_PERFORMANCE_DATA }, + { L"HKEY_USERS", HKEY_USERS } + }; + + // Verify that the supplied key path is well-formed + size_t backslash = key.find_first_of(L"\\"); + if (backslash == wstring_view::npos || backslash >= (key.size()-1)) { + throw CreateError(L"invalid registry key path: " + wstring(key)); + } + + // Split the root key name from the rest of the path + wstring rootKeyName = wstring(key.substr(0, backslash)); + wstring keyPath = wstring(key.substr(backslash + 1)); + + // Identify the handle for the specified root key + auto rootKey = rootKeys.find(rootKeyName); + if (rootKey == rootKeys.end()) { + throw CreateError(L"unknown registry root key: " + rootKeyName); + } + + // Attempt to open the key + unique_hkey keyHandle; + auto error = CheckWin32(RegOpenKeyExW(rootKey->second, keyPath.c_str(), 0, KEY_READ, keyHandle.put())); + if (error) { + throw error.Wrap(L"failed to open registry key " + wstring(key)); + } + + return keyHandle; +} + +void RegistryQuery::ProcessRuntimeFiles(Device& device, wstring_view key, bool isWow64) +{ + try + { + // Determine whether we are adding runtime files to the device's System32 list or SysWOW64 list + auto& list = (isWow64) ? device.RuntimeFilesWow64 : device.RuntimeFiles; + + // Attempt to open the specified registry key and enumerate its REG_MULTI_SZ values + unique_hkey registryKey = RegistryQuery::OpenKeyFromString(device.DriverRegistryKey + L"\\" + wstring(key)); + auto files = RegistryQuery::EnumerateMultiStringValues(registryKey); + for (const auto& pair : files) + { + if (!pair.second.empty()) + { + // Construct a RuntimeFile from the string values + RuntimeFile newFile(pair.second[0], ((pair.second.size() == 2) ? pair.second[1] : L"")); + + // Check whether the destination filename for the runtime file clashes with an existing file + auto existing = std::find_if(list.begin(), list.end(), [newFile](RuntimeFile f) { + return f.DestinationFilename == newFile.DestinationFilename; + }); + + // Only add the new runtime file to the list if there's no clash + if (existing == list.end()) { + list.push_back(newFile); + } + else { + LOG(L"{}: ignoring runtime file with duplicate destination filename {}", key, newFile.DestinationFilename); + } + } + } + } + catch (const DeviceDiscoveryError& err) { + LOG(L"Could not enumerate runtime files for the {} key: {}", key, err.message); + } +} + +void RegistryQuery::FillDriverDetails(Device& device) +{ + // Log the device ID to provide context for any subsequent log messages and errors + LOG(L"Querying device driver registry details for device {}", device.ID); + + // Attempt to open the DirectX adapter for the device + auto adapterDetails = ObjectHelpers::GetZeroedStruct(); + adapterDetails.AdapterLuid = LuidFromInt64(device.DeviceAdapter.InstanceLuid); + auto error = CheckNtStatus(D3DKMTOpenAdapterFromLuid(&adapterDetails)); + if (error) + { + throw error.Wrap( + L"D3DKMTOpenAdapterFromLuid failed to open adapter with LUID " + + std::to_wstring(device.DeviceAdapter.InstanceLuid) + ); + } + + // Ensure we automatically close the adapter handle when we finish + unique_adapter_handle adapter(adapterDetails.hAdapter); + + // Retrieve the path to the driver store directory for the adapter + QueryD3DRegistryInfo queryDriverStore; + queryDriverStore.SetFilesystemQuery(D3DDDI_QUERYREGISTRY_DRIVERSTOREPATH); + queryDriverStore.PerformQuery(adapter); + device.DriverStorePath = wstring(queryDriverStore.RegistryInfo->OutputString); + + // If the driver store path begins with the "\SystemRoot" prefix then expand it + wstring prefix = L"\\SystemRoot"; + wstring systemRoot = wstring(wil::GetEnvironmentVariableW(L"SystemRoot").get()); + if (device.DriverStorePath.find(prefix, 0) == 0) { + device.DriverStorePath = device.DriverStorePath.replace(0, prefix.size(), systemRoot); + } + + // Determine whether we're running on the host or inside a container + // (e.g. when using a client tool to verify that a device has been mounted correctly) + if (device.DriverStorePath.find(L"HostDriverStore", 0) != wstring::npos) + { + // We have no way of enumerating the CopyToVmWhenNewer subkey inside a container, so stop processing here + LOG(L"Running inside a container, skipping runtime file enumeration"); + return; + } + + // Retrieve the list of additional runtime files that need to be copied to the System32 directory + RegistryQuery::ProcessRuntimeFiles(device, L"CopyToVmOverwrite", false); + RegistryQuery::ProcessRuntimeFiles(device, L"CopyToVmWhenNewer", false); + + // Retrieve the list of additional runtime files that need to be copied to the SysWOW64 directory + RegistryQuery::ProcessRuntimeFiles(device, L"CopyToVmOverwriteWow64", true); + RegistryQuery::ProcessRuntimeFiles(device, L"CopyToVmWhenNewerWow64", true); +} diff --git a/library/src/RegistryQuery.h b/library/src/RegistryQuery.h new file mode 100644 index 0000000..c66ce2f --- /dev/null +++ b/library/src/RegistryQuery.h @@ -0,0 +1,28 @@ +#pragma once + +#include "Device.h" + +using std::map; +using std::vector; +using std::wstring; +using std::wstring_view; +using wil::unique_hkey; + +// Provides functionality for querying the Windows registry +namespace RegistryQuery +{ + // Enumerates the values of the supplied registry key and parses their data as REG_MULTI_SZ + map< wstring, vector > EnumerateMultiStringValues(unique_hkey& key); + + // Extracts the individual strings of a REG_MULTI_SZ registry value + vector ExtractMultiStringValue(const wchar_t* data, size_t numBytes); + + // Parses a registry key path and opens it using the appropriate root key + unique_hkey OpenKeyFromString(wstring_view key); + + // Enumerates the runtime files for a device as listed under the specified registry key + void ProcessRuntimeFiles(Device& device, wstring_view key, bool isWow64); + + // Queries the registry to retrieve driver-related details for the supplied PnP device + void FillDriverDetails(Device& device); +} diff --git a/library/src/SafeArray.cpp b/library/src/SafeArray.cpp new file mode 100644 index 0000000..cbcf917 --- /dev/null +++ b/library/src/SafeArray.cpp @@ -0,0 +1,34 @@ +#include "SafeArray.h" + +unique_variant SafeArrayFactory::CreateStringArray(initializer_list elems) +{ + // Create our array bounds descriptor + SAFEARRAYBOUND bounds; + bounds.lLbound = 0; + bounds.cElements = elems.size(); + + // Create a VARIANT to hold our array + unique_variant vtArray; + vtArray.vt = VT_ARRAY | VT_BSTR; + + // Create the SAFEARRAY and lock it for data access + vtArray.parray = SafeArrayCreate(VT_BSTR, 1, &bounds); + auto error = CheckHresult(SafeArrayLock(vtArray.parray)); + if (error) { + throw error.Wrap(L"SafeArrayLock failed"); + } + + // Populate the array with the supplied elements + BSTR* array = reinterpret_cast(vtArray.parray->pvData); + int index = 0; + for (const auto& elem : elems) + { + // Note that a SAFEARRAY owns the memory of its elements, so we transfer ownership of each BSTR + array[index] = wil::make_bstr(elem.c_str()).release(); + index++; + } + + // Unlock the array + SafeArrayUnlock(vtArray.parray); + return vtArray; +} diff --git a/library/src/SafeArray.h b/library/src/SafeArray.h new file mode 100644 index 0000000..3e5c57e --- /dev/null +++ b/library/src/SafeArray.h @@ -0,0 +1,71 @@ +#pragma once + +#include "ErrorHandling.h" + +using std::initializer_list; +using std::wstring; +using wil::unique_variant; + + +// Provides functionality for iterating over the contents of a one-dimensional SAFEARRAY +template +class SafeArrayIterator +{ + public: + SafeArrayIterator(SAFEARRAY* array) + { + // Lock the array for data access + this->array = array; + this->reinterpretedArray = reinterpret_cast(this->array->pvData); + auto error = CheckHresult(SafeArrayLock(this->array)); + if (error) { + throw error.Wrap(L"SafeArrayLock failed"); + } + + // Retrieve the array bounds and compute the number of elements + long lowerBound = 0; + long upperBound = 0; + SafeArrayGetLBound(this->array, 1, &lowerBound); + SafeArrayGetUBound(this->array, 1, &upperBound); + this->numElements = (upperBound - lowerBound) + 1; + } + + ~SafeArrayIterator() + { + // Unlock the array + SafeArrayUnlock(this->array); + this->array = nullptr; + this->reinterpretedArray = nullptr; + } + + T* begin() { + return this->reinterpretedArray; + } + + const T* begin() const { + return this->reinterpretedArray; + } + + T* end() { + return this->reinterpretedArray + this->numElements; + } + + const T* end() const { + return this->reinterpretedArray + this->numElements; + } + + private: + SAFEARRAY* array; + T* reinterpretedArray; + long numElements; +}; + + +// Provides functionality for creating one-dimensional SAFEARRAY instances for specific element types +class SafeArrayFactory +{ + public: + + // Creates a SAFEARRAY of BSTR strings and wraps it in a VARIANT + static unique_variant CreateStringArray(initializer_list elems); +}; diff --git a/library/src/WmiQuery.cpp b/library/src/WmiQuery.cpp new file mode 100644 index 0000000..54fd471 --- /dev/null +++ b/library/src/WmiQuery.cpp @@ -0,0 +1,355 @@ +#include "WmiQuery.h" +#include "ErrorHandling.h" +#include "SafeArray.h" + +#include +#include + +using std::set; +using wil::unique_variant; +using winrt::hstring; + +namespace +{ + // Device property key for retrieving the DirectX adapter LUID + const DEVPROPKEY DEVPKEY_Device_AdapterLuid = { + { 0x60b193cb, 0x5276, 0x4d0f, { 0x96, 0xfc, 0xf1, 0x73, 0xab, 0xad, 0x3e, 0xc6 } }, + 2 + }; + + // Formats a DEVPROPKEY as a string in the form "{00000000-0000-0000-0000-000000000000} 0" + wstring DevPropKeyToString(const DEVPROPKEY& key) + { + return fmt::format( + L"{{{:0>8X}-{:0>4X}-{:0>4X}-{:0>2X}{:0>2X}-{:0>2X}{:0>2X}{:0>2X}{:0>2X}{:0>2X}{:0>2X}}} {}", + key.fmtid.Data1, + key.fmtid.Data2, + key.fmtid.Data3, + key.fmtid.Data4[0], + key.fmtid.Data4[1], + key.fmtid.Data4[2], + key.fmtid.Data4[3], + key.fmtid.Data4[4], + key.fmtid.Data4[5], + key.fmtid.Data4[6], + key.fmtid.Data4[7], + key.pid + ); + } + + // Formats a PnP hardware ID for use in a WQL query + wstring FormatHardwareID(const DXCoreHardwareID& dxHardwareID) + { + // Determine whether the subsystem ID value includes a subsystem vendor ID + bool haveSubsystemVendor = 0xffff0000 & dxHardwareID.subSysID; + + // Build a PCI hardware identifier string as per: + // + // and insert wildcards for the subsystem vendor ID (if absent) and the device instance + return fmt::format( + L"PCI\\\\VEN_{:0>4X}&DEV_{:0>4X}&SUBSYS_{:0>4X}{}&REV_{:0>2X}%", + dxHardwareID.vendorID, + dxHardwareID.deviceID, + dxHardwareID.subSysID, + (haveSubsystemVendor ? L"" : L"%"), + dxHardwareID.revision + ); + } +} + +WmiQuery::WmiQuery() +{ + // Create a reusable error object + DeviceDiscoveryError error; + + // Generate the string identifier for the DEVPKEY_Device_AdapterLuid device property key + this->devPropKeyLUID = DevPropKeyToString(DEVPKEY_Device_AdapterLuid); + + // Create our IWbemLocator instance + CatchHresult(error, this->wbemLocator = winrt::create_instance(CLSID_WbemLocator)); + if (error) { + throw error.Wrap(L"failed to create an IWbemLocator instance"); + } + + // Connect to the WMI service and retrieve a service proxy object + error = CheckHresult(this->wbemLocator->ConnectServer( + wil::make_bstr(L"ROOT\\CIMV2").get(), + nullptr, + nullptr, + nullptr, + 0, + nullptr, + nullptr, + this->wbemServices.put() + )); + if (error) { + throw error.Wrap(L"failed to connect to the WMI service"); + } + + // Set the security level for the service proxy + error = CheckHresult(CoSetProxyBlanket( + this->wbemServices.get(), + RPC_C_AUTHN_WINNT, + RPC_C_AUTHZ_NONE, + nullptr, + RPC_C_AUTHN_LEVEL_CALL, + RPC_C_IMP_LEVEL_IMPERSONATE, + nullptr, + EOAC_NONE + )); + if (error) { + throw error.Wrap(L"failed to set the security level for the WMI service proxy"); + } + + // Retrieve the CIM class definition for the Win32_PnPEntity class + error = CheckHresult(this->wbemServices->GetObject( + wil::make_bstr(L"Win32_PnPEntity").get(), + 0, + nullptr, + this->pnpEntityClass.put(), + nullptr + )); + if (error) { + throw error.Wrap(L"failed to retrieve the CIM class definition for the Win32_PnPEntity class"); + } + + // Retrieve the input parameters class for the `GetDeviceProperties` method of the CIM class definition + error = CheckHresult(this->pnpEntityClass->GetMethod(L"GetDeviceProperties", 0, this->inputParameters.put(), nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve the input parameters class for Win32_PnPEntity::GetDeviceProperties"); + } +} + +vector WmiQuery::GetDevicesForAdapters(const map& adapters) +{ + // If we don't have any adapters then don't query WMI + if (adapters.empty()) + { + LOG(L"Empty adapter list provided, skipping WMI query"); + return {}; + } + + // Gather the unique PnP hardware IDs from the DirectX adapters for use in our WQL query string + set hardwareIDs; + for (auto const& adapter : adapters) { + hardwareIDs.insert(FormatHardwareID(adapter.second.HardwareID)); + } + + // Build the WQL query string to retrieve the PnP devices associated with the adapters + wstring query = L"SELECT * FROM Win32_PnPEntity WHERE Present = TRUE AND ("; + int index = 0; + int last = hardwareIDs.size() - 1; + for (auto const& id : hardwareIDs) + { + query += L"DeviceID LIKE \"" + id + L"\"" + ((index < last) ? L" OR " : L""); + index++; + } + query += L")"; + + // Log the query string + LOG(L"Executing WQL query: {}", query); + + // Execute the query + com_ptr enumerator; + auto error = CheckHresult(wbemServices->ExecQuery( + wil::make_bstr(L"WQL").get(), + wil::make_bstr(query.c_str()).get(), + 0, + nullptr, + enumerator.put() + )); + if (error) { + throw error.Wrap(L"WQL query execution failed"); + } + + // Iterate over the retrieved PnP devices and match them to their corresponding DirectX adapters + vector devices; + for (int index = 0; ; index++) + { + // Retrieve the device for the current loop iteration + ULONG numReturned = 0; + com_ptr device; + auto error = CheckHresult(enumerator->Next(WBEM_INFINITE, 1, device.put(), &numReturned)); + if (error) { + throw error.Wrap(L"enumerating PnP devices failed"); + } + if (numReturned == 0) { + break; + } + + // Extract the details for the device and determine whether it matches any of our adapters + Device details = this->ExtractDeviceDetails(device); + auto matchingAdapter = adapters.find(details.DeviceAdapter.InstanceLuid); + if (matchingAdapter != adapters.end()) + { + // Log the match + LOG(L"Matched adapter LUID {} to PnP device {}", details.DeviceAdapter.InstanceLuid, details.ID); + + // Replace the device's adapter details with the matching adapter + details.DeviceAdapter = matchingAdapter->second; + + // Include the device in our results + devices.push_back(details); + } + } + + return devices; +} + +Device WmiQuery::ExtractDeviceDetails(const com_ptr& device) const +{ + Device details; + DeviceDiscoveryError error; + + // Retrieve the unique PnP device ID of the device + unique_variant vtDeviceID; + error = CheckHresult(device->Get(L"DeviceID", 0, &vtDeviceID, nullptr, nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve DeviceID property of PnP device"); + } + details.ID = winrt::to_hstring(vtDeviceID.bstrVal); + + // Retrieve the human-readable description of the device + unique_variant vtDescription; + error = CheckHresult(device->Get(L"Description", 0, &vtDescription, nullptr, nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve Description property of PnP device"); + } + details.Description = winrt::to_hstring(vtDescription.bstrVal); + + // Retrieve the vendor of the device + unique_variant vtVendor; + error = CheckHresult(device->Get(L"Manufacturer", 0, &vtVendor, nullptr, nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve Manufacturer property of PnP device"); + } + details.Vendor = winrt::to_hstring(vtVendor.bstrVal); + + // Retrieve the object path for the instance so we can call instance methods with it + unique_variant vtPath; + error = CheckHresult(device->Get(L"__Path", 0, &vtPath, nullptr, nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve __Path property of PnP device"); + } + + // Create an instance of the input parameters type for the `GetDeviceProperties` instance method + com_ptr inputArgs; + error = CheckHresult(this->inputParameters->SpawnInstance(0, inputArgs.put())); + if (error) { + throw error.Wrap(L"failed to spawn input parameters instance for Win32_PnPEntity::GetDeviceProperties"); + } + + // Populate the input parameters with the list of decive property keys we want to retrieve + unique_variant vtPropertyKeys = SafeArrayFactory::CreateStringArray({ + L"DEVPKEY_Device_Driver", + L"DEVPKEY_Device_LocationPaths", + this->devPropKeyLUID + }); + error = CheckHresult(inputArgs->Put(L"devicePropertyKeys", 0, &vtPropertyKeys, CIM_FLAG_ARRAY | CIM_STRING)); + if (error) { + throw error.Wrap(L"failed to assign input parameters array for Win32_PnPEntity::GetDeviceProperties"); + } + + // Call the `GetDeviceProperties` instance method + com_ptr callResult; + error = CheckHresult(this->wbemServices->ExecMethod( + vtPath.bstrVal, + wil::make_bstr(L"GetDeviceProperties").get(), + 0, + nullptr, + inputArgs.get(), + nullptr, + callResult.put() + )); + if (error) { + throw error.Wrap(L"failed to invoke Win32_PnPEntity::GetDeviceProperties()"); + } + + // Retrieve the return value + com_ptr returnValue; + error = CheckHresult(callResult->GetResultObject(WBEM_INFINITE, returnValue.put())); + if (error) { + throw error.Wrap(L"failed to retrieve return value for Win32_PnPEntity::GetDeviceProperties"); + } + + // Extract the device properties array and verify that it matches the expected type + unique_variant vtPropertiesArray; + error = CheckHresult(returnValue->Get(L"deviceProperties", 0, &vtPropertiesArray, nullptr, nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve deviceProperties property of Win32_PnPEntity::GetDeviceProperties return value"); + } + if (vtPropertiesArray.vt != (VT_ARRAY | VT_UNKNOWN)) { + throw CreateError(L"deviceProperties value was not an array of IUnknown objects"); + } + + // Iterate over the device properties array + SafeArrayIterator propertiesIterator(vtPropertiesArray.parray); + for (auto element : propertiesIterator) + { + // Cast the property object to an IWbemClassObject + IWbemClassObject* object = nullptr; + error = CheckHresult(element->QueryInterface(&object)); + if (error) { + throw error.Wrap(L"IUnknown::QueryInterface() failed for Win32_PnPDeviceProperty object"); + } + + // Retrieve the key name of the property + unique_variant vtKeyName; + error = CheckHresult(object->Get(L"KeyName", 0, &vtKeyName, nullptr, nullptr)); + if (error) { + throw error.Wrap(L"failed to retrieve KeyName property of PnP device property"); + } + wstring keyName(winrt::to_hstring(vtKeyName.bstrVal)); + + // Attempt to retrieve the value of the property + unique_variant data; + HRESULT result = object->Get(L"Data", 0, &data, nullptr, nullptr); + if (FAILED(result)) + { + // The property has no value, so ignore it + continue; + } + + // Determine which device property we are dealing with + if (keyName == L"DEVPKEY_Device_Driver") + { + // Verify that the device driver value is of the expected type + if (data.vt != VT_BSTR) { + throw CreateError(L"DeviceDriver value was not a string"); + } + + // Construct the full path to the registry key for the device's driver + details.DriverRegistryKey = + L"HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\" + + winrt::to_hstring(data.bstrVal); + } + else if (keyName == L"DEVPKEY_Device_LocationPaths") + { + // Verify that the LocationPaths array is of the expected type + if (data.vt != (VT_ARRAY | VT_BSTR)) { + throw CreateError(L"LocationPaths value was not an array of strings"); + } + + // Retrieve the first element from the LocationPaths array + SafeArrayIterator locationIterator(data.parray); + details.LocationPath = winrt::to_hstring(*locationIterator.begin()); + } + else if (keyName == this->devPropKeyLUID) + { + // Determine whether the LUID value is represented as a raw 64-bit integer or a string representation + if (data.vt == VT_I8) { + details.DeviceAdapter.InstanceLuid = data.llVal; + } + else if (data.vt == VT_BSTR) + { + // Parse the string back into a 64-bit integer + details.DeviceAdapter.InstanceLuid = std::stoll(winrt::to_string(data.bstrVal)); + } + else { + throw CreateError(L"LUID value was not a 64-bit integer or a string"); + } + } + } + + return details; +} diff --git a/library/src/WmiQuery.h b/library/src/WmiQuery.h new file mode 100644 index 0000000..c52aa08 --- /dev/null +++ b/library/src/WmiQuery.h @@ -0,0 +1,34 @@ +#pragma once + +#include "Adapter.h" +#include "Device.h" + +using std::map; +using std::vector; +using std::wstring; +using winrt::com_ptr; + +// Provides functionality for querying Windows Management Instrumentation (WMI) +class WmiQuery +{ + public: + + WmiQuery(); + + // Retrieves the device details for the underlying PnP devices associated with the supplied DirectX adapters + vector GetDevicesForAdapters(const map& adapters); + + private: + + // Extracts the details from a PnP device + Device ExtractDeviceDetails(const com_ptr& device) const; + + // Our COM objects for communicating with WMI + com_ptr wbemLocator; + com_ptr wbemServices; + com_ptr pnpEntityClass; + com_ptr inputParameters; + + // The string identifier for the DEVPROPKEY_GPU_LUID device property key + wstring devPropKeyLUID; +}; diff --git a/library/src/pch.h b/library/src/pch.h new file mode 100644 index 0000000..e3c3169 --- /dev/null +++ b/library/src/pch.h @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Prevent the GetObject => GetObjectW macro definition from the Windows headers from interfering with the IDL for IWbemServices +#ifdef GetObject +#undef GetObject +#endif +#include + +// Enable wchar_t support for filenames in spdlog +#define SPDLOG_WCHAR_FILENAMES +#include +#define LOG(...) SPDLOG_INFO(__VA_ARGS__) + +// Include the range formatting support from fmt to facilitate logging container types +#include +#define FMT(x) fmt::format(L"{}", x) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/library/test/test-device-discovery-cpp.cpp b/library/test/test-device-discovery-cpp.cpp new file mode 100644 index 0000000..9a5bbb4 --- /dev/null +++ b/library/test/test-device-discovery-cpp.cpp @@ -0,0 +1,82 @@ +#include "DeviceDiscovery.h" + +#include +#include +#include +#include +#include +using std::endl; +using std::vector; +using std::wstring; +using std::wclog; +using std::wcout; + +wstring FormatBoolean(bool value) { + return (value ? L"true" : L"false"); +} + +int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) +{ + // Gather our command-line arguments + vector args; + for (int i = 0; i < argc; ++i) { + args.push_back(argv[i]); + } + + // Enable verbose logging for the device discovery library if it has been requested + if (std::find(args.begin(), args.end(), L"--verbose") != args.end()) { + EnableDiscoveryLogging(); + } + + try + { + // Perform device discovery + DeviceDiscovery discovery; + discovery.DiscoverDevices(DeviceFilter::AllDevices, true, true); + int numDevices = discovery.GetNumDevices(); + wcout << L"DirectX device discovery library version " << GetDiscoveryLibraryVersion() << endl; + wcout << L"Discovered " << numDevices << L" devices.\n" << endl; + + // Print the details for each device + for (int device = 0; device < numDevices; ++device) + { + wcout << L"[Device " << device << L" details]\n\n"; + wcout << L"PnP Hardware ID: " << discovery.GetDeviceID(device) << L"\n"; + wcout << L"DX Adapter LUID: " << discovery.GetDeviceAdapterLUID(device) << L"\n"; + wcout << L"Description: " << discovery.GetDeviceDescription(device) << L"\n"; + wcout << L"Driver Registry Key: " << discovery.GetDeviceDriverRegistryKey(device) << L"\n"; + wcout << L"DriverStore Path: " << discovery.GetDeviceDriverStorePath(device) << L"\n"; + wcout << L"LocationPath: " << discovery.GetDeviceLocationPath(device) << L"\n"; + wcout << L"Vendor: " << discovery.GetDeviceVendor(device) << L"\n"; + wcout << L"Is Integrated: " << FormatBoolean(discovery.IsDeviceIntegrated(device)) << L"\n"; + wcout << L"Is Detachable: " << FormatBoolean(discovery.IsDeviceDetachable(device)) << L"\n"; + wcout << L"Supports Display: " << FormatBoolean(discovery.DoesDeviceSupportDisplay(device)) << L"\n"; + wcout << L"Supports Compute: " << FormatBoolean(discovery.DoesDeviceSupportCompute(device)) << L"\n"; + + int numRuntimeFiles = discovery.GetNumRuntimeFiles(device); + wcout << L"\n" << numRuntimeFiles << L" Additional System32 runtime files:\n"; + for (int file = 0; file < numRuntimeFiles; ++file) + { + wcout << L" " + << discovery.GetRuntimeFileSource(device, file) << " => " + << discovery.GetRuntimeFileDestination(device, file) << "\n"; + } + + int numRuntimeFilesWow64 = discovery.GetNumRuntimeFilesWow64(device); + wcout << L"\n" << numRuntimeFilesWow64 << L" Additional SysWOW64 runtime files:\n"; + for (int file = 0; file < numRuntimeFilesWow64; ++file) + { + wcout << L" " + << discovery.GetRuntimeFileSourceWow64(device, file) << L" => " + << discovery.GetRuntimeFileDestinationWow64(device, file) << L"\n"; + } + + wcout << endl; + } + } + catch (const DeviceDiscoveryException& err) { + wclog << L"Error: " << err.what() << endl; + } + + return 0; +} diff --git a/library/vcpkg.json b/library/vcpkg.json new file mode 100644 index 0000000..ac8335d --- /dev/null +++ b/library/vcpkg.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json", + "name": "directx-device-discovery", + "version": "0.0.1", + "dependencies": [ + "cppwinrt", + "fmt", + { + "name": "spdlog", + "features": ["wchar"] + }, + "wil" + ], + "supports": "windows & x64", + "builtin-baseline": "ca8bde3748134247a725c215c4f3a364ee9126fe" +} diff --git a/plugins/cmd/device-plugin-mcdm/main.go b/plugins/cmd/device-plugin-mcdm/main.go new file mode 100644 index 0000000..29aee81 --- /dev/null +++ b/plugins/cmd/device-plugin-mcdm/main.go @@ -0,0 +1,12 @@ +//go:build windows + +package main + +import ( + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "github.com/tensorworks/directx-device-plugins/plugins/internal/plugin" +) + +func main() { + plugin.CommonMain("mcdm", "directx.microsoft.com/compute", discovery.ComputeOnly) +} diff --git a/plugins/cmd/device-plugin-wddm/main.go b/plugins/cmd/device-plugin-wddm/main.go new file mode 100644 index 0000000..8ad9692 --- /dev/null +++ b/plugins/cmd/device-plugin-wddm/main.go @@ -0,0 +1,12 @@ +//go:build windows + +package main + +import ( + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "github.com/tensorworks/directx-device-plugins/plugins/internal/plugin" +) + +func main() { + plugin.CommonMain("wddm", "directx.microsoft.com/display", discovery.DisplayAndCompute) +} diff --git a/plugins/cmd/gen-device-mounts/main.go b/plugins/cmd/gen-device-mounts/main.go new file mode 100644 index 0000000..d0dcf84 --- /dev/null +++ b/plugins/cmd/gen-device-mounts/main.go @@ -0,0 +1,245 @@ +//go:build windows + +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "strings" + + "github.com/spf13/pflag" + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "github.com/tensorworks/directx-device-plugins/plugins/internal/mount" +) + +// Prints output to stderr +func ePrint(a ...any) (n int, err error) { + return fmt.Fprint(os.Stdout, a...) +} + +// Prints output to stderr, with spaces between operands and a trailing newline +func ePrintln(a ...any) (n int, err error) { + return fmt.Fprintln(os.Stdout, a...) +} + +// Determines whether the specified value exists in the supplied list of values +func contains[T comparable](values []T, value T) bool { + for _, existing := range values { + if existing == value { + return true + } + } + + return false +} + +// Formats a list of numeric values +func formatNumbers[T uint | int64](values []T) string { + formatted := []string{} + for _, value := range values { + formatted = append(formatted, fmt.Sprint(value)) + } + return strings.Join(formatted, ", ") +} + +// Formats a list of string values +func formatStrings(values []string, delimiter string) string { + formatted := []string{} + for _, str := range values { + formatted = append(formatted, fmt.Sprint("\"", str, "\"")) + } + return strings.Join(formatted, delimiter) +} + +func main() { + + // Configure our custom help message + pflag.CommandLine.SetOutput(os.Stderr) + pflag.Usage = func() { + ePrintln("gen-device-mounts: generates flags for exposing devices to containers with `ctr run`") + ePrintln("\nUsage syntax:", os.Args[0], "[-h] [--format text|json] [--all] [--index ] [--luid ] [--path ] [--run] [--verbose] []") + ePrintln("\nOptions:") + pflag.PrintDefaults() + ePrintln(strings.Join([]string{ + "", + "The list of available DirectX devices (including their enumeration indices, LUID values, and PCI paths)", + "can be retrieved by running either `test-device-discovery-cpp.exe` or `test-device-discovery-go.exe`.", + "", + "NOTES REGARDING OTHER FRONTENDS", + "-------------------------------", + "", + "Docker:", + "", + "Although Docker version 23.0.0 introduced support for exposing individial devices using their PCI location", + "paths, it still lacks the ability to bind-mount individual files rather than directories, which prevents", + "it from using the flags generated by `gen-device-mounts`. This is due to its continued use of the HCSv1", + "API rather than the newer HCSv2 API used by containerd. When Docker eventually migrates to using HCSv2", + "(or using containerd under Windows the way it does under Linux) then `gen-device-mounts` will be updated", + "to add an option to invoke `docker run` instead of `ctr run` when --run is specified.", + "", + "nerdctl:", + "", + "nerdctl is currently blocked by two outstanding issues that prevent it from using the flags generated by", + "`gen-device-mounts`:", + "", + "- https://github.com/containerd/nerdctl/pull/2079", + "- https://github.com/containerd/nerdctl/issues/759", + "", + "Once these blockers have been resolved then `gen-device-mounts` will be updated to add an option to invoke", + "`nerdctl run` instead of `ctr run` when --run is specified.", + }, "\n")) + os.Exit(1) + } + + // Parse our command-line arguments + allDevices := pflag.Bool("all", false, "Expose all available DirectX devices") + outputFormat := pflag.String("format", "text", "The output format for generated flags (\"text\" or \"json\")") + devicesByIndex := pflag.UintSlice("index", []uint{}, "Expose the DirectX device with the specified enumeration index (can be specified multiple times)") + devicesByLUID := pflag.Int64Slice("luid", []int64{}, "Expose the DirectX device with the specified LUID (can be specified multiple times)") + devicesByPath := pflag.StringSlice("path", []string{}, "Expose the DirectX device with the specified PCI path (can be specified multiple times)") + runContainer := pflag.Bool("run", false, "run") + verbose := pflag.Bool("verbose", false, "Enable verbose output") + pflag.Parse() + + // Verify that a valid output format was specified + if !contains([]string{"text", "json"}, *outputFormat) { + log.Fatalln("Error: unknown output format \"", *outputFormat, "\" (supported formats are \"text\" and \"json\")") + } + + // Print our device selection criteria + if *verbose { + ePrintln("Device selection criteria:") + if *allDevices { + ePrintln("- Include all available DirectX devices") + } else { + if len(*devicesByIndex) > 0 { + ePrintln("- Include DirectX devices with the following enumeration indices:", formatNumbers(*devicesByIndex)) + } + if len(*devicesByLUID) > 0 { + ePrintln("- Include DirectX devices with the following LUID values:", formatNumbers(*devicesByLUID)) + } + if len(*devicesByPath) > 0 { + ePrintln("- Include DirectX devices with the following PCI paths:", formatStrings(*devicesByPath, ", ")) + } + } + ePrintln() + } + + // Attempt to load the DirectX device discovery library + if err := discovery.LoadDiscoveryLibrary(); err != nil { + log.Fatalln("Error:", err) + } + + // Create a new DeviceDiscovery object + deviceDiscovery, err := discovery.NewDeviceDiscovery() + if err != nil { + log.Fatalln("Error:", err) + } + + // Perform device discovery + if err := deviceDiscovery.DiscoverDevices(discovery.AllDevices, true, true); err != nil { + log.Fatalln("Error:", err) + } + + // Filter the list of devices based on the specified selection criteria + filtered := []*discovery.Device{} + for index, device := range deviceDiscovery.Devices { + if *allDevices || contains(*devicesByIndex, uint(index)) || contains(*devicesByLUID, device.AdapterLUID) || contains(*devicesByPath, device.LocationPath) { + filtered = append(filtered, device) + } + } + + // Print the details of the selected devices + if *verbose { + ePrint("Selected ", len(filtered), " device(s) based on selection criteria:\n") + for index, device := range filtered { + ePrint("- Index ", index, ", LUID ", device.AdapterLUID, ", PCI Path ", device.LocationPath, "\n") + } + ePrintln() + } + + // Append our default runtime file mounts to the lists for each device + for _, device := range deviceDiscovery.Devices { + + // Determine whether we have any additional runtime files for the device vendor + files, haveFiles := mount.DefaultMounts[strings.ToLower(device.Vendor)] + filesWow64, haveFilesWow64 := mount.DefaultMountsWow64[strings.ToLower(device.Vendor)] + + // Merge any additions for System32 + if haveFiles { + ignored := device.AppendRuntimeFiles(files) + for _, file := range ignored { + ePrintln("Ignoring additional 64-bit runtime file because it clashes with an existing filename: ", file) + } + } + + // Merge any additions for SysWOW64 + if haveFilesWow64 { + ignored := device.AppendRuntimeFilesWow64(filesWow64) + for _, file := range ignored { + ePrintln("Ignoring additional 32-bit runtime file because it clashes with an existing filename: ", file) + } + } + } + + // Generate the device specs and runtime file mounts for the selected devices + specs := mount.SpecsForDevices(filtered) + mounts := mount.MountsForDevices(filtered) + + // Generate the flags for mounting the devices + flags := []string{} + for _, spec := range specs { + flags = append(flags, "--device", spec.HostPath) + } + for _, mount := range mounts { + flags = append(flags, "--mount", fmt.Sprint("src=", mount.HostPath, ",dst=", mount.GetContainerPath())) + } + + // Determine whether we are just printing the flags, or running a container with them + if *runContainer { + + // Create a command object to represent our `ctr run` invocation + cmd := exec.Command("ctr", "run", "--rm") + + // Allow the child process to inherit all standard streams + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + // Append both our generated flags and any loose command-line arguments to the invocation + cmd.Args = append(cmd.Args, flags...) + cmd.Args = append(cmd.Args, pflag.Args()...) + + // Print the generated command to stderr, wrapping each flag in quotes + ePrintln(formatStrings(cmd.Args, " ")) + + // Attempt to run `ctr run` + if err := cmd.Run(); err != nil { + log.Fatalln("Error:", err) + } + + } else { + + // Determine which format we are using to print the list of flags + if *outputFormat == "json" { + + // Attempt to format the flags as a JSON array + formatted, err := json.Marshal(flags) + if err != nil { + log.Fatalln("Error:", err) + } + + // Print the JSON array to stdout + fmt.Println(formatted) + + } else { + + // Print the list of flags to stdout, wrapping each flag in quotes + fmt.Println(formatStrings(flags, " ")) + + } + } +} diff --git a/plugins/cmd/query-hcs-capabilities/main.go b/plugins/cmd/query-hcs-capabilities/main.go new file mode 100644 index 0000000..07315d6 --- /dev/null +++ b/plugins/cmd/query-hcs-capabilities/main.go @@ -0,0 +1,150 @@ +//go:build windows + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os/exec" + "strings" + "unsafe" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" +) + +var ( + vmcompute = windows.NewLazyDLL("vmcompute.dll") + hcsGetServiceProperties = vmcompute.NewProc("HcsGetServiceProperties") +) + +// HCS schema Version structure: +type Version struct { + Major uint32 + Minor uint32 +} + +// HCS schema BasicInformation structure: +type BasicInformation struct { + SupportedSchemaVersions []*Version +} + +// Modified version of the HCS schema ServiceProperties structure: +// Note that we just treat the array as containing BasicInformation objects, since that's what our specific query returns +type ServiceProperties struct { + Properties []*BasicInformation +} + +func getSupportedSchemas() ([]*Version, error) { + + // Attempt to load vmcompute.dll + if err := vmcompute.Load(); err != nil { + return nil, fmt.Errorf("failed to load %s: %s", vmcompute.Name, err.Error()) + } + + // Convert our query string into a UTF-16 pointer + queryPtr, err := windows.UTF16PtrFromString("{\"PropertyTypes\": [\"Basic\"]}") + if err != nil { + return nil, fmt.Errorf("failed to convert string to UTF-16: %s", err.Error()) + } + + // Call HcsGetServiceProperties() to query the supported schema version + var resultPtr *uint16 = nil + retval, _, _ := hcsGetServiceProperties.Call( + uintptr(unsafe.Pointer(queryPtr)), + uintptr(unsafe.Pointer(&resultPtr)), + ) + + // Verify that the query was successful + if retval != 0 { + return nil, fmt.Errorf("HcsGetServiceProperties() failed: %v", windows.Errno(retval)) + } + + // Convert the result into a JSON string + result := windows.UTF16PtrToString((*uint16)(unsafe.Pointer(resultPtr))) + + // Parse the JSON + serviceProperties := &ServiceProperties{} + if err := json.Unmarshal([]byte(result), &serviceProperties); err != nil { + return nil, err + } + + // Verify that we have at least one supported schema version + if len(serviceProperties.Properties) == 0 || len(serviceProperties.Properties[0].SupportedSchemaVersions) == 0 { + return nil, errors.New("HcsGetServiceProperties() returned zero supported schema versions") + } + + // Return the list of supported schema versions + return serviceProperties.Properties[0].SupportedSchemaVersions, nil +} + +func getWindowsVersion() (string, error) { + + // Use `RtlGetVersion()` to query the Windows version number, so manifest semantics are ignored + versionInfo := windows.RtlGetVersion() + + // Use PowerShell to query WMI for the system caption, since `ProductName` in the registry is no longer reliable + productName, err := exec.Command("powershell", "-Command", "(Get-WmiObject -Class Win32_OperatingSystem).Caption").Output() + if err != nil { + return "", fmt.Errorf("failed to query WMI for the product version string: %s", err.Error()) + } + + // Open the registry key for the Windows version information + key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) + if err != nil { + return "", fmt.Errorf("failed to open the registry key \"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\": %s", err.Error()) + } + defer key.Close() + + // Attempt to retrieve the display version on newer systems, falling back to the old release ID value on older systems + displayVersion, _, err := key.GetStringValue("DisplayVersion") + if err != nil { + displayVersion, _, err = key.GetStringValue("ReleaseId") + if err != nil { + return "", fmt.Errorf("failed to retrieve either the \"DisplayVersion\" or \"ReleaseId\" registry value: %s", err.Error()) + } + } + + // Retrieve the revision number, since this isn't included in the `RtlGetVersion()` output + revisionNumber, _, err := key.GetIntegerValue("UBR") + if err != nil { + return "", fmt.Errorf("failed to retrieve the \"UBR\" registry value: %s", err.Error()) + } + + // Build an aggregated version string from the retrieved values + return fmt.Sprintf( + "%s, version %s (OS build %d.%d.%d.%d)", + strings.TrimSpace(string(productName)), + displayVersion, + versionInfo.MajorVersion, + versionInfo.MinorVersion, + versionInfo.BuildNumber, + revisionNumber, + ), nil +} + +func main() { + + // Retrieve the Windows version information + windowsVersion, err := getWindowsVersion() + if err != nil { + log.Fatalf("Failed to retrieve Windows version information: %s", err.Error()) + } + + // Query the Host Compute Service (HCS) for the list of supported schema versions + supportedSchemas, err := getSupportedSchemas() + if err != nil { + log.Fatalf("Failed to retrieve the supported HCS schema version: %s", err.Error()) + } + + // Print the Windows version details and supported schema version + fmt.Println("Operating system version:") + fmt.Println(windowsVersion) + fmt.Println() + fmt.Println("Supported HCS schema versions:") + for _, version := range supportedSchemas { + fmt.Printf("- %d.%d", version.Major, version.Minor) + } +} diff --git a/plugins/cmd/test-device-discovery-go/main.go b/plugins/cmd/test-device-discovery-go/main.go new file mode 100644 index 0000000..fb17d0e --- /dev/null +++ b/plugins/cmd/test-device-discovery-go/main.go @@ -0,0 +1,71 @@ +//go:build windows + +package main + +import ( + "flag" + "fmt" + "log" + + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" +) + +func main() { + + // Parse our command-line arguments + verbose := flag.Bool("verbose", false, "enable verbose logging") + flag.Parse() + + // Attempt to load the DirectX device discovery library + if err := discovery.LoadDiscoveryLibrary(); err != nil { + log.Fatalln("Error:", err) + } + + // Enable verbose logging for the device discovery library if it has been requested + if *verbose { + discovery.EnableDiscoveryLogging() + } + + // Create a new DeviceDiscovery object + deviceDiscovery, err := discovery.NewDeviceDiscovery() + if err != nil { + log.Fatalln("Error:", err) + } + + // Perform device discovery + if err := deviceDiscovery.DiscoverDevices(discovery.AllDevices, true, true); err != nil { + log.Fatalln("Error:", err) + } + + // Print the library version string and the number of discovered devices + fmt.Print("DirectX device discovery library version ", discovery.GetDiscoveryLibraryVersion(), "\n") + fmt.Print("Discovered ", len(deviceDiscovery.Devices), " devices.\n\n") + + // Print the details for each device + for index, device := range deviceDiscovery.Devices { + fmt.Print("[Device ", index, " details]\n\n") + fmt.Println("PnP Hardware ID: ", device.ID) + fmt.Println("DX Adapter LUID: ", device.AdapterLUID) + fmt.Println("Description: ", device.Description) + fmt.Println("Driver Registry Key:", device.DriverRegistryKey) + fmt.Println("DriverStore Path: ", device.DriverStorePath) + fmt.Println("LocationPath: ", device.LocationPath) + fmt.Println("Vendor: ", device.Vendor) + fmt.Println("Is Integrated: ", device.IsIntegrated) + fmt.Println("Is Detachable: ", device.IsDetachable) + fmt.Println("Supports Display: ", device.SupportsDisplay) + fmt.Println("Supports Compute: ", device.SupportsCompute) + + fmt.Print("\n", len(device.RuntimeFiles), " Additional System32 runtime files:\n") + for _, file := range device.RuntimeFiles { + fmt.Println(" ", file.SourcePath, "=>", file.DestinationFilename) + } + + fmt.Print("\n", len(device.RuntimeFilesWow64), " Additional SysWOW64 runtime files:\n") + for _, file := range device.RuntimeFilesWow64 { + fmt.Println(" ", file.SourcePath, "=>", file.DestinationFilename) + } + + fmt.Print("\n") + } +} diff --git a/plugins/go.mod b/plugins/go.mod new file mode 100644 index 0000000..ff03917 --- /dev/null +++ b/plugins/go.mod @@ -0,0 +1,37 @@ +module github.com/tensorworks/directx-device-plugins/plugins + +go 1.18 + +require ( + github.com/fsnotify/fsnotify v1.5.4 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.12.0 + go.uber.org/zap v1.19.0 + golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + google.golang.org/grpc v1.46.2 + k8s.io/kubelet v0.24.1 +) + +require ( + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.6 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/spf13/afero v1.8.2 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/subosito/gotenv v1.3.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/ini.v1 v1.66.4 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect +) diff --git a/plugins/go.sum b/plugins/go.sum new file mode 100644 index 0000000..74ffbc2 --- /dev/null +++ b/plugins/go.sum @@ -0,0 +1,798 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= +github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= +github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= +golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= +gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.24.1/go.mod h1:JhoOvNiLXKTPQ60zh2g0ewpA+bnEYf5q44Flhquh4vQ= +k8s.io/apimachinery v0.24.1/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/client-go v0.24.1/go.mod h1:f1kIDqcEYmwXS/vTbbhopMUbhKp2JhOeVTfxgaCIlF8= +k8s.io/component-base v0.24.1/go.mod h1:DW5vQGYVCog8WYpNob3PMmmsY8A3L9QZNg4j/dV3s38= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/kubelet v0.24.1 h1:CLgXZ9kKDQoNQFSwKk6vUE5gXNaX1/s8VM8Oq/P5S+o= +k8s.io/kubelet v0.24.1/go.mod h1:LShXfjNO1or7ktsorODSOu8+Kd5dHzWF3DtVLXeP3JE= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/plugins/internal/discovery/device.go b/plugins/internal/discovery/device.go new file mode 100644 index 0000000..4f9844d --- /dev/null +++ b/plugins/internal/discovery/device.go @@ -0,0 +1,85 @@ +//go:build windows + +package discovery + +// Represents a DirectX device +type Device struct { + + // The unique PNP hardware identifier for the device + ID string + + // A human-readable description of the device (e.g. the model name) + Description string + + // The registry key that contains the driver details for the device + DriverRegistryKey string + + // The absolute path to the directory in the driver store that contains the driver files for the device + DriverStorePath string + + // The path to the physical location of the device in the system + LocationPath string + + // The list of additional files that need to be copied from the driver store to the System32 directory in order to use the device with non-DirectX runtimes + RuntimeFiles []*RuntimeFile + + // The list of additional files that need to be copied from the driver store to the SysWOW64 directory in order to use the device with non-DirectX runtimes + RuntimeFilesWow64 []*RuntimeFile + + // The vendor of the device (e.g. AMD, Intel, NVIDIA) + Vendor string + + // The DirectX adapter LUID associated with the PnP device + AdapterLUID int64 + + // Specifies whether the device is an integrated GPU (as opposed to a discrete GPU) + IsIntegrated bool + + // Specifies whether the device is a detachable device (i.e. the device can be removed at runtime) + IsDetachable bool + + // Specifies whether the device supports display + // (i.e. supports either the DXCORE_ADAPTER_ATTRIBUTE_D3D11_GRAPHICS or DXCORE_ADAPTER_ATTRIBUTE_D3D12_GRAPHICS attributes) + SupportsDisplay bool + + // Specifies whether the device supports compute (i.e. supports the DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE attribute) + SupportsCompute bool +} + +// Appends the supplied list of 64-bit runtime files, ignoring and returning any files that clash with existing mount destinations +func (d *Device) AppendRuntimeFiles(files []*RuntimeFile) []*RuntimeFile { + merged, ignored := mergeRuntimeFiles(d.RuntimeFiles, files) + d.RuntimeFiles = merged + return ignored +} + +// Appends the supplied list of 32-bit runtime files, ignoring and returning any files that clash with existing mount destinations +func (d *Device) AppendRuntimeFilesWow64(files []*RuntimeFile) []*RuntimeFile { + merged, ignored := mergeRuntimeFiles(d.RuntimeFilesWow64, files) + d.RuntimeFilesWow64 = merged + return ignored +} + +// Merges two lists of runtime files, ignoring any files that clash with existing mount destinations. Returns both the merged list and the list of ignored files. +func mergeRuntimeFiles(files []*RuntimeFile, additions []*RuntimeFile) ([]*RuntimeFile, []*RuntimeFile) { + merged := files + ignored := []*RuntimeFile{} + + // Add each additional file to the list if it doesn't clash with an existing destination filename +outer: + for _, additionalFile := range additions { + + // Determine whether we have an existing file with the same destination as the new file + for _, existingFile := range merged { + if existingFile.DestinationFilename == additionalFile.DestinationFilename { + ignored = append(ignored, additionalFile) + continue outer + } + } + + // Add the file to the list + merged = append(merged, additionalFile) + } + + return merged, ignored +} diff --git a/plugins/internal/discovery/device_discovery.go b/plugins/internal/discovery/device_discovery.go new file mode 100644 index 0000000..810d5e2 --- /dev/null +++ b/plugins/internal/discovery/device_discovery.go @@ -0,0 +1,466 @@ +//go:build windows + +package discovery + +import ( + "errors" + "fmt" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + discoverydll = windows.NewLazyDLL("directx-device-discovery.dll") + procGetDiscoveryLibraryVersion = discoverydll.NewProc("GetDiscoveryLibraryVersion") + procDisableDiscoveryLogging = discoverydll.NewProc("DisableDiscoveryLogging") + procEnableDiscoveryLogging = discoverydll.NewProc("EnableDiscoveryLogging") + procCreateDeviceDiscoveryInstance = discoverydll.NewProc("CreateDeviceDiscoveryInstance") + procDestroyDeviceDiscoveryInstance = discoverydll.NewProc("DestroyDeviceDiscoveryInstance") + procGetLastErrorMessage = discoverydll.NewProc("DeviceDiscovery_GetLastErrorMessage") + procIsRefreshRequired = discoverydll.NewProc("DeviceDiscovery_IsRefreshRequired") + procDiscoverDevices = discoverydll.NewProc("DeviceDiscovery_DiscoverDevices") + procGetNumDevices = discoverydll.NewProc("DeviceDiscovery_GetNumDevices") + procGetDeviceAdapterLUID = discoverydll.NewProc("DeviceDiscovery_GetDeviceAdapterLUID") + procGetDeviceID = discoverydll.NewProc("DeviceDiscovery_GetDeviceID") + procGetDeviceDescription = discoverydll.NewProc("DeviceDiscovery_GetDeviceDescription") + procGetDeviceDriverRegistryKey = discoverydll.NewProc("DeviceDiscovery_GetDeviceDriverRegistryKey") + procGetDeviceDriverStorePath = discoverydll.NewProc("DeviceDiscovery_GetDeviceDriverStorePath") + procGetDeviceLocationPath = discoverydll.NewProc("DeviceDiscovery_GetDeviceLocationPath") + procGetDeviceVendor = discoverydll.NewProc("DeviceDiscovery_GetDeviceVendor") + procGetNumRuntimeFiles = discoverydll.NewProc("DeviceDiscovery_GetNumRuntimeFiles") + procGetRuntimeFileSource = discoverydll.NewProc("DeviceDiscovery_GetRuntimeFileSource") + procGetRuntimeFileDestination = discoverydll.NewProc("DeviceDiscovery_GetRuntimeFileDestination") + procGetNumRuntimeFilesWow64 = discoverydll.NewProc("DeviceDiscovery_GetNumRuntimeFilesWow64") + procGetRuntimeFileSourceWow64 = discoverydll.NewProc("DeviceDiscovery_GetRuntimeFileSourceWow64") + procGetRuntimeFileDestinationWow64 = discoverydll.NewProc("DeviceDiscovery_GetRuntimeFileDestinationWow64") + procIsDeviceIntegrated = discoverydll.NewProc("DeviceDiscovery_IsDeviceIntegrated") + procIsDeviceDetachable = discoverydll.NewProc("DeviceDiscovery_IsDeviceDetachable") + procDoesDeviceSupportDisplay = discoverydll.NewProc("DeviceDiscovery_DoesDeviceSupportDisplay") + procDoesDeviceSupportCompute = discoverydll.NewProc("DeviceDiscovery_DoesDeviceSupportCompute") +) + +type DeviceDiscovery struct { + + // The handle to the underlying DeviceDiscovery object + handle uintptr + + // The list of discovered devices + Devices []*Device +} + +// Attempts to load the DirectX device discovery library and returns an error if loading fails +// (This is useful for catching failures gracefully rather than triggering a panic upon lazy-load) +func LoadDiscoveryLibrary() error { + + if err := discoverydll.Load(); err != nil { + return fmt.Errorf("failed to load %s: %s", discoverydll.Name, err.Error()) + } + + return nil +} + +// Wrapper function for GetDiscoveryLibraryVersion +func GetDiscoveryLibraryVersion() string { + result, _, _ := procGetDiscoveryLibraryVersion.Call() + return windows.UTF16PtrToString((*uint16)(unsafe.Pointer(result))) +} + +// Wrapper function for DisableDiscoveryLogging +func DisableDiscoveryLogging() { + procDisableDiscoveryLogging.Call() +} + +// Wrapper function for EnableDiscoveryLogging +func EnableDiscoveryLogging() { + procEnableDiscoveryLogging.Call() +} + +func NewDeviceDiscovery() (*DeviceDiscovery, error) { + + // Attempt to create a DeviceDiscovery instance + result, _, _ := procCreateDeviceDiscoveryInstance.Call() + if result == 0 { + return nil, errors.New("failed to create the DeviceDiscovery instance") + } + + return &DeviceDiscovery{ + handle: result, + Devices: []*Device{}, + }, nil +} + +func (d *DeviceDiscovery) Destroy() { + procDestroyDeviceDiscoveryInstance.Call(d.handle) +} + +// Formats a boolean argument for passing to a library function +func (d *DeviceDiscovery) booleanArgument(arg bool) uintptr { + if arg { + return 1 + } else { + return 0 + } +} + +// Handles the result of a library function that returns a boolean +func (d *DeviceDiscovery) handleBooleanResult(result uintptr, r2 uintptr, lastError error) (bool, error) { + if int32(result) == -1 { + return false, d.getLastErrorMessage() + } + + return result == 1, nil +} + +// Handles the result of a library function that returns an unsigned 32-bit integer +func (d *DeviceDiscovery) handleUint32Result(result uintptr, r2 uintptr, lastError error) (uint32, error) { + if int32(result) == -1 { + return 0, d.getLastErrorMessage() + } + + return uint32(result), nil +} + +// Handles the result of a library function that returns a signed 64-bit integer +func (d *DeviceDiscovery) handleInt64Result(result uintptr, r2 uintptr, lastError error) (int64, error) { + if int64(result) == -1 { + return 0, d.getLastErrorMessage() + } + + return int64(result), nil +} + +// Handles the result of a library function that returns a UTF-16 string +func (d *DeviceDiscovery) handleStringResult(result uintptr, r2 uintptr, lastError error) (string, error) { + if result == 0 { + return "", d.getLastErrorMessage() + } + + return windows.UTF16PtrToString((*uint16)(unsafe.Pointer(result))), nil +} + +// Retrieves the error message for the last library function call +func (d *DeviceDiscovery) getLastErrorMessage() error { + + // Retrieve the last error message from the library and convert it to a Go error + result, _, _ := procGetLastErrorMessage.Call(d.handle) + errorMessage := windows.UTF16PtrToString((*uint16)(unsafe.Pointer(result))) + + if errorMessage != "" { + return errors.New(errorMessage) + } + + return nil +} + +// Performs device discovery and populates our list of devices +func (d *DeviceDiscovery) DiscoverDevices(filter DeviceFilter, includeIntegrated bool, includeDetachable bool) error { + + // Attempt to perform device discovery + result, _, _ := procDiscoverDevices.Call(d.handle, uintptr(filter), d.booleanArgument(includeIntegrated), d.booleanArgument(includeDetachable)) + if int32(result) == -1 { + return d.getLastErrorMessage() + } + + // Determine the number of discovered devices + numDevices, err := d.getNumDevices() + if err != nil { + return err + } + + // Clear the existing list of devices + d.Devices = []*Device{} + + // Retrieve the details of each device in turn + for index := 0; index < int(numDevices); index += 1 { + + // Attempt to retrieve the device details + device, err := d.getDevice(index) + if err != nil { + return err + } + + // Add the device to our list + d.Devices = append(d.Devices, device) + } + + return nil +} + +// Retrieves the details for an individual device +func (d *DeviceDiscovery) getDevice(device int) (*Device, error) { + + // Attempt to retrieve the device ID + id, err := d.getDeviceID(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the device description + description, err := d.getDeviceDescription(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the device driver registry key + registry, err := d.getDeviceDriverRegistryKey(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the device driver store path + driverStore, err := d.getDeviceDriverStorePath(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the device location path + location, err := d.getDeviceLocationPath(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the device vendor + vendor, err := d.getDeviceVendor(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the device adapter LUID + luid, err := d.getDeviceAdapterLUID(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the integrated device hardware flag + integrated, err := d.isDeviceIntegrated(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the detachable device hardware flag + detachable, err := d.isDeviceDetachable(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the display support flag + display, err := d.doesDeviceSupportDisplay(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the compute support flag + compute, err := d.doesDeviceSupportCompute(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the number of additional runtime files for System32 + numRuntimeFiles, err := d.getNumRuntimeFiles(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the list of additional runtime files for System32 + runtimeFiles := []*RuntimeFile{} + for file := 0; file < int(numRuntimeFiles); file += 1 { + + // Attempt to retrieve the source path for the file + sourcePath, err := d.getRuntimeFileSource(device, file) + if err != nil { + return nil, err + } + + // Attempt to retrieve the destination filename for the file + destinationFilename, err := d.getRuntimeFileDestination(device, file) + if err != nil { + return nil, err + } + + // Add the file details to the list + runtimeFiles = append(runtimeFiles, &RuntimeFile{ + SourcePath: sourcePath, + DestinationFilename: destinationFilename, + }) + } + + // Attempt to retrieve the number of additional runtime files for SysWOW64 + numRuntimeFilesWow64, err := d.getNumRuntimeFilesWow64(device) + if err != nil { + return nil, err + } + + // Attempt to retrieve the list of additional runtime files for System32 + runtimeFilesWow64 := []*RuntimeFile{} + for file := 0; file < int(numRuntimeFilesWow64); file += 1 { + + // Attempt to retrieve the source path for the file + sourcePath, err := d.getRuntimeFileSourceWow64(device, file) + if err != nil { + return nil, err + } + + // Attempt to retrieve the destination filename for the file + destinationFilename, err := d.getRuntimeFileDestinationWow64(device, file) + if err != nil { + return nil, err + } + + // Add the file details to the list + runtimeFilesWow64 = append(runtimeFilesWow64, &RuntimeFile{ + SourcePath: sourcePath, + DestinationFilename: destinationFilename, + }) + } + + // Construct a Device object from the retrieved data + return &Device{ + ID: id, + Description: description, + DriverRegistryKey: registry, + DriverStorePath: driverStore, + LocationPath: location, + RuntimeFiles: runtimeFiles, + RuntimeFilesWow64: runtimeFilesWow64, + Vendor: vendor, + AdapterLUID: luid, + IsIntegrated: integrated, + IsDetachable: detachable, + SupportsDisplay: display, + SupportsCompute: compute, + }, nil +} + +// Wrapper function for DeviceDiscovery_IsRefreshRequired +func (d *DeviceDiscovery) IsRefreshRequired() (bool, error) { + return d.handleBooleanResult( + procIsRefreshRequired.Call(d.handle), + ) +} + +// Wrapper function for DeviceDiscovery_GetNumDevices +func (d *DeviceDiscovery) getNumDevices() (uint32, error) { + return d.handleUint32Result( + procGetNumDevices.Call(d.handle), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceAdapterLUID +func (d *DeviceDiscovery) getDeviceAdapterLUID(device int) (int64, error) { + return d.handleInt64Result( + procGetDeviceAdapterLUID.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceID +func (d *DeviceDiscovery) getDeviceID(device int) (string, error) { + return d.handleStringResult( + procGetDeviceID.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceDescription +func (d *DeviceDiscovery) getDeviceDescription(device int) (string, error) { + return d.handleStringResult( + procGetDeviceDescription.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceDriverRegistryKey +func (d *DeviceDiscovery) getDeviceDriverRegistryKey(device int) (string, error) { + return d.handleStringResult( + procGetDeviceDriverRegistryKey.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceDriverStorePath +func (d *DeviceDiscovery) getDeviceDriverStorePath(device int) (string, error) { + return d.handleStringResult( + procGetDeviceDriverStorePath.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceLocationPath +func (d *DeviceDiscovery) getDeviceLocationPath(device int) (string, error) { + return d.handleStringResult( + procGetDeviceLocationPath.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetDeviceVendor +func (d *DeviceDiscovery) getDeviceVendor(device int) (string, error) { + return d.handleStringResult( + procGetDeviceVendor.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetNumRuntimeFiles +func (d *DeviceDiscovery) getNumRuntimeFiles(device int) (uint32, error) { + return d.handleUint32Result( + procGetNumRuntimeFiles.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetRuntimeFileSource +func (d *DeviceDiscovery) getRuntimeFileSource(device int, file int) (string, error) { + return d.handleStringResult( + procGetRuntimeFileSource.Call(d.handle, uintptr(device), uintptr(file)), + ) +} + +// Wrapper function for DeviceDiscovery_GetRuntimeFileDestination +func (d *DeviceDiscovery) getRuntimeFileDestination(device int, file int) (string, error) { + return d.handleStringResult( + procGetRuntimeFileDestination.Call(d.handle, uintptr(device), uintptr(file)), + ) +} + +// Wrapper function for DeviceDiscovery_GetNumRuntimeFilesWow64 +func (d *DeviceDiscovery) getNumRuntimeFilesWow64(device int) (uint32, error) { + return d.handleUint32Result( + procGetNumRuntimeFilesWow64.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_GetRuntimeFileSourceWow64 +func (d *DeviceDiscovery) getRuntimeFileSourceWow64(device int, file int) (string, error) { + return d.handleStringResult( + procGetRuntimeFileSourceWow64.Call(d.handle, uintptr(device), uintptr(file)), + ) +} + +// Wrapper function for DeviceDiscovery_GetRuntimeFileDestinationWow64 +func (d *DeviceDiscovery) getRuntimeFileDestinationWow64(device int, file int) (string, error) { + return d.handleStringResult( + procGetRuntimeFileDestinationWow64.Call(d.handle, uintptr(device), uintptr(file)), + ) +} + +// Wrapper function for DeviceDiscovery_IsDeviceIntegrated +func (d *DeviceDiscovery) isDeviceIntegrated(device int) (bool, error) { + return d.handleBooleanResult( + procIsDeviceIntegrated.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_IsDeviceDetachable +func (d *DeviceDiscovery) isDeviceDetachable(device int) (bool, error) { + return d.handleBooleanResult( + procIsDeviceDetachable.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_DoesDeviceSupportDisplay +func (d *DeviceDiscovery) doesDeviceSupportDisplay(device int) (bool, error) { + return d.handleBooleanResult( + procDoesDeviceSupportDisplay.Call(d.handle, uintptr(device)), + ) +} + +// Wrapper function for DeviceDiscovery_DoesDeviceSupportCompute +func (d *DeviceDiscovery) doesDeviceSupportCompute(device int) (bool, error) { + return d.handleBooleanResult( + procDoesDeviceSupportCompute.Call(d.handle, uintptr(device)), + ) +} diff --git a/plugins/internal/discovery/device_filter.go b/plugins/internal/discovery/device_filter.go new file mode 100644 index 0000000..bcf35ad --- /dev/null +++ b/plugins/internal/discovery/device_filter.go @@ -0,0 +1,14 @@ +//go:build windows + +package discovery + +type DeviceFilter int32 + +const ( + AllDevices DeviceFilter = 0 + DisplaySupported DeviceFilter = 1 + ComputeSupported DeviceFilter = 2 + DisplayOnly DeviceFilter = 3 + ComputeOnly DeviceFilter = 4 + DisplayAndCompute DeviceFilter = 5 +) diff --git a/plugins/internal/discovery/runtime_file.go b/plugins/internal/discovery/runtime_file.go new file mode 100644 index 0000000..39f3a46 --- /dev/null +++ b/plugins/internal/discovery/runtime_file.go @@ -0,0 +1,14 @@ +//go:build windows + +package discovery + +// Represents an additional file that needs to be copied from the driver store to the system directory in order to use a device with non-DirectX runtimes +// (For details, see: ) +type RuntimeFile struct { + + // The relative path to the file in the driver store + SourcePath string `mapstructure:"source"` + + // The filename that the file should be given when copied to the destination directory + DestinationFilename string `mapstructure:"destination"` +} diff --git a/plugins/internal/mount/default_mounts.go b/plugins/internal/mount/default_mounts.go new file mode 100644 index 0000000..f88998e --- /dev/null +++ b/plugins/internal/mount/default_mounts.go @@ -0,0 +1,31 @@ +//go:build windows + +package mount + +import ( + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" +) + +// The default 64-bit runtime mounts that we add for each device vendor, to supplement those specified in the registry +var DefaultMounts = map[string][]*discovery.RuntimeFile{ + VendorNvidia: { + { + SourcePath: "nvidia-smi.exe", + DestinationFilename: "nvidia-smi.exe", + }, + { + SourcePath: "vulkaninfo-x64.exe", + DestinationFilename: "vulkaninfo.exe", + }, + }, +} + +// The default 32-bit runtime mounts that we add for each device vendor, to supplement those specified in the registry +var DefaultMountsWow64 = map[string][]*discovery.RuntimeFile{ + VendorNvidia: { + { + SourcePath: "vulkaninfo-x86.exe", + DestinationFilename: "vulkaninfo.exe", + }, + }, +} diff --git a/plugins/internal/mount/device_mounts.go b/plugins/internal/mount/device_mounts.go new file mode 100644 index 0000000..f0bb713 --- /dev/null +++ b/plugins/internal/mount/device_mounts.go @@ -0,0 +1,91 @@ +//go:build windows + +package mount + +import ( + "os" + "path/filepath" + + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" +) + +// Generates the device specs for the supplied list of devices +func SpecsForDevices(devices []*discovery.Device) []*pluginapi.DeviceSpec { + + specs := []*pluginapi.DeviceSpec{} + + // Provide the physical location path for each device, avoiding duplicates (duplicate paths can occur when + // multitenancy is enabled and two requested device IDs map to the same underlying physical device) + for _, device := range devices { + specs = appendUniqueSpec(specs, &pluginapi.DeviceSpec{ + HostPath: "vpci-location-path://" + device.LocationPath, + ContainerPath: "", + Permissions: "", + }) + } + + return specs +} + +// Generates the runtime file mounts for the supplied list of devices +func MountsForDevices(devices []*discovery.Device) []*pluginapi.Mount { + + mounts := []*pluginapi.Mount{} + + for _, device := range devices { + + // Generates the mounts for a list of runtime files + generateMounts := func(files []*discovery.RuntimeFile, destinationRoot string) { + for _, file := range files { + + // Resolve the absolute paths to the host source file and the container destination file + source := filepath.Join(device.DriverStorePath, file.SourcePath) + destination := filepath.Join(destinationRoot, file.DestinationFilename) + + // Only mount the file if it exists on the host and can be accessed, and isn't a duplicate + // (Note that duplicate container paths can occur not only when mounting multiple devices + // from a single vendor, but also when device drivers from different vendors mount files + // to the same target path, which means that a container will only see the files from the + // first device's vendor when collisions occur between different device drivers) + _, err := os.Stat(source) + if err == nil { + mounts = appendUniqueMount(mounts, &pluginapi.Mount{ + HostPath: source, + ContainerPath: destination, + ReadOnly: true, + }) + } + } + } + + // Generate the mounts for both the System32 and SysWOW64 runtime files + generateMounts(device.RuntimeFiles, "C:\\Windows\\System32") + generateMounts(device.RuntimeFilesWow64, "C:\\Windows\\SysWOW64") + } + + return mounts +} + +// Appends a device spec to an existing list of device specs if it's not already present in the list +func appendUniqueSpec(specs []*pluginapi.DeviceSpec, newSpec *pluginapi.DeviceSpec) []*pluginapi.DeviceSpec { + for _, existing := range specs { + if existing.HostPath == newSpec.HostPath { + return specs + } + } + + return append(specs, newSpec) +} + +// Appends a mount to an existing list of mounts if it's not already present in the list +func appendUniqueMount(mounts []*pluginapi.Mount, newMount *pluginapi.Mount) []*pluginapi.Mount { + for _, existing := range mounts { + if existing.ContainerPath == newMount.ContainerPath { + return mounts + } + } + + return append(mounts, newMount) +} diff --git a/plugins/internal/mount/vendors.go b/plugins/internal/mount/vendors.go new file mode 100644 index 0000000..d498659 --- /dev/null +++ b/plugins/internal/mount/vendors.go @@ -0,0 +1,6 @@ +//go:build windows + +package mount + +// Vendor identifier for NVIDIA devices +const VendorNvidia = "nvidia" diff --git a/plugins/internal/plugin/common_main.go b/plugins/internal/plugin/common_main.go new file mode 100644 index 0000000..7b7c6ff --- /dev/null +++ b/plugins/internal/plugin/common_main.go @@ -0,0 +1,83 @@ +//go:build windows + +package plugin + +import ( + "log" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "go.uber.org/zap" +) + +// The version number for the plugin +const version = "0.0.1" + +func CommonMain(pluginName string, resourceName string, filter discovery.DeviceFilter) { + + // Create a logger that prints debug and higher verbosity level messages + logger, err := zap.NewDevelopment() + if err != nil { + log.Fatalln("Error: failed to create the logger:", err) + } + + // Sugar the logger + sugar := logger.Sugar() + defer sugar.Sync() + + // Log the plugin name and version + sugar.Infof("Kubernetes device plugin for %s, version %s", strings.ToUpper(pluginName), version) + + // Load the plugin configuration data + config, err := LoadConfig(pluginName, sugar) + if err != nil { + sugar.Errorf("Error: failed to load the device plugin configuration: %v", err) + return + } + + // Create the device plugin and start the device watcher + server, err := NewDevicePlugin(pluginName, version, resourceName, filter, config, sugar) + if err != nil { + sugar.Errorf("Error: failed to create the device plugin: %v", err) + return + } + + //Ensure the plugin is destroyed and the device watcher stopped when we complete execution + defer server.Destroy() + + // Attempt to start the plugin's gRPC server + if err := server.StartServer(); err != nil { + sugar.Errorf("Error: failed to start the gRPC server: %v", err) + return + } + + // Ensure we perform a graceful shutdown of the gRPC server before we destroy the plugin + defer server.StopServer() + + // Attempt to register the device plugin with the Kubelet + if err := server.RegisterWithKubelet(); err != nil { + sugar.Errorf("Error: failed to register the device plugin with the Kubelet: %v", err) + return + } + + // Wire up a signal handler to receive shutdown requests + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + // Serve requests until we receive a shutdown request or an error occurs + sugar.Info("Serving until a shutdown request is received") + for { + select { + case sig := <-signals: + sugar.Infow("Received signal", "signal", sig) + return + + case err := <-server.Errors: + sugar.Errorf("Error: %v", err) + return + } + } +} diff --git a/plugins/internal/plugin/deletion_watcher.go b/plugins/internal/plugin/deletion_watcher.go new file mode 100644 index 0000000..88e616c --- /dev/null +++ b/plugins/internal/plugin/deletion_watcher.go @@ -0,0 +1,77 @@ +//go:build windows + +package plugin + +import ( + "github.com/fsnotify/fsnotify" +) + +type DeletionWatcher struct { + watcher *fsnotify.Watcher + Deleted chan struct{} + Errors chan error +} + +func WatchForDeletion(file string) (*DeletionWatcher, error) { + + // Create a new filesystem watcher + fsWatcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + // Add a watch for the specified file + err = fsWatcher.Add(file) + if err != nil { + return nil, err + } + + // Wrap the filesystem watcher in a deletion watcher + deletionWatcher := &DeletionWatcher{ + watcher: fsWatcher, + Deleted: make(chan struct{}, 1), + Errors: make(chan error, 1), + } + + // Start the watcher goroutine + go deletionWatcher.watch() + return deletionWatcher, nil +} + +// Cancels the watch +func (d *DeletionWatcher) Cancel() { + d.watcher.Close() +} + +func (d *DeletionWatcher) watch() { + + // Ensure the underlying filesystem watcher is closed when we are done + defer d.watcher.Close() + + // Ensure the channels are closed when we are done + defer close(d.Deleted) + defer close(d.Errors) + + // Process events and errors + for { + select { + case event, ok := <-d.watcher.Events: + if !ok { + return + } + + // Check whether the event is a deletion event + if event.Op&fsnotify.Remove == fsnotify.Remove { + d.Deleted <- struct{}{} + return + } + + case err, ok := <-d.watcher.Errors: + if !ok { + return + } + + d.Errors <- err + } + } +} diff --git a/plugins/internal/plugin/device_plugin.go b/plugins/internal/plugin/device_plugin.go new file mode 100644 index 0000000..e6882bf --- /dev/null +++ b/plugins/internal/plugin/device_plugin.go @@ -0,0 +1,405 @@ +//go:build windows + +package plugin + +import ( + "context" + "fmt" + "net" + "path/filepath" + "strings" + "sync" + "time" + + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "github.com/tensorworks/directx-device-plugins/plugins/internal/mount" +) + +type DevicePlugin struct { + + // The name of the plugin + name string + + // The configuration data for the plugin + config *PluginConfig + + // The Unix socket on which the plugin's gRPC server listens for connections + endpoint string + endpointDeleted *DeletionWatcher + + // The resource name that the plugin advertises to the Kubelet + resourceName string + + // The device watcher that monitors the available DirectX devices + watcher *DeviceWatcher + + // The most recent device list received from the device watcher, and a mutex to protect concurrent access + currentDevices []*discovery.Device + devicesMutex sync.Mutex + + // The logger used to log diagnostic information + logger *zap.SugaredLogger + + // The gRPC server that services requests from the Kubelet + server *grpc.Server + + // The channel used to trigger a restart of the gRPC server in the event of a Kubelet restart + restart chan struct{} + + // The channel used to stop the ListAndWatch streaming RPC during server shutdown + stopListWatch chan struct{} + + // The channel used for reporting errors while the gRPC server is running + Errors chan error +} + +// Creates a new device plugin +func NewDevicePlugin(pluginName string, pluginVersion string, resourceName string, filter discovery.DeviceFilter, config *PluginConfig, logger *zap.SugaredLogger) (*DevicePlugin, error) { + + // Attempt to create a new DeviceWatcher + watcher, err := NewDeviceWatcher( + pluginVersion, + filter, + config.IncludeIntegrated, + config.IncludeDetachable, + config.AdditionalMounts, + config.AdditionalMountsWow64, + logger, + ) + if err != nil { + return nil, err + } + + // Verify that device watcher can successfully list devices + select { + case <-watcher.Updates: + logger.Info("Initial device list retrieved successfully") + + case <-watcher.Errors: + watcher.Destroy() + return nil, fmt.Errorf("failed to perform device discovery: %v", err) + } + + // Create a new device plugin instance with the device watcher + plugin := &DevicePlugin{ + name: pluginName, + config: config, + endpoint: "", + endpointDeleted: nil, + resourceName: resourceName, + watcher: watcher, + currentDevices: []*discovery.Device{}, + devicesMutex: sync.Mutex{}, + logger: logger, + server: nil, + restart: make(chan struct{}, 1), + stopListWatch: nil, + Errors: make(chan error, 1), + } + + // Forward any device watcher errors to the plugin's error channel + go func() { + for err := range plugin.watcher.Errors { + plugin.Errors <- err + } + }() + + // Restart the plugin's gRPC server and perform plugin registration again in the event of a Kubelet restart + go func() { + for range plugin.restart { + + // Restart the gRPC server with a new Unix socket filename since the Kubelet will delete the old one + if err := plugin.RestartServer(); err != nil { + plugin.Errors <- err + } + + // Register the device plugin with the new Kubelet instance + if err := plugin.RegisterWithKubelet(); err != nil { + plugin.Errors <- err + } + } + }() + + return plugin, nil +} + +// Starts the gRPC server for the device plugin +func (p *DevicePlugin) StartServer() error { + + // Create a new gRPC server instance + // (Note that this is necessary to support restarts, since a server instance cannot be reused after it has stopped serving) + p.server = grpc.NewServer() + + // Register our service implementation with the gRPC server + p.logger.Info("Registering the service implementation with the gRPC server") + pluginapi.RegisterDevicePluginServer(p.server, p) + + // Append a timestamp to the filename for the gRPC server's Unix socket to ensure it is unique + p.endpoint = filepath.Join(pluginapi.DevicePluginPathWindows, fmt.Sprintf("%s-%d.sock", p.name, time.Now().UnixMilli())) + + // Attempt to listen for connections on our Unix socket + p.logger.Infow("Listening on endpoint", "endpoint", p.endpoint) + listener, err := net.Listen("unix", p.endpoint) + if err != nil { + return err + } + + // Create the shutdown channel for stopping the ListAndWatch streaming RPC + p.stopListWatch = make(chan struct{}) + + // Create a file deletion watcher for our Unix socket + endpointDeleted, err := WatchForDeletion(p.endpoint) + if err != nil { + return err + } + + // We detect Kubelet restarts by detecting the deletion of our socket + p.endpointDeleted = endpointDeleted + go func() { + for { + select { + + case err, ok := <-p.endpointDeleted.Errors: + if !ok { + p.logger.Info("DeletionWatcher error channel closed") + return + } + p.Errors <- err + + case _, ok := <-p.endpointDeleted.Deleted: + if !ok { + p.logger.Info("DeletionWatcher deletion channel closed") + return + } + + p.logger.Info("Endpoint deletion detected, triggering a restart of the gRPC server") + p.restart <- struct{}{} + } + } + }() + + // Start the gRPC server in a new goroutine and send any errors back through our error channel + go func() { + p.logger.Info("Starting the gRPC server") + if err := p.server.Serve(listener); err != nil { + p.Errors <- err + } + }() + + return nil +} + +// Gracefully stops the gRPC server for the device plugin +func (p *DevicePlugin) StopServer() { + + // If StopServer() is called before StartServer() then do nothing + if p.server == nil { + return + } + + // Stop the ListAndWatch streaming RPC if it is running + close(p.stopListWatch) + + // Stop watching our Unix socket for deletion events + p.endpointDeleted.Cancel() + + // Attempt to perform a graceful shutdown of the server (this will delete the Unix socket) + p.logger.Info("Gracefully stopping the gRPC server") + p.server.GracefulStop() + p.server = nil +} + +// Restarts the gRPC server for the device plugin, generating a new Unix socket filename +func (p *DevicePlugin) RestartServer() error { + p.StopServer() + return p.StartServer() +} + +// Destroys our underlying resources +func (p *DevicePlugin) Destroy() { + p.watcher.Destroy() + close(p.restart) + close(p.Errors) +} + +// Registers the device plugin with the Kubelet +func (p *DevicePlugin) RegisterWithKubelet() error { + + // Set a 60 second timeout when attempting to connect to the Kubelet + ctxConnect, cancelConnect := context.WithTimeout(context.Background(), time.Minute) + defer cancelConnect() + + // Create a dialler that treats the Kubelet's endpoint as a Unix socket rather than a TCP address + dialler := grpc.WithContextDialer(func(ctx context.Context, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", address) + }) + + // Attempt to connect to the Kubelet's gRPC service using the socket path for Windows + p.logger.Infow("Connecting to the Kubelet", "endpoint", pluginapi.KubeletSocketWindows) + conn, err := grpc.DialContext( + ctxConnect, + pluginapi.KubeletSocketWindows, + grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + dialler, + ) + if err != nil { + return fmt.Errorf("failed to connect to the Kubelet's gRPC service: %v", err) + } + defer conn.Close() + + // Prepare a registration request + request := &pluginapi.RegisterRequest{ + Version: pluginapi.Version, + Endpoint: filepath.Base(p.endpoint), + ResourceName: p.resourceName, + } + + // Set a 60 second timeout when attempting to register with the Kubelet + ctxRegister, cancelRegister := context.WithTimeout(context.Background(), time.Minute) + defer cancelRegister() + + // Create a registration client and attempt to send our registration request + p.logger.Infow("Sending registration request to the Kubelet", "request", request) + client := pluginapi.NewRegistrationClient(conn) + if _, err := client.Register(ctxRegister, request); err != nil { + return fmt.Errorf("failed to register the device plugin with the Kubelet: %v", err) + } + + p.logger.Info("Successfully registered the device plugin with the Kubelet") + return nil +} + +func (p *DevicePlugin) GetDevicePluginOptions(ctx context.Context, request *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) { + + // Instruct the Kubelet not to call the GetPreferredAllocation or PreStartContainer RPCs, since they aren't necessary + p.logger.Info("GetDevicePluginOptions RPC invoked") + return &pluginapi.DevicePluginOptions{ + GetPreferredAllocationAvailable: false, + PreStartRequired: false, + }, nil +} + +func (p *DevicePlugin) ListAndWatch(request *pluginapi.Empty, stream pluginapi.DevicePlugin_ListAndWatchServer) error { + + // Force a device list refresh to ensure we have an initial list for the Kubelet + p.logger.Info("ListAndWatch streaming RPC started, refreshing the device list") + p.watcher.ForceRefresh() + + // Continue sending updates as our device list changes or until shutdown is requested + for { + select { + + case <-p.stopListWatch: + p.logger.Info("Shutdown requested, stopping ListAndWatch streaming RPC") + return nil + + case <-stream.Context().Done(): + p.logger.Info("Kubelet disconnect detected, stopping ListAndWatch streaming RPC") + return nil + + case devices := <-p.watcher.Updates: + p.logger.Infow("Received new device list", "devices", devices) + + // Store the device list + p.devicesMutex.Lock() + p.currentDevices = devices + p.devicesMutex.Unlock() + + // Convert the device discovery devices to Kubernetes device plugin API devices + kubeletDevices := []*pluginapi.Device{} + for _, device := range devices { + + // Advertise each device multiple times, as per our multitenancy setting + for i := uint32(0); i < p.config.Multitenancy; i += 1 { + kubeletDevices = append(kubeletDevices, &pluginapi.Device{ + ID: fmt.Sprintf("%s\\%d", device.ID, i), + Health: pluginapi.Healthy, + }) + } + } + + // Send the device list to the Kubelet + p.logger.Infow("Sending device list to Kubelet", "devices", kubeletDevices) + stream.Send(&pluginapi.ListAndWatchResponse{ + Devices: kubeletDevices, + }) + } + } +} + +func (p *DevicePlugin) GetPreferredAllocation(context.Context, *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) { + + // This RPC should never be called + return nil, status.Error(codes.Unimplemented, "GetPreferredAllocation is not implemented") +} + +// Retrieves the device with the specified ID +func (p *DevicePlugin) GetDeviceForID(deviceID string) (*discovery.Device, error) { + + // Strip the multitenancy suffix from the device ID + backslash := strings.LastIndex(deviceID, "\\") + if backslash == -1 { + return nil, fmt.Errorf("malformed device ID \"%s\"", deviceID) + } + stripped := deviceID[0:backslash] + + // Lock the mutex for the device list + p.devicesMutex.Lock() + defer p.devicesMutex.Unlock() + + // Search for a device with the specified ID + for _, device := range p.currentDevices { + if device.ID == stripped { + return device, nil + } + } + + return nil, fmt.Errorf("could not find device with ID \"%s\"", stripped) +} + +func (p *DevicePlugin) Allocate(ctx context.Context, request *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { + + p.logger.Infow("Allocate RPC invoked, processing allocation request", "request", request) + response := &pluginapi.AllocateResponse{} + + // Process each of the container requests + for _, containerReq := range request.ContainerRequests { + + // Gather the list of requested devices for the container + devices := []*discovery.Device{} + for _, deviceID := range containerReq.DevicesIDs { + + // Verify that the requested device exists + device, err := p.GetDeviceForID(deviceID) + if err != nil { + return nil, err + } + + // Add the device to the list + devices = append(devices, device) + } + + // Generate the device specs and runtime file mounts for the requested devices, appending the container response to our overall response + response.ContainerResponses = append(response.ContainerResponses, &pluginapi.ContainerAllocateResponse{ + Devices: mount.SpecsForDevices(devices), + Mounts: mount.MountsForDevices(devices), + }) + } + + p.logger.Infow("Sending allocation response", "response", response) + return response, nil +} + +func (p *DevicePlugin) PreStartContainer(context.Context, *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) { + + // This RPC should never be called + return nil, status.Error(codes.Unimplemented, "PreStartContainer is not implemented") +} diff --git a/plugins/internal/plugin/device_watcher.go b/plugins/internal/plugin/device_watcher.go new file mode 100644 index 0000000..6232ef6 --- /dev/null +++ b/plugins/internal/plugin/device_watcher.go @@ -0,0 +1,205 @@ +//go:build windows + +package plugin + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "go.uber.org/zap" +) + +// Watches for device updates +type DeviceWatcher struct { + + // Our interface to the underlying DeviceDiscovery object from the DirectX device discovery library + deviceDiscovery *discovery.DeviceDiscovery + + // The filter used to control which devices are reported + deviceFilter discovery.DeviceFilter + + // Whether to include integrated GPUs when reporting devices + includeIntegrated bool + + // Whether to include detachable devices when reporting devices + includeDetachable bool + + // The list of additional runtime files for each device vendor that will be added to each device's list for System32 + additionalRuntimeFiles map[string][]*discovery.RuntimeFile + + // The list of additional runtime files for each device vendor that will be added to each device's list for SysWOW64 + additionalRuntimeFilesWow64 map[string][]*discovery.RuntimeFile + + // The logger used to log diagnostic information + logger *zap.SugaredLogger + + // The channel used to request a forced refresh of the device list + refresh chan struct{} + + // The channel used to stop the device discovery goroutine + shutdown chan struct{} + + // The channel used to report errors + Errors chan error + + // The channel used to report device updates + Updates chan []*discovery.Device +} + +func NewDeviceWatcher( + expectedVersion string, + deviceFilter discovery.DeviceFilter, + includeIntegrated bool, + includeDetachable bool, + additionalRuntimeFiles map[string][]*discovery.RuntimeFile, + additionalRuntimeFilesWow64 map[string][]*discovery.RuntimeFile, + logger *zap.SugaredLogger, +) (*DeviceWatcher, error) { + + // Attempt to load the DirectX device discovery library + if err := discovery.LoadDiscoveryLibrary(); err != nil { + return nil, err + } + + // Verify that the version of the device discovery library matches our expected version + libraryVersion := discovery.GetDiscoveryLibraryVersion() + if libraryVersion != expectedVersion { + return nil, fmt.Errorf( + "device discovery library version mismatch (found %s, expected %s)", + libraryVersion, + expectedVersion, + ) + } + + // Enable verbose logging for the device discovery library + discovery.EnableDiscoveryLogging() + + // Create a new DeviceDiscovery object + deviceDiscovery, err := discovery.NewDeviceDiscovery() + if err != nil { + return nil, err + } + + // Create the DeviceWatcher + watcher := &DeviceWatcher{ + deviceDiscovery: deviceDiscovery, + deviceFilter: deviceFilter, + includeIntegrated: includeIntegrated, + includeDetachable: includeDetachable, + additionalRuntimeFiles: additionalRuntimeFiles, + additionalRuntimeFilesWow64: additionalRuntimeFilesWow64, + logger: logger, + refresh: make(chan struct{}, 1), + shutdown: make(chan struct{}), + Errors: make(chan error, 1), + Updates: make(chan []*discovery.Device, 1), + } + + // Start the watcher goroutine + go watcher.watchDevices() + + return watcher, nil +} + +// Stops our goroutine and destroys the underlying DeviceDiscovery object +func (d *DeviceWatcher) Destroy() { + close(d.shutdown) + close(d.refresh) +} + +// Forces a refresh of the device list, irrespective of whether the current list is stale +func (d *DeviceWatcher) ForceRefresh() { + d.refresh <- struct{}{} +} + +// Merges any additional runtime files into the list for a device +func (d *DeviceWatcher) mergeRuntimeFiles(device *discovery.Device) { + + // Determine whether we have any additional runtime files for the device vendor + files, haveFiles := d.additionalRuntimeFiles[strings.ToLower(device.Vendor)] + filesWow64, haveFilesWow64 := d.additionalRuntimeFilesWow64[strings.ToLower(device.Vendor)] + + // Merge any additions for System32 + if haveFiles { + ignored := device.AppendRuntimeFiles(files) + for _, file := range ignored { + d.logger.Infow("Ignoring additional 64-bit runtime file because it clashes with an existing filename", "file", file) + } + } + + // Merge any additions for SysWOW64 + if haveFilesWow64 { + ignored := device.AppendRuntimeFilesWow64(filesWow64) + for _, file := range ignored { + d.logger.Infow("Ignoring additional 32-bit runtime file because it clashes with an existing filename", "file", file) + } + } +} + +// Refreshes the list of devices and reports the new list +func (d *DeviceWatcher) refreshDevices() error { + + // Refresh the list of devices + if err := d.deviceDiscovery.DiscoverDevices(d.deviceFilter, d.includeIntegrated, d.includeDetachable); err != nil { + return err + } + + // Process any additional runtime files for each device + for _, device := range d.deviceDiscovery.Devices { + d.mergeRuntimeFiles(device) + } + + // Report the new device list + d.Updates <- d.deviceDiscovery.Devices + return nil +} + +// The main device watch loop +func (d *DeviceWatcher) watchDevices() { + + // Destroy the underlying DeviceDiscovery object when the loop completes + defer d.deviceDiscovery.Destroy() + + // Use a context for waiting between polling operations rather than sleeping, so we remain responsive to shutdown and refresh events + sleep, cancelSleep := context.WithTimeout(context.Background(), time.Second*0) + defer cancelSleep() + + // Continue sending device updates until shutdown is requested: + forceRefresh := false + for { + select { + + case <-d.shutdown: + return + + case <-d.refresh: + forceRefresh = true + cancelSleep() + + case <-sleep.Done(): + + // Poll for device list changes + refresh, err := d.deviceDiscovery.IsRefreshRequired() + if err != nil { + d.Errors <- err + return + } + + // Retrieve the updated device list if one is available or if a forced refresh has been requested + if refresh || forceRefresh { + if err := d.refreshDevices(); err != nil { + d.Errors <- err + return + } + } + + // Wait 10 seconds before polling again + forceRefresh = false + sleep, cancelSleep = context.WithTimeout(context.Background(), time.Second*10) + defer cancelSleep() + } + } +} diff --git a/plugins/internal/plugin/plugin_configuration.go b/plugins/internal/plugin/plugin_configuration.go new file mode 100644 index 0000000..47ae4ea --- /dev/null +++ b/plugins/internal/plugin/plugin_configuration.go @@ -0,0 +1,145 @@ +//go:build windows + +package plugin + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/spf13/viper" + "github.com/tensorworks/directx-device-plugins/plugins/internal/discovery" + "github.com/tensorworks/directx-device-plugins/plugins/internal/mount" + "go.uber.org/zap" + "golang.org/x/exp/maps" +) + +// PluginConfig represents the available configuration options for a device plugin +type PluginConfig struct { + + // The number of containers that can access each device simultaneously (set this to 1 for exclusive access) + Multitenancy uint32 + + // Specifies whether we advertise integrated devices (i.e. integrated GPUs) + IncludeIntegrated bool + + // Specifies whether we advertise detachable devices (e.g. external GPUs) + IncludeDetachable bool + + // The list of additional runtime files to be mounted to System32 for each device vendor + AdditionalMounts map[string][]*discovery.RuntimeFile + + // The list of additional runtime files to be mounted to SysWOW64 for each device vendor + AdditionalMountsWow64 map[string][]*discovery.RuntimeFile +} + +// Appends a default set of mounts to the supplied mounts, converting all vendor names to lower case to ensure consistency +func appendMounts(mounts map[string][]*discovery.RuntimeFile, defaults map[string][]*discovery.RuntimeFile) map[string][]*discovery.RuntimeFile { + + // Gather the set of unique vendor names, converting all names to lower case + vendors := make(map[string]bool) + for _, vendor := range append(maps.Keys(mounts), maps.Keys(defaults)...) { + vendorLower := strings.ToLower(vendor) + if !vendors[vendorLower] { + vendors[vendorLower] = true + } + } + + // Process the mounts for each vendor in turn + appended := make(map[string][]*discovery.RuntimeFile) + for vendor := range vendors { + appended[vendor] = []*discovery.RuntimeFile{} + + // Add the mounts for the vendor if we have any + vendorMounts, haveMounts := mounts[vendor] + if haveMounts { + appended[vendor] = append(appended[vendor], vendorMounts...) + } + + // Add the defaults for the vendor if we have any + vendorDefaults, haveDefaults := defaults[vendor] + if haveDefaults { + appended[vendor] = append(appended[vendor], vendorDefaults...) + } + } + + return appended +} + +// Load loads the configuration data from the runtime environment. +func LoadConfig(pluginName string, logger *zap.SugaredLogger) (*PluginConfig, error) { + + // Set our default configuration values + v := viper.New() + v.SetDefault("multitenancy", 0) + v.SetDefault("includeIntegrated", false) + v.SetDefault("includeDetachable", false) + v.SetDefault("additionalMounts", make(map[string][]*discovery.RuntimeFile)) + v.SetDefault("additionalMountsWow64", make(map[string][]*discovery.RuntimeFile)) + + // The names of our environment variables reflect the plugin name + envPrefix := fmt.Sprint(strings.ToUpper(pluginName), "_DEVICE_PLUGIN_") + v.BindEnv("multitenancy", fmt.Sprint(envPrefix, "MULTITENANCY")) + v.BindEnv("includeIntegrated", fmt.Sprint(envPrefix, "INCLUDE_INTEGRATED")) + v.BindEnv("includeDetachable", fmt.Sprint(envPrefix, "INCLUDE_DETACHABLE")) + + // Check if a config file path was explicitly specified through an environment variable + configPath, configPathExists := os.LookupEnv(fmt.Sprint(envPrefix, "CONFIG_FILE")) + if configPathExists { + + // Verify that the specified value is an absolute path + if !filepath.IsAbs(configPath) { + return nil, errors.New("configuration file path must be an absolute path") + } + + // Verify that the specified file exists + if _, err := os.Stat(configPath); errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("specified configuration file does not exist: %s", configPath) + } + + // Use the specified path + v.SetConfigFile(configPath) + + } else { + + // The default name of our YAML configuration file reflects the plugin name + v.SetConfigName(pluginName) + v.SetConfigType("yaml") + + // We search for the configuration file in both our global config directory and the current working directory + v.AddConfigPath(".") + v.AddConfigPath("\\etc\\directx-device-plugins") + } + + // Attempt to parse our YAML configuration file if it exists + if err := v.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + logger.Infow("Configuration file not found, using configuration values from environment variables") + } else { + return nil, err + } + } + + // Load the parsed configuration values into our struct + c := &PluginConfig{} + if err := v.Unmarshal(c); err != nil { + return nil, err + } + + // Enforce a minimum value of 1 for multitenancy + if c.Multitenancy == 0 { + c.Multitenancy = 1 + } + + // Append our default mounts to any user-supplied values + c.AdditionalMounts = appendMounts(c.AdditionalMounts, mount.DefaultMounts) + c.AdditionalMountsWow64 = appendMounts(c.AdditionalMountsWow64, mount.DefaultMountsWow64) + + // Log the parsed configuration values + logger.Infow("Parsed configuration data", "config", c) + + return c, nil +} diff --git a/update-version.bat b/update-version.bat new file mode 100644 index 0000000..5155922 --- /dev/null +++ b/update-version.bat @@ -0,0 +1 @@ +@powershell -ExecutionPolicy Bypass -File "%~dp0.\update-version.ps1" %* diff --git a/update-version.ps1 b/update-version.ps1 new file mode 100644 index 0000000..a9038d3 --- /dev/null +++ b/update-version.ps1 @@ -0,0 +1,51 @@ +Param ( + [parameter(Mandatory=$true, Position=0, HelpMessage = "The new version string")] + $version +) + + +# Patches a file using the specified regular expression search string and replacement +function Patch-File { + Param ( + $Path, + $Search, + $Replace + ) + + Write-Host "Updating $Path" -ForegroundColor Cyan + $content = Get-Content -Path $Path -Raw + $content = $content -replace $Search, $Replace + Set-Content -Path $Path -NoNewline -Value $content +} + +# Patches image tags in a YAML file +function Patch-ImageTags { + Param ($Path) + Patch-File -Path $Path ` + -Search '"index.docker.io/tensorworks/(.+):(.+)"' ` + -Replace "`"index.docker.io/tensorworks/`$1:$version`"" +} + + +# Update the version strings in all of our files, ready for a new release +Write-Host "Updating version strings to $version..." -ForegroundColor Green + +# Update the version string for the device discovery library +Patch-File -Path "$PSScriptRoot\library\src\DeviceDiscovery.cpp" ` + -Search '#define LIBRARY_VERSION L"(.+)"' ` + -Replace "#define LIBRARY_VERSION L`"$version`"" + +# Update the version string for the device plugins +Patch-File -Path "$PSScriptRoot\plugins\internal\plugin\common_main.go" ` + -Search 'const version = "(.+)"' ` + -Replace "const version = `"$version`"" + +# Update the deployment YAML files +foreach ($file in Get-ChildItem -Path "$PSScriptRoot\deployments\*.yml") { + Patch-ImageTags -Path $file.FullName +} + +# Update the example YAML files +foreach ($file in Get-ChildItem -Path "$PSScriptRoot\examples\*\*.yml") { + Patch-ImageTags -Path $file.FullName +}