Skip to content

Commit

Permalink
[CI] Add "loader" support to conformance testing
Browse files Browse the repository at this point in the history
This expands our CI to test the loader; the dispatcher that is
used when multiple adapters are availabe.

Previously, the unit tests forced a specific adapter, via
`UR_ADAPTERS_FORCE_LOAD`. Now an extra "loader" target is created
for each test suite, which doesn't set that variable and allows
the loader to be used.

In addition, the test runner's adapter selection logic has been
rewritten to support a "--backend" variable, which allows you to
select a backend (OpenCL, Level Zero, etc.).

The platform selection has also been expanded to support
filtering by a backend to match that platform only on that backend
(e.g. `UR_CTS_ADAPTER_PLATFORM="opencl:Fictional Corp(R)"`).

The old "run on hardware" jobs should behave the same (they have
the loader tests disabled), but there is a new
`combined-opencl-level-zero` job that tests a build with both
OpenCL and Level Zero available.
  • Loading branch information
RossBrunton committed Sep 30, 2024
1 parent bcd9032 commit e995457
Show file tree
Hide file tree
Showing 75 changed files with 332 additions and 68 deletions.
25 changes: 21 additions & 4 deletions .github/workflows/build-hw-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,21 @@ on:
adapter_name:
required: true
type: string
other_adapter_name:
required: false
type: string
default: ""
runner_name:
required: true
type: string
platform:
required: false
type: string
default: ""
other_platform:
required: false
type: string
default: ""
static_loader:
required: false
type: string
Expand All @@ -39,9 +47,14 @@ jobs:
if: github.repository == 'oneapi-src/unified-runtime' # run only on upstream; forks won't have the HW
strategy:
matrix:
adapter: [
{name: "${{inputs.adapter_name}}", platform: "${{inputs.platform}}", static_Loader: "${{inputs.static_loader}}", static_adapter: "${{inputs.static_loader}}"},
]
adapter: [{
name: "${{inputs.adapter_name}}",
other_name: "${{inputs.other_adapter_name}}",
platform: "${{inputs.platform}}",
other_platform: "${{inputs.other_platform}}",
static_Loader: "${{inputs.static_loader}}",
static_adapter: "${{inputs.static_loader}}"
}]
build_type: [Debug, Release]
compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
# TODO: The latest L0 loader segfaults when built with clang.
Expand Down Expand Up @@ -83,10 +96,12 @@ jobs:
-DUR_DEVELOPER_MODE=ON
-DUR_BUILD_TESTS=ON
-DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{}=ON', matrix.adapter.other_name) || '' }}
-DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
-DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
-DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
-DUR_SYCL_LIBRARY_DIR=${{github.workspace}}/dpcpp_compiler/lib
-DUR_CONFORMANCE_TEST_LOADER=${{ matrix.adapter.other_name != '' && 'ON' || 'OFF' }}
${{ matrix.adapter.name == 'HIP' && '-DUR_CONFORMANCE_AMD_ARCH=gfx1030' || '' }}
${{ matrix.adapter.name == 'HIP' && '-DUR_HIP_PLATFORM=AMD' || '' }}
Expand All @@ -97,10 +112,12 @@ jobs:
- name: Test adapter specific
working-directory: ${{github.workspace}}/build
run: ctest -C ${{matrix.build_type}} --output-on-failure -L "adapter-specific" --timeout 180
# Don't run adapter specific tests when building multiple adapters
if: ${{ matrix.adapter.other_name == '' }}

- name: Test adapters
working-directory: ${{github.workspace}}/build
run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --output-on-failure -L "conformance" --timeout 180
run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.name}}:${{matrix.adapter.platform}};${{matrix.adapter.other_name}}:${{matrix.adapter.other_platform}}" ctest -C ${{matrix.build_type}} --output-on-failure -L "conformance" --timeout 180

- name: Get information about platform
if: ${{ always() }}
Expand Down
9 changes: 9 additions & 0 deletions .github/workflows/cmake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,15 @@ jobs:
adapter_name: NATIVE_CPU
runner_name: NATIVE_CPU

combined-opencl-level-zero:
name: OpenCL + Level Zero
uses: ./.github/workflows/build-hw-reusable.yml
with:
adapter_name: OPENCL
other_adapter_name: L0
runner_name: OPENCL
platform: "Intel(R) OpenCL"

e2e-level-zero:
name: E2E L0
permissions:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/multi_device.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ jobs:
-DUR_DEVELOPER_MODE=ON
-DUR_BUILD_TESTS=ON
-DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
-DUR_CONFORMANCE_TEST_LOADER=OFF
-DUR_TEST_DEVICES_COUNT=2
- name: Build
Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
"List of sycl targets to build CTS device binaries for")
set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
option(UR_CONFORMANCE_ENABLE_MATCH_FILES "Enable CTS match files" ON)
option(UR_CONFORMANCE_TEST_LOADER "Also test the loader in the conformance tests" ON)
set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
"Path to external 'level_zero' adapter source dir")
set(UR_ADAPTER_OPENCL_SOURCE_DIR "" CACHE PATH
Expand Down
3 changes: 3 additions & 0 deletions source/loader/ur_ldrddi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,9 @@ __urdlllocal ur_result_t UR_APICALL urDeviceGetInfo(

[[maybe_unused]] auto context = getContext();

// For testing
abort();

// extract platform's function pointer table
auto dditable = reinterpret_cast<ur_device_object_t *>(hDevice)->dditable;
auto pfnGetInfo = dditable->ur.Device.pfnGetInfo;
Expand Down
2 changes: 1 addition & 1 deletion test/adapters/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ function(add_adapter_memcheck_test name)
add_test(NAME ${test_name}
COMMAND ${CMAKE_COMMAND}
-D TEST_FILE=valgrind
-D TEST_ARGS="--tool=memcheck --leak-check=full $<TARGET_FILE:${target}> --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_DEVICES_COUNT}"
-D TEST_ARGS="--tool=memcheck --leak-check=full $<TARGET_FILE:${target}> --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_DEVICES_COUNT}"
-D MODE=stderr
-D MATCH_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${name}_memcheck.match
-P ${PROJECT_SOURCE_DIR}/cmake/match.cmake
Expand Down
41 changes: 32 additions & 9 deletions test/conformance/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ set(UR_CONFORMANCE_DEVICE_BINARIES_DIR
"${CMAKE_CURRENT_BINARY_DIR}/device_binaries" CACHE INTERNAL
"Internal cache variable for device binaries directory")

function(add_test_adapter name adapter)
function(add_test_adapter name adapter backend force)
if(NOT "${ARGN}" STREQUAL "")
set(EXTRA_NAME "-${ARGN}")
endif()
set(TEST_TARGET_NAME test-${name})
set(TEST_NAME ${name}-${adapter}${EXTRA_NAME})

set(TEST_COMMAND
"${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
"${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
)
set(MATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}${EXTRA_NAME}.match")

Expand All @@ -41,7 +41,9 @@ function(add_test_adapter name adapter)
)
endif()

set(TEST_ENV UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
if(${force})
set(TEST_ENV UR_ADAPTERS_FORCE_LOAD="$<TARGET_FILE:ur_${adapter}>")
endif()
if(UR_CONFORMANCE_ENABLE_MATCH_FILES)
list(APPEND TEST_ENV GTEST_COLOR=no)
endif()
Expand All @@ -66,22 +68,43 @@ function(add_conformance_test name)
unit_tests_helpers)

if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_cuda)
add_test_adapter(${name} adapter_cuda CUDA ON)
endif()
if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_hip)
add_test_adapter(${name} adapter_hip HIP ON)
endif()
if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_level_zero)
add_test_adapter(${name} adapter_level_zero LEVEL_ZERO ON)
endif()
if(UR_BUILD_ADAPTER_L0_V2)
add_test_adapter(${name} adapter_level_zero_v2)
add_test_adapter(${name} adapter_level_zero_v2 LEVEL_ZERO ON)
endif()
if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_opencl)
add_test_adapter(${name} adapter_opencl OPENCL ON)
endif()
if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_native_cpu)
add_test_adapter(${name} adapter_native_cpu NATIVE_CPU ON)
endif()

if(UR_CONFORMANCE_TEST_LOADER)
if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_cuda_loader CUDA OFF)
endif()
if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_hip_loader HIP OFF)
endif()
if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_level_zero_loader LEVEL_ZERO OFF)
endif()
if(UR_BUILD_ADAPTER_L0_V2)
add_test_adapter(${name} adapter_level_zero_v2_loader LEVEL_ZERO OFF)
endif()
if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_opencl_loader OPENCL OFF)
endif()
if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
add_test_adapter(${name} adapter_native_cpu_loader NATIVE_CPU OFF)
endif()
endif()

if(NOT (UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_HIP
Expand Down
4 changes: 3 additions & 1 deletion test/conformance/cts_exe.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@
parser.add_argument("--test_command", help="Ctest test case")
parser.add_argument("--devices_count", type=str, help="Number of devices on which tests will be run")
parser.add_argument("--platforms_count", type=str, help="Number of platforms on which tests will be run")
parser.add_argument("--backend", type=str, help="Number of platforms on which tests will be run")
args = parser.parse_args()

result = subprocess.Popen([args.test_command, '--gtest_brief=1', # nosec B603
f'--devices_count={args.devices_count}',
f'--platforms_count={args.platforms_count}'],
f'--platforms_count={args.platforms_count}',
f'--backend={args.backend}'],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)

pat = re.compile(r'\[( )*FAILED( )*\]')
Expand Down
1 change: 1 addition & 0 deletions test/conformance/device/device_adapter_cuda_loader.match
1 change: 1 addition & 0 deletions test/conformance/device/device_adapter_hip_loader.match
1 change: 1 addition & 0 deletions test/conformance/device/device_adapter_opencl_loader.match
20 changes: 20 additions & 0 deletions test/conformance/device_code/single.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright (C) 2023 Intel Corporation
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include <sycl/sycl.hpp>

class Single;

int main() {
sycl::queue deviceQueue;
sycl::range<1> numOfItems{1};

deviceQueue.submit([&](sycl::handler &cgh) {
auto kern = [=]() {};
cgh.single_task<Single>(kern);
});

return 0;
}
1 change: 1 addition & 0 deletions test/conformance/enqueue/enqueue_adapter_cuda_loader.match
1 change: 1 addition & 0 deletions test/conformance/enqueue/enqueue_adapter_hip_loader.match
1 change: 1 addition & 0 deletions test/conformance/event/event_adapter_cuda_loader.match
1 change: 1 addition & 0 deletions test/conformance/event/event_adapter_hip_loader.match
1 change: 1 addition & 0 deletions test/conformance/kernel/kernel_adapter_cuda_loader.match
1 change: 1 addition & 0 deletions test/conformance/kernel/kernel_adapter_hip_loader.match
1 change: 1 addition & 0 deletions test/conformance/kernel/kernel_adapter_opencl_loader.match
1 change: 1 addition & 0 deletions test/conformance/memory/memory_adapter_cuda_loader.match
1 change: 1 addition & 0 deletions test/conformance/memory/memory_adapter_hip_loader.match
1 change: 1 addition & 0 deletions test/conformance/memory/memory_adapter_opencl_loader.match
1 change: 1 addition & 0 deletions test/conformance/program/program_adapter_cuda_loader.match
1 change: 1 addition & 0 deletions test/conformance/program/program_adapter_hip_loader.match
Loading

0 comments on commit e995457

Please sign in to comment.