From cf69f6c93ae7061e005cdffaba9f7ec89d892fbe Mon Sep 17 00:00:00 2001 From: Kristian Ollikainen <14197772+DatCaptainHorse@users.noreply.github.com> Date: Sun, 7 Jul 2024 11:06:48 +0300 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(server):=20Add=20Intel/AMD=20G?= =?UTF-8?q?PU=20support=20(#84)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description ### This is a DRAFT - Changes will be discussed and made upon requests! In nutshell, this adds support for running Nestri with Intel and AMD GPU's. Both integrated and dedicated. It took a few days to find a trick for having output without dummy plugs or connected displays, but I think I got it. `gpu-screen-recorder` requires a custom patch to skip the check for connected displays (as we're using a xrandr workaround which makes them stay "unconnected") Most likely fixes #68 ### Changes The NVIDIA sections have been split in their own code branches since there's some NVIDIA specific things I didn't feel approriate to poke more than necessary for the goal of this PR. Added a script with helper functions related to GPU discovery and gathering some basic info off from them (note: it might be better to declare the helper script arrays outside it's initially run function). The helper scripts rely on `lshw`. NVIDIA code was slightly adjusted to use the bus-id's provided by the helper functions to have some code re-use. Cleaned up few things on the side. --------- Co-authored-by: Kristian Ollikainen Co-authored-by: Wanjohi <71614375+wanjohiryan@users.noreply.github.com> --- .patches/connectcheckskip.patch | 28 ++++ .scripts/entrypoint.sh | 222 +++++++++++++++++++++-------- .scripts/gpu_helpers.sh | 243 ++++++++++++++++++++++++++++++++ base.Dockerfile | 5 + server.Dockerfile | 3 +- 5 files changed, 441 insertions(+), 60 deletions(-) create mode 100644 .patches/connectcheckskip.patch create mode 100644 .scripts/gpu_helpers.sh diff --git a/.patches/connectcheckskip.patch b/.patches/connectcheckskip.patch new file mode 100644 index 00000000..01b25ad2 --- /dev/null +++ b/.patches/connectcheckskip.patch @@ -0,0 +1,28 @@ +diff --git a/src/utils.c b/src/utils.c +index e00f3c5..4f1f0bf 100644 +--- a/src/utils.c ++++ b/src/utils.c +@@ -71,7 +71,7 @@ void for_each_active_monitor_output_x11(Display *display, active_monitor_callbac + char display_name[256]; + for(int i = 0; i < screen_res->noutput; ++i) { + XRROutputInfo *out_info = XRRGetOutputInfo(display, screen_res, screen_res->outputs[i]); +- if(out_info && out_info->crtc && out_info->connection == RR_Connected) { ++ if(out_info && out_info->crtc) { + XRRCrtcInfo *crt_info = XRRGetCrtcInfo(display, screen_res, out_info->crtc); + if(crt_info && crt_info->mode) { + const XRRModeInfo *mode_info = get_mode_info(screen_res, crt_info->mode); +@@ -218,10 +218,10 @@ static void for_each_active_monitor_output_drm(const gsr_egl *egl, active_monito + if(connector_type) + ++connector_type->count; + +- if(connector->connection != DRM_MODE_CONNECTED) { +- drmModeFreeConnector(connector); +- continue; +- } ++ //if(connector->connection != DRM_MODE_CONNECTED) { ++ // drmModeFreeConnector(connector); ++ // continue; ++ //} + + if(connector_type) + ++connector_type->count_active; diff --git a/.scripts/entrypoint.sh b/.scripts/entrypoint.sh index 121313c3..fedf9953 100644 --- a/.scripts/entrypoint.sh +++ b/.scripts/entrypoint.sh @@ -1,6 +1,9 @@ #!/bin/bash -e trap "echo TRAPed signal" HUP INT QUIT TERM +# Include our gpu helper functions +source /etc/gpu_helpers.sh + # Create and modify permissions of XDG_RUNTIME_DIR sudo -u nestri mkdir -pm700 /tmp/runtime-1000 sudo chown nestri:nestri /tmp/runtime-1000 @@ -63,34 +66,8 @@ sudo /etc/init.d/dbus start # Install Proton-GE for this user nestri-proton -i -# Install NVIDIA userspace driver components including X graphic libraries -if ! command -v nvidia-xconfig &> /dev/null; then - # Driver version is provided by the kernel through the container toolkit - export DRIVER_ARCH="$(dpkg --print-architecture | sed -e 's/arm64/aarch64/' -e 's/armhf/32bit-ARM/' -e 's/i.*86/x86/' -e 's/amd64/x86_64/' -e 's/unknown/x86_64/')" - export DRIVER_VERSION="$(head -n1 /dev/null fi if grep -Fxq "allowed_users=console" /etc/X11/Xwrapper.config; then @@ -102,22 +79,6 @@ if [ -f "/etc/X11/xorg.conf" ]; then sudo rm -f "/etc/X11/xorg.conf" fi -# Get first GPU device if all devices are available or `NVIDIA_VISIBLE_DEVICES` is not set -if [ "$NVIDIA_VISIBLE_DEVICES" == "all" ] || [ -z "$NVIDIA_VISIBLE_DEVICES" ]; then - export GPU_SELECT="$(sudo nvidia-smi --query-gpu=uuid --format=csv | sed -n 2p)" -# Get first GPU device out of the visible devices in other situations -else - export GPU_SELECT="$(sudo nvidia-smi --id=$(echo "$NVIDIA_VISIBLE_DEVICES" | cut -d ',' -f1) --query-gpu=uuid --format=csv | sed -n 2p)" - if [ -z "$GPU_SELECT" ]; then - export GPU_SELECT="$(sudo nvidia-smi --query-gpu=uuid --format=csv | sed -n 2p)" - fi -fi - -if [ -z "$GPU_SELECT" ]; then - echo "No NVIDIA GPUs detected or nvidia-container-toolkit not configured. Exiting." - exit 1 -fi - # Setting `VIDEO_PORT` to none disables RANDR/XRANDR, do not set this if using datacenter GPUs if [ "${VIDEO_PORT,,}" = "none" ]; then export CONNECTED_MONITOR="--use-display-device=None" @@ -126,26 +87,127 @@ else export CONNECTED_MONITOR="--connected-monitor=${VIDEO_PORT}" fi -# Bus ID from nvidia-smi is in hexadecimal format and should be converted to decimal format (including the domain) which Xorg understands, required because nvidia-xconfig doesn't work as intended in a container -HEX_ID="$(sudo nvidia-smi --query-gpu=pci.bus_id --id="$GPU_SELECT" --format=csv | sed -n 2p)" -IFS=":." ARR_ID=($HEX_ID) -unset IFS -BUS_ID="PCI:$((16#${ARR_ID[1]}))@$((16#${ARR_ID[0]})):$((16#${ARR_ID[2]})):$((16#${ARR_ID[3]}))" # A custom modeline should be generated because there is no monitor to fetch this information normally -export MODELINE="$(cvt -r "${SIZEW}" "${SIZEH}" "${REFRESH}" | sed -n 2p)" -# Generate /etc/X11/xorg.conf with nvidia-xconfig -sudo nvidia-xconfig --virtual="${SIZEW}x${SIZEH}" --depth="$CDEPTH" --mode="$(echo "$MODELINE" | awk '{print $2}' | tr -d '\"')" --allow-empty-initial-configuration --no-probe-all-gpus --busid="$BUS_ID" --include-implicit-metamodes --mode-debug --no-sli --no-base-mosaic --only-one-x-screen ${CONNECTED_MONITOR} -# Guarantee that the X server starts without a monitor by adding more options to the configuration -sudo sed -i '/Driver\s\+"nvidia"/a\ Option "ModeValidation" "NoMaxPClkCheck,NoEdidMaxPClkCheck,NoMaxSizeCheck,NoHorizSyncCheck,NoVertRefreshCheck,NoVirtualSizeCheck,NoExtendedGpuCapabilitiesCheck,NoTotalSizeCheck,NoDualLinkDVICheck,NoDisplayPortBandwidthCheck,AllowNon3DVisionModes,AllowNonHDMI3DModes,AllowNonEdidModes,NoEdidHDMI2Check,AllowDpInterlaced"' /etc/X11/xorg.conf -# Add custom generated modeline to the configuration -sudo sed -i '/Section\s\+"Monitor"/a\ '"$MODELINE" /etc/X11/xorg.conf -# Prevent interference between GPUs, add this to the host or other containers running Xorg as well -echo -e "Section \"ServerFlags\"\n Option \"AutoAddGPU\" \"false\"\nEndSection" | sudo tee -a /etc/X11/xorg.conf > /dev/null +custom_modeline="$(cvt -r "${SIZEW}" "${SIZEH}" "${REFRESH}" | sed -n 2p)" +custom_modeline_settings="$(echo "$custom_modeline" | sed 's/Modeline //')" +custom_modeline_identifier="$(echo "$custom_modeline_settings" | awk '{print $1}' | tr -d '"')" + +# Pre-populate GPU information manually +if ! check_and_populate_gpus; then + exit 1 +fi + +# Select the GPU based on user input or first one available +selected_gpu="${GPU_SELECTION,,:-}" +if [[ -z "$selected_gpu" ]]; then + selected_gpu="${gpu_map[0]}" # Select first available GPU + echo "No GPU selected, using first one available: $selected_gpu" +elif ! selected_gpu=$(check_selected_gpu "$selected_gpu"); then + exit 1 +fi + +# Print selected GPU information +echo "Selected GPU: $(print_gpu_info "$selected_gpu")" +echo "" + +# Get GPU vendor as separate variable +selected_gpu_vendor=$(get_gpu_vendor "$selected_gpu") +# Convert lshw gathered bus id into Xorg compatible one +xorg_bus_id=$(get_gpu_bus_xorg "$selected_gpu") + +# Check if the selected GPU is an NVIDIA GPU +if [[ "${selected_gpu_vendor,,}" =~ "nvidia" ]]; then + echo "Selected GPU is NVIDIA. Handling NVIDIA-specific configuration..." + + # Install NVIDIA userspace driver components including X graphic libraries + if ! command -v nvidia-xconfig &> /dev/null; then + # Driver version is provided by the kernel through the container toolkit + export DRIVER_ARCH="$(dpkg --print-architecture | sed -e 's/arm64/aarch64/' -e 's/armhf/32bit-ARM/' -e 's/i.*86/x86/' -e 's/amd64/x86_64/' -e 's/unknown/x86_64/')" + export DRIVER_VERSION="$(head -n1 /dev/null +else + echo "Selected GPU is non-NVIDIA. Handling common configuration..." + + # We need permissions for the GPU(s) + sudo chown -R root:root /dev/dri/* + sudo chmod -R 777 /dev/dri/* + + # Create common config file + sudo touch /etc/X11/xorg.conf + config_common_xorg=" +Section \"ServerLayout\" + Identifier \"Layout0\" + Screen 0 \"Screen0\" + InputDevice \"Keyboard0\" \"CoreKeyboard\" + InputDevice \"Mouse0\" \"CorePointer\" +EndSection + +Section \"InputDevice\" + Identifier \"Mouse0\" + Driver \"mouse\" + Option \"Protocol\" \"auto\" + Option \"Device\" \"/dev/mouse\" + Option \"Emulate3Buttons\" \"no\" + Option \"ZAxisMapping\" \"4 5\" +EndSection + +Section \"InputDevice\" + Identifier \"Keyboard0\" + Driver \"kbd\" +EndSection + +Section \"Device\" + Identifier \"Device0\" + Driver \"modesetting\" + BusID \"$xorg_bus_id\" +EndSection + +Section \"Screen\" + Identifier \"Screen0\" + Device \"Device0\" + Option \"ModeDebug\" \"True\" +EndSection + +Section \"ServerFLags\" + Option \"AutoAddGPU\" \"off\" +EndSection +" + echo "$config_common_xorg" | sudo tee /etc/X11/xorg.conf > /dev/null +fi # Default display is :0 across the container export DISPLAY=":0" # Run Xorg server with required extensions -/usr/bin/Xorg vt7 -noreset -novtswitch -sharevts -dpi "${DPI}" +extension "COMPOSITE" +extension "DAMAGE" +extension "GLX" +extension "RANDR" +extension "RENDER" +extension "MIT-SHM" +extension "XFIXES" +extension "XTEST" "${DISPLAY}" & +/usr/bin/Xorg vt7 -noreset -novtswitch -sharevts -dpi "${DPI}" -fakescreenfps "${REFRESH}" +extension "COMPOSITE" +extension "DAMAGE" +extension "GLX" +extension "RANDR" +extension "RENDER" +extension "MIT-SHM" +extension "XFIXES" +extension "XTEST" "${DISPLAY}" & # Wait for X11 to start echo "Waiting for X socket" @@ -157,10 +219,52 @@ echo "$(date +"[%Y-%m-%d %H:%M:%S]") Waiting for X socket" until [ -S "/tmp/.X11-unix/X${DISPLAY/:/}" ]; do sleep 1; done echo "$(date +"[%Y-%m-%d %H:%M:%S]") X socket is ready" +# Additional non-NVIDIA configuration required +if [[ ! "${selected_gpu_vendor,,}" =~ "nvidia" ]]; then + # Get a list of all available outputs (connected or disconnected) + all_outputs=($(xrandr --query | awk '/ connected| disconnected/ {print $1}')) + + for selected_output in "${all_outputs[@]}"; do + # Create a unique mode identifier by appending the output name + unique_mode_identifier="${selected_output}-${custom_modeline_identifier}" + + # Create a unique modeline setting with the new identifier + unique_modeline_settings="$(echo "$custom_modeline_settings" | sed "s/$custom_modeline_identifier/$unique_mode_identifier/" | tr -d '"')" + + # Check if the mode already exists for this output (avoid duplicates) + if xrandr --query | grep "$selected_output" | grep -q "$unique_mode_identifier"; then + echo "Mode '$unique_mode_identifier' already exists for output '$selected_output', skipping.." + continue + fi + + # Add the new mode for the specific output (using the unique settings variable) + if xrandr --newmode $unique_modeline_settings; then + echo "Successfully added mode '$unique_mode_identifier' for output '$selected_output'" + + # Configure the output to use the new mode + if xrandr --addmode "$selected_output" "$unique_mode_identifier" && \ + xrandr --output "$selected_output" --primary --mode "$unique_mode_identifier"; then + echo "Successfully configured output '$selected_output' to use mode '$unique_mode_identifier'" + break + fi + fi + + echo "Failed to configure output '$selected_output' to use mode '$unique_mode_identifier', trying the next output.." + done + + if [[ "$selected_output" == "${all_outputs[-1]}" ]]; then + echo "Could not configure any output with the desired mode" + exit 1 + fi +fi + +# Make sure gpu-screen-recorder is owned by nestri +sudo chown nestri:nestri /usr/bin/gpu-screen-recorder + if [[ -z "${SESSION_ID}" ]]; then echo "$(date +"[%Y-%m-%d %H:%M:%S]") No stream name was found, did you forget to set the env variable NAME?" && exit 1 else - /usr/bin/gpu-screen-recorder -w screen -c flv -f 60 -a "$(pactl get-default-sink).monitor" | ffmpeg -hide_banner -v quiet -i pipe:0 -c copy -f mp4 -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset - | /usr/bin/warp --name "${SESSION_ID}" https://fst.so:4443 & + /usr/bin/gpu-screen-recorder -v no -w screen -c flv -f "${REFRESH}" -a "$(pactl get-default-sink).monitor" | ffmpeg -hide_banner -v quiet -i pipe:0 -c copy -f mp4 -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset - | /usr/bin/warp --name "${SESSION_ID}" https://fst.so:4443 & fi openbox-session & diff --git a/.scripts/gpu_helpers.sh b/.scripts/gpu_helpers.sh new file mode 100644 index 00000000..eac2cf96 --- /dev/null +++ b/.scripts/gpu_helpers.sh @@ -0,0 +1,243 @@ +#!/bin/bash -e + +# Various helper functions for handling available GPUs + +declare -ga gpu_map +declare -gA gpu_bus_map +declare -gA gpu_product_map +declare -gA vendor_index_map +declare -gA vendor_full_map + +# Map to help get shorter vendor identifiers +declare -A vendor_keywords=( + ["advanced micro devices"]='amd' + ["ati"]='amd' + ["amd"]='amd' + ["radeon"]='amd' + ["nvidia"]='nvidia' + ["intel"]='intel' +) + +get_gpu_info() { + # Clear out previous data + gpu_map=() + gpu_bus_map=() + gpu_product_map=() + vendor_index_map=() + vendor_full_map=() + + local vendor="" + local product="" + local bus_info="" + local vendor_full="" + + while read -r line; do + line="${line##*( )}" + + if [[ "${line,,}" =~ "vendor:" ]]; then + vendor="" + vendor_full=$(echo "$line" | awk '{$1=""; print $0}' | xargs) + + # Look for short vendor keyword in line + for keyword in "${!vendor_keywords[@]}"; do + if [[ "${line,,}" == *"$keyword"* ]]; then + vendor="${vendor_keywords[$keyword]}" + break + fi + done + + # If no vendor keywords match, use first word + if [[ -z "$vendor" ]]; then + vendor=$(echo "$vendor_full" | awk '{print tolower($1)}') + fi + elif [[ "${line,,}" =~ "product:" ]]; then + product=$(echo "$line" | awk '{$1=""; print $0}' | xargs) + elif [[ "${line,,}" =~ "bus info:" ]]; then + bus_info=$(echo "$line" | awk '{print $3}') + fi + + if [[ -n "$vendor" && -n "$product" && -n "$bus_info" && ! "${line,,}" =~ \*"-display" ]]; then + # We have gathered all GPU info necessary, store it + + # Check if vendor index is being tracked + if [[ -z "${vendor_index_map[$vendor]}" ]]; then + # Start new vendor index tracking + vendor_index_map[$vendor]=0 + else + # Another GPU of same vendor, increment index + vendor_index_map[$vendor]="$((vendor_index_map[$vendor] + 1))" + fi + + # Resolved GPU index + local gpu_index="${vendor_index_map[$vendor]}" + local gpu_key="$vendor:$gpu_index" + + # Store info in maps + gpu_map+=("$gpu_key") + gpu_bus_map["$gpu_key"]="$bus_info" + gpu_product_map["$gpu_key"]="$product" + vendor_full_map["$gpu_key"]="$vendor_full" + + # Clear values for additional GPUs + vendor="" + product="" + bus_info="" + vendor_full="" + fi + + if [[ "${line,,}" =~ \*"-display" ]]; then + # New GPU found before storing, clear incomplete values to prevent mixing + vendor="" + product="" + bus_info="" + vendor_full="" + fi + done < <(sudo lshw -c video) +} + +check_and_populate_gpus() { + if [[ "${#gpu_map[@]}" -eq 0 ]]; then + get_gpu_info # Gather info incase info not gathered yet + if [[ "${#gpu_map[@]}" -eq 0 ]]; then + echo "No GPUs found on this system" >&2 + return 1 + fi + fi +} + +check_selected_gpu() { + local selected_gpu="${1,,}" + + if [[ ! " ${gpu_map[*]} " =~ " $selected_gpu " ]]; then + echo "No such GPU: '$selected_gpu'" >&2 + return 1 + fi + + echo "$selected_gpu" +} + +list_available_gpus() { + if ! check_and_populate_gpus; then + return 1 + fi + + echo "Available GPUs:" >&2 + for gpu in "${gpu_map[@]}"; do + echo " [$gpu] \"${gpu_product_map[$gpu]}\" @[${gpu_bus_map[$gpu]}]" + done +} + +convert_bus_id_to_xorg() { + local bus_info="$1" + IFS=":." read -ra bus_parts <<< "${bus_info#pci@????:}" # Remove "pci@" and the following 4 characters (domain) + + # Check if bus_info has the correct format (at least 3 parts after removing domain) + if [[ "${#bus_parts[@]}" -lt 3 ]]; then + echo "Invalid bus info format: $bus_info" >&2 + return 1 + fi + + # Convert each part from hexadecimal to decimal + bus_info_xorg="PCI:" + for part in "${bus_parts[@]}"; do + bus_info_xorg+="$((16#$part)):" + done + bus_info_xorg="${bus_info_xorg%:}" # Remove the trailing colon + + echo "$bus_info_xorg" +} + +print_gpu_info() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo "[$selected_gpu]" + echo " Vendor: ${vendor_full_map[$selected_gpu]}" + echo " Product: ${gpu_product_map[$selected_gpu]}" + echo " Bus: ${gpu_bus_map[$selected_gpu]}" + echo +} + +get_gpu_vendor() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo "${selected_gpu%%:*}" +} + +get_gpu_vendor_full() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo "${vendor_full_map[$selected_gpu]}" +} + +get_gpu_index() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo "${selected_gpu#*:}" +} + +get_gpu_product() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo "${gpu_product_map[$selected_gpu]}" +} + +get_gpu_bus() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo "${gpu_bus_map[$selected_gpu]}" +} + +get_gpu_bus_xorg() { + if ! check_and_populate_gpus; then + return 1 + fi + + local selected_gpu + if ! selected_gpu=$(check_selected_gpu "$1"); then + return 1 + fi + + echo $(convert_bus_id_to_xorg "${gpu_bus_map[$selected_gpu]}") +} diff --git a/base.Dockerfile b/base.Dockerfile index eeec1420..e5b6daf1 100644 --- a/base.Dockerfile +++ b/base.Dockerfile @@ -203,6 +203,8 @@ ENV \ # Disable VSYNC for NVIDIA GPUs __GL_SYNC_TO_VBLANK=0 +COPY .patches /etc/ + #Build and install gpu-screen-recorder RUN apt-get update -y \ && apt-get install -y \ @@ -236,6 +238,8 @@ RUN apt-get update -y \ libxi-dev \ libxdamage-dev \ libxfixes-dev \ + libxi-dev \ + libxdamage-dev \ libpulse-dev \ libswresample-dev \ libva-dev \ @@ -255,6 +259,7 @@ RUN apt-get update -y \ && find . -maxdepth 1 -type f -name "*libnvrtc.so.*" -exec sh -c 'ln -snf $(basename {}) libnvrtc.so' \; \ && mkdir -p /usr/local/nvidia/lib && mv -f libnvrtc* /usr/local/nvidia/lib \ && git clone https://repo.dec05eba.com/gpu-screen-recorder && cd gpu-screen-recorder \ + && git apply /etc/connectcheckskip.patch \ && meson setup build \ && meson configure --prefix=/usr --buildtype=release -Dsystemd=true -Dstrip=true build \ && ninja -C build install diff --git a/server.Dockerfile b/server.Dockerfile index b2fcad14..c5cf10d2 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -20,6 +20,7 @@ RUN apt-get update -y \ mangohud \ gamescope \ openbox \ + lshw \ && setcap cap_sys_nice+ep /usr/games/gamescope \ && rm -rf /var/lib/apt/lists/* @@ -84,4 +85,4 @@ EXPOSE 8080/udp WORKDIR /home/${USERNAME} -ENTRYPOINT ["/usr/bin/supervisord"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/supervisord"]