Skip to content

Commit

Permalink
Update the Azure Pipeline build to use Bookworm and Ubuntu 22.04 (#937)
Browse files Browse the repository at this point in the history
Update the Azure Pipeline build to use Bookworm and Ubuntu 22.04

Signed-off-by: Saikrishna Arcot <[email protected]>
  • Loading branch information
saiarcot895 authored Dec 7, 2024
1 parent fb6ce44 commit 7425c42
Show file tree
Hide file tree
Showing 6 changed files with 273 additions and 87 deletions.
6 changes: 3 additions & 3 deletions .azure-pipelines/build-sairedis-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ jobs:
set -ex
sudo apt-get update
sudo apt-get install -qq -y \
qtbase5-dev \
libdbus-glib-1-dev \
libpcsclite-dev \
docbook-to-man \
Expand All @@ -90,7 +89,7 @@ jobs:
sudo mkdir -m 755 /var/run/sswsyncd
sudo apt-get install -y rsyslog
sudo service rsyslog start
sudo rsyslogd
displayName: "Install dependencies"
- task: DownloadPipelineArtifact@2
Expand Down Expand Up @@ -137,7 +136,8 @@ jobs:
displayName: "Compile sonic sairedis"
- script: |
sudo cp azsyslog.conf /etc/rsyslog.conf
sudo service rsyslog restart
sudo killall rsyslogd
sudo rsyslogd
displayName: "Update rsyslog.conf"
- ${{ if eq(parameters.run_unit_test, true) }}:
- script: |
Expand Down
14 changes: 2 additions & 12 deletions .azure-pipelines/build-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,16 +98,6 @@ jobs:
mv ../*.deb .
displayName: "Compile sonic swss common with coverage enabled"
- ${{ if eq(parameters.run_unit_test, true) }}:
- script: |
set -ex
git clone https://github.com/gcovr/gcovr.git
cd gcovr/
git checkout 5.2
sudo pip3 install setuptools
sudo python3 setup.py install
cd ..
sudo rm -rf gcovr
displayName: "Install gcovr 5.2 (for --exclude-throw-branches support)"
- script: |
set -ex
sudo pip install Pympler==0.8 pytest
Expand Down Expand Up @@ -142,9 +132,9 @@ jobs:
set -ex
# Install .NET CORE
curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
sudo apt-add-repository https://packages.microsoft.com/debian/12/prod
sudo apt-get update
sudo apt-get install -y dotnet-sdk-6.0
sudo apt-get install -y dotnet-sdk-8.0
displayName: "Install .NET CORE"
- task: PublishCodeCoverageResults@1
inputs:
Expand Down
24 changes: 22 additions & 2 deletions .azure-pipelines/docker-sonic-vs/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,31 @@
FROM docker-sonic-vs

ARG docker_container_name
ARG need_dbg

COPY ["debs", "/debs"]

# Remove existing packages first before installing the new/current packages. This is to overcome limitations with
# Docker's diff detection mechanism, where only the file size and the modification timestamp (which will remain the
# same, even though contents have changed) are checked between the previous and current layer.
RUN dpkg --purge libswsscommon python3-swsscommon sonic-db-cli libsaimetadata libsairedis libsaivs syncd-vs swss sonic-eventd libdashapi
RUN dpkg -i /debs/libdashapi_1.0.0_amd64.deb /debs/libswsscommon_1.0.0_amd64.deb /debs/python3-swsscommon_1.0.0_amd64.deb /debs/sonic-db-cli_1.0.0_amd64.deb /debs/libsaimetadata_1.0.0_amd64.deb /debs/libsairedis_1.0.0_amd64.deb /debs/libsaivs_1.0.0_amd64.deb /debs/syncd-vs_1.0.0_amd64.deb /debs/swss_1.0.0_amd64.deb
RUN dpkg --purge libswsscommon python3-swsscommon sonic-db-cli libsaimetadata libsairedis libsaivs syncd-vs swss sonic-eventd libdashapi

RUN apt-get update

RUN apt install -y /debs/libdashapi_1.0.0_amd64.deb \
/debs/libswsscommon_1.0.0_amd64.deb \
/debs/python3-swsscommon_1.0.0_amd64.deb \
/debs/sonic-db-cli_1.0.0_amd64.deb \
/debs/libsaimetadata_1.0.0_amd64.deb \
/debs/libsairedis_1.0.0_amd64.deb \
/debs/libsaivs_1.0.0_amd64.deb \
/debs/syncd-vs_1.0.0_amd64.deb \
/debs/swss_1.0.0_amd64.deb

RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/libswsscommon-dbgsym_1.0.0_amd64.deb ; fi

COPY ["start.sh", "/usr/bin/"]

RUN pip3 install scapy==2.5.0

RUN apt-get -y install software-properties-common libdatetime-perl libcapture-tiny-perl build-essential libcpanel-json-xs-perl git python3-protobuf
187 changes: 187 additions & 0 deletions .azure-pipelines/docker-sonic-vs/start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
#!/bin/bash -e

# Generate configuration

# NOTE: 'PLATFORM' and 'HWSKU' environment variables are set
# in the Dockerfile so that they persist for the life of the container

ln -sf /usr/share/sonic/device/$PLATFORM /usr/share/sonic/platform
ln -sf /usr/share/sonic/device/$PLATFORM/$HWSKU /usr/share/sonic/hwsku

SWITCH_TYPE=switch
PLATFORM_CONF=platform.json
if [[ $HWSKU == "DPU-2P" ]]; then
SWITCH_TYPE=dpu
PLATFORM_CONF=platform-dpu-2p.json
fi

pushd /usr/share/sonic/hwsku

# filter available front panel ports in lanemap.ini
[ -f lanemap.ini.orig ] || cp lanemap.ini lanemap.ini.orig
for p in $(ip link show | grep -oE "eth[0-9]+" | grep -v eth0); do
grep ^$p: lanemap.ini.orig
done > lanemap.ini

# filter available sonic front panel ports in port_config.ini
[ -f port_config.ini.orig ] || cp port_config.ini port_config.ini.orig
grep ^# port_config.ini.orig > port_config.ini
for lanes in $(awk -F ':' '{print $2}' lanemap.ini); do
grep -E "\s$lanes\s" port_config.ini.orig
done >> port_config.ini

popd

[ -d /etc/sonic ] || mkdir -p /etc/sonic

# Note: libswsscommon requires a dabase_config file in /var/run/redis/sonic-db/
# Prepare this file before any dependent application, such as sonic-cfggen
mkdir -p /var/run/redis/sonic-db
cp /etc/default/sonic-db/database_config.json /var/run/redis/sonic-db/

SYSTEM_MAC_ADDRESS=$(ip link show eth0 | grep ether | awk '{print $2}')
sonic-cfggen -t /usr/share/sonic/templates/init_cfg.json.j2 -a "{\"system_mac\": \"$SYSTEM_MAC_ADDRESS\", \"switch_type\": \"$SWITCH_TYPE\"}" > /etc/sonic/init_cfg.json

if [[ -f /usr/share/sonic/virtual_chassis/default_config.json ]]; then
sonic-cfggen -j /etc/sonic/init_cfg.json -j /usr/share/sonic/virtual_chassis/default_config.json --print-data > /tmp/init_cfg.json
mv /tmp/init_cfg.json /etc/sonic/init_cfg.json
fi

if [ -f /etc/sonic/config_db.json ]; then
sonic-cfggen -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db.json --print-data > /tmp/config_db.json
mv /tmp/config_db.json /etc/sonic/config_db.json
else
# generate and merge buffers configuration into config file
if [ -f /usr/share/sonic/hwsku/buffers.json.j2 ]; then
sonic-cfggen -k $HWSKU -p /usr/share/sonic/device/$PLATFORM/$PLATFORM_CONF -t /usr/share/sonic/hwsku/buffers.json.j2 > /tmp/buffers.json
buffers_cmd="-j /tmp/buffers.json"
fi
if [ -f /usr/share/sonic/hwsku/qos.json.j2 ]; then
sonic-cfggen -j /etc/sonic/init_cfg.json -t /usr/share/sonic/hwsku/qos.json.j2 > /tmp/qos.json
qos_cmd="-j /tmp/qos.json"
fi

sonic-cfggen -p /usr/share/sonic/device/$PLATFORM/$PLATFORM_CONF -k $HWSKU --print-data > /tmp/ports.json
# change admin_status from up to down; Test cases dependent
sed -i "s/up/down/g" /tmp/ports.json
sonic-cfggen -j /etc/sonic/init_cfg.json $buffers_cmd $qos_cmd -j /tmp/ports.json --print-data > /etc/sonic/config_db.json
fi

sonic-cfggen -t /usr/share/sonic/templates/copp_cfg.j2 > /etc/sonic/copp_cfg.json

if [ "$HWSKU" == "Mellanox-SN2700" ]; then
cp /usr/share/sonic/hwsku/sai_mlnx.profile /usr/share/sonic/hwsku/sai.profile
elif [ "$HWSKU" == "DPU-2P" ]; then
cp /usr/share/sonic/hwsku/sai_dpu_2p.profile /usr/share/sonic/hwsku/sai.profile
fi

mkdir -p /etc/swss/config.d/

rm -f /var/run/rsyslogd.pid

supervisorctl start rsyslogd

supervisord_cfg="/etc/supervisor/conf.d/supervisord.conf"
chassisdb_cfg_file="/usr/share/sonic/virtual_chassis/default_config.json"
chassisdb_cfg_file_default="/etc/default/sonic-db/default_chassis_cfg.json"
host_template="/usr/share/sonic/templates/hostname.j2"
db_cfg_file="/var/run/redis/sonic-db/database_config.json"
db_cfg_file_tmp="/var/run/redis/sonic-db/database_config.json.tmp"

if [ -r "$chassisdb_cfg_file" ]; then
echo $(sonic-cfggen -j $chassisdb_cfg_file -t $host_template) >> /etc/hosts
else
chassisdb_cfg_file="$chassisdb_cfg_file_default"
echo "10.8.1.200 redis_chassis.server" >> /etc/hosts
fi

supervisorctl start redis-server

start_chassis_db=`sonic-cfggen -v DEVICE_METADATA.localhost.start_chassis_db -y $chassisdb_cfg_file`
if [[ "$HOSTNAME" == *"supervisor"* ]] || [ "$start_chassis_db" == "1" ]; then
supervisorctl start redis-chassis
fi

conn_chassis_db=`sonic-cfggen -v DEVICE_METADATA.localhost.connect_to_chassis_db -y $chassisdb_cfg_file`
if [ "$start_chassis_db" != "1" ] && [ "$conn_chassis_db" != "1" ]; then
cp $db_cfg_file $db_cfg_file_tmp
update_chassisdb_config -j $db_cfg_file_tmp -d
cp $db_cfg_file_tmp $db_cfg_file
fi

if [ "$conn_chassis_db" == "1" ]; then
if [ -f /usr/share/sonic/virtual_chassis/coreportindexmap.ini ]; then
cp /usr/share/sonic/virtual_chassis/coreportindexmap.ini /usr/share/sonic/hwsku/

pushd /usr/share/sonic/hwsku

# filter available front panel ports in coreportindexmap.ini
[ -f coreportindexmap.ini.orig ] || cp coreportindexmap.ini coreportindexmap.ini.orig
for p in $(ip link show | grep -oE "eth[0-9]+" | grep -v eth0); do
grep ^$p: coreportindexmap.ini.orig
done > coreportindexmap.ini

popd
fi
fi

/usr/bin/configdb-load.sh

if [ "$HWSKU" = "brcm_gearbox_vs" ]; then
supervisorctl start gbsyncd
supervisorctl start gearsyncd
fi

supervisorctl start syncd

supervisorctl start portsyncd

supervisorctl start orchagent

supervisorctl start coppmgrd

supervisorctl start neighsyncd

supervisorctl start fdbsyncd

supervisorctl start teamsyncd

supervisorctl start fpmsyncd

supervisorctl start teammgrd

supervisorctl start vrfmgrd

supervisorctl start portmgrd

supervisorctl start intfmgrd

supervisorctl start vlanmgrd

supervisorctl start zebra

supervisorctl start mgmtd

supervisorctl start staticd

supervisorctl start buffermgrd

supervisorctl start nbrmgrd

supervisorctl start vxlanmgrd

supervisorctl start sflowmgrd

supervisorctl start natmgrd

supervisorctl start natsyncd

supervisorctl start tunnelmgrd

supervisorctl start fabricmgrd

# Start arp_update when VLAN exists
VLAN=`sonic-cfggen -d -v 'VLAN.keys() | join(" ") if VLAN'`
if [ "$VLAN" != "" ]; then
supervisorctl start arp_update
fi
38 changes: 10 additions & 28 deletions .azure-pipelines/test-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
displayName: vstest
timeoutInMinutes: ${{ parameters.timeout }}

pool: sonic-common
pool: sonictest

steps:
- checkout: self
Expand Down Expand Up @@ -52,11 +52,10 @@ jobs:

- script: |
set -ex
ls -l
sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh
sudo apt-get install -y libhiredis0.14 libyang0.16
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb
sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
Expand All @@ -72,34 +71,17 @@ jobs:
sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
docker ps
ip netns list
sudo /sbin/ip link add Vrf1 type vrf table 1001 || { echo 'vrf command failed' ; exit 1; }
sudo /sbin/ip link del Vrf1 type vrf table 1001
pushd sonic-swss/tests
# run pytests in sets of 20
all_tests=$(ls test_*.py)
all_tests=$(ls test_*.py | xargs)
all_tests="${all_tests} p4rt dash"
test_set=()
for test in ${all_tests}; do
test_set+=("${test}")
if [ ${#test_set[@]} -ge 20 ]; then
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
container_count=$(docker ps -q -a | wc -l)
if [ ${container_count} -gt 0 ]; then
docker stop $(docker ps -q -a)
docker rm $(docker ps -q -a)
fi
test_set=()
fi
done
if [ ${#test_set[@]} -gt 0 ]; then
test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
container_count=$(docker ps -q -a | wc -l)
if [ ${container_count} -gt 0 ]; then
docker stop $(docker ps -q -a)
docker rm $(docker ps -q -a)
fi
fi
# Run the tests in parallel and retry
retry=3
IMAGE_NAME=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
echo $all_tests | xargs -n 1 | xargs -P 8 -I TEST_MODULE sudo ./run-tests.sh "$IMAGE_NAME" "--force-recreate-dvs" "TEST_MODULE" 3
rm -rf $(Build.ArtifactStagingDirectory)/download
displayName: "Run vs tests"
Expand Down
Loading

0 comments on commit 7425c42

Please sign in to comment.