Merge branch 'master' into rpc_from_proto

.github/workflows/codeql-analysis.yml

@@ -13,6 +13,10 @@ on:
       - '202[0-9][0-9][0-9]'
   workflow_dispatch:
 
+env:
+  BUILD_BRANCH: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || github.ref_name }}
+
+
 jobs:
   analyze:
     name: Analyze
@@ -31,6 +35,20 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    # Checkout sonic-mgmt-common repository which is used by sonic-gnmi
+    - name: Checkout sonic-mgmt-common repository
+      uses: actions/checkout@v3
+      with:
+        repository: sonic-net/sonic-mgmt-common
+        path: sonic-mgmt-common
+        ref: refs/heads/${{ env.BUILD_BRANCH }}
+
+    # Update go.mod to use local sonic-mgmt-common.
+    # This is the same hack used in the CI pipeline. See lgtm.yml.
+    # We should find a better way to do this.
+    - name: Update go.mod for sonic-mgmt-common
+      run: sed -i 's@replace github.com/Azure/sonic-mgmt-common => ../sonic-mgmt-common@replace github.com/Azure/sonic-mgmt-common => ./sonic-mgmt-common@g' go.mod
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/[email protected]

.gitignore

@@ -1,11 +1,26 @@
+# Build directory
 build/
+
+# Debian packaging files
 debian/.debhelper/
 debian/files
 debian/sonic-telemetry.debhelper.log
 debian/sonic-telemetry.substvars
 debian/sonic-telemetry/
-vendor
-src
-cvl
-translib
-__pycache__
+debian/sonic-gnmi.debhelper.log
+debian/sonic-gnmi.substvars
+debian/sonic-gnmi/
+
+# Vendor directory
+vendor/
+
+# Source directories
+src/
+cvl/
+translib/
+
+# SWSS common generated files
+swsscommon/swsscommon.go
+swsscommon/swsscommon.i
+swsscommon/swsscommon_wrap.cxx
+swsscommon/swsscommon_wrap.h

Makefile

@@ -27,6 +27,8 @@ export GOBIN := $(abspath $(BUILD_DIR))
 export PATH := $(PATH):$(GOBIN):$(shell dirname $(GO))
 export CGO_LDFLAGS := -lswsscommon -lhiredis
 export CGO_CXXFLAGS := -I/usr/include/swss -w -Wall -fpermissive
+export MEMCHECK_CGO_LDFLAGS := $(CGO_LDFLAGS) -fsanitize=address
+export MEMCHECK_CGO_CXXFLAGS := $(CGO_CXXFLAGS) -fsanitize=leak
 
 ifeq ($(ENABLE_TRANSLIB_WRITE),y)
 BLD_TAGS := gnmi_translib_write
@@ -39,6 +41,11 @@ ifneq ($(BLD_TAGS),)
 BLD_FLAGS := -tags "$(strip $(BLD_TAGS))"
 endif
 
+MEMCHECK_TAGS := $(BLD_TAGS) gnmi_memcheck
+ifneq ($(MEMCHECK_TAGS),)
+MEMCHECK_FLAGS := -tags "$(strip $(MEMCHECK_TAGS))"
+endif
+
 ENABLE_DIALOUT_VALUE := 1
 ifeq ($(ENABLE_DIALOUT),n)
 ENABLE_DIALOUT_VALUE = 0
@@ -55,18 +62,18 @@ go.mod:
 
 $(GO_DEPS): go.mod $(PATCHES) swsscommon_wrap $(GNOI_YANG)
 	$(GO) mod vendor
-	$(GO) mod download golang.org/x/[email protected]
-	$(GO) mod download github.com/jipanyang/[email protected]
-	cp -r $(GOPATH)/pkg/mod/golang.org/x/[email protected]/* vendor/golang.org/x/crypto/
-	cp -r $(GOPATH)/pkg/mod/github.com/jipanyang/[email protected]/* vendor/github.com/jipanyang/gnxi/
+	$(GO) mod download github.com/google/[email protected]
+	cp -r $(GOPATH)/pkg/mod/github.com/google/[email protected]/* vendor/github.com/google/gnxi/
+
+# Apply patch from sonic-mgmt-common, ignore glog.patch because glog version changed
+	sed -i 's/patch -d $${DEST_DIR}\/github.com\/golang\/glog/\#patch -d $${DEST_DIR}\/github.com\/golang\/glog/g' $(MGMT_COMMON_DIR)/patches/apply.sh
 	$(MGMT_COMMON_DIR)/patches/apply.sh vendor
+	sed -i 's/#patch -d $${DEST_DIR}\/github.com\/golang\/glog/patch -d $${DEST_DIR}\/github.com\/golang\/glog/g' $(MGMT_COMMON_DIR)/patches/apply.sh
+
 	chmod -R u+w vendor
-	patch -d vendor -p0 < patches/gnmi_cli.all.patch
-	patch -d vendor -p0 < patches/gnmi_set.patch
-	patch -d vendor -p0 < patches/gnmi_get.patch
 	patch -d vendor -p0 < patches/gnmi_path.patch
 	patch -d vendor -p0 < patches/gnmi_xpath.patch
-	git apply patches/0001-Updated-to-filter-and-write-to-file.patch
+
 	touch $@
 
 go-deps: $(GO_DEPS)
@@ -75,31 +82,57 @@ go-deps-clean:
 	$(RM) -r vendor
 
 sonic-gnmi: $(GO_DEPS)
+# advancetls 1.0.0 release need following patch to build by go-1.19
+	patch -d vendor -p0 < patches/0002-Fix-advance-tls-build-with-go-119.patch
+# build service first which depends on advancetls
 ifeq ($(CROSS_BUILD_ENVIRON),y)
 	$(GO) build -o ${GOBIN}/telemetry -mod=vendor $(BLD_FLAGS) github.com/sonic-net/sonic-gnmi/telemetry
 ifneq ($(ENABLE_DIALOUT_VALUE),0)
 	$(GO) build -o ${GOBIN}/dialout_client_cli -mod=vendor $(BLD_FLAGS) github.com/sonic-net/sonic-gnmi/dialout/dialout_client_cli
 endif
-	$(GO) build -o ${GOBIN}/gnmi_get -mod=vendor github.com/jipanyang/gnxi/gnmi_get
-	$(GO) build -o ${GOBIN}/gnmi_set -mod=vendor github.com/jipanyang/gnxi/gnmi_set
-	$(GO) build -o ${GOBIN}/gnmi_cli -mod=vendor github.com/openconfig/gnmi/cmd/gnmi_cli
 	$(GO) build -o ${GOBIN}/gnoi_client -mod=vendor github.com/sonic-net/sonic-gnmi/gnoi_client
 	$(GO) build -o ${GOBIN}/gnmi_dump -mod=vendor github.com/sonic-net/sonic-gnmi/gnmi_dump
 else
 	$(GO) install -mod=vendor $(BLD_FLAGS) github.com/sonic-net/sonic-gnmi/telemetry
 ifneq ($(ENABLE_DIALOUT_VALUE),0)
 	$(GO) install -mod=vendor $(BLD_FLAGS) github.com/sonic-net/sonic-gnmi/dialout/dialout_client_cli
 endif
-	$(GO) install -mod=vendor github.com/jipanyang/gnxi/gnmi_get
-	$(GO) install -mod=vendor github.com/jipanyang/gnxi/gnmi_set
-	$(GO) install -mod=vendor github.com/openconfig/gnmi/cmd/gnmi_cli
 	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/gnoi_client
 	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/gnmi_dump
 	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/build/gnoi_yang/client/gnoi_openconfig_client
 	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/build/gnoi_yang/client/gnoi_sonic_client
 
 endif
 
+# download and apply patch for gnmi client, which will break advancetls
+# backup crypto and gnxi
+	mkdir backup_crypto
+	cp -r vendor/golang.org/x/crypto/* backup_crypto/
+
+# download and patch crypto and gnxi
+	$(GO) mod download golang.org/x/[email protected]
+	cp -r $(GOPATH)/pkg/mod/golang.org/x/[email protected]/* vendor/golang.org/x/crypto/
+	chmod -R u+w vendor
+	patch -d vendor -p0 < patches/gnmi_cli.all.patch
+	patch -d vendor -p0 < patches/gnmi_set.patch
+	patch -d vendor -p0 < patches/gnmi_get.patch
+	git apply patches/0001-Updated-to-filter-and-write-to-file.patch
+	git apply patches/0003-Fix-client-json-parsing-issue.patch
+
+ifeq ($(CROSS_BUILD_ENVIRON),y)
+	$(GO) build -o ${GOBIN}/gnmi_get -mod=vendor github.com/google/gnxi/gnmi_get
+	$(GO) build -o ${GOBIN}/gnmi_set -mod=vendor github.com/google/gnxi/gnmi_set
+	$(GO) build -o ${GOBIN}/gnmi_cli -mod=vendor github.com/openconfig/gnmi/cmd/gnmi_cli
+else
+	$(GO) install -mod=vendor github.com/google/gnxi/gnmi_get
+	$(GO) install -mod=vendor github.com/google/gnxi/gnmi_set
+	$(GO) install -mod=vendor github.com/openconfig/gnmi/cmd/gnmi_cli
+endif
+
+# restore old version
+	rm -rf vendor/golang.org/x/crypto/
+	mv backup_crypto/ vendor/golang.org/x/crypto/
+
 swsscommon_wrap:
 	make -C swsscommon
 
@@ -178,12 +211,19 @@ endif
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-data.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_data_client
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-dbus.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_service_client
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(TESTENV) $(GO) test -race -coverprofile=coverage-translutils.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/transl_utils
-	$(GO) get github.com/axw/gocov/...
-	$(GO) get github.com/AlekSi/gocov-xml
+	$(GO) install github.com/axw/gocov/[email protected]
+	$(GO) install github.com/AlekSi/gocov-xml@latest
 	$(GO) mod vendor
 	gocov convert coverage-*.txt | gocov-xml -source $(shell pwd) > coverage.xml
 	rm -rf coverage-*.txt 
 
+check_memleak: $(DBCONFG) $(ENVFILE)
+	sudo CGO_LDFLAGS="$(MEMCHECK_CGO_LDFLAGS)" CGO_CXXFLAGS="$(MEMCHECK_CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-telemetry.txt -covermode=atomic -mod=vendor $(MEMCHECK_FLAGS) -v github.com/sonic-net/sonic-gnmi/telemetry
+	sudo CGO_LDFLAGS="$(MEMCHECK_CGO_LDFLAGS)" CGO_CXXFLAGS="$(MEMCHECK_CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-config.txt -covermode=atomic $(MEMCHECK_FLAGS) -v github.com/sonic-net/sonic-gnmi/sonic_db_config
+	sudo CGO_LDFLAGS="$(MEMCHECK_CGO_LDFLAGS)" CGO_CXXFLAGS="$(MEMCHECK_CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-gnmi.txt -covermode=atomic -mod=vendor $(MEMCHECK_FLAGS) -v github.com/sonic-net/sonic-gnmi/gnmi_server -coverpkg ../... -run TestGNMINative
+	sudo CGO_LDFLAGS="$(MEMCHECK_CGO_LDFLAGS)" CGO_CXXFLAGS="$(MEMCHECK_CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-data.txt -covermode=atomic -mod=vendor $(MEMCHECK_FLAGS) -v github.com/sonic-net/sonic-gnmi/sonic_data_client
+
+
 clean:
 	$(RM) -r build
 	$(RM) -r vendor

azure-pipelines.yml

@@ -23,6 +23,8 @@ variables:
       value: $(System.PullRequest.TargetBranch)
     ${{ else }}:
       value: $(Build.SourceBranchName)
+  - name: UNIT_TEST_FLAG
+    value: 'ENABLE_TRANSLIB_WRITE=y'
 
 resources:
   repositories:
@@ -53,7 +55,7 @@ stages:
       DIFF_COVER_WORKING_DIRECTORY: $(System.DefaultWorkingDirectory)/sonic-gnmi
 
     container:
-      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bullseye:latest
+      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bookworm:latest
 
     steps:
     - checkout: self
@@ -80,14 +82,15 @@ stages:
         runVersion: 'latestFromBranch'
         runBranch: 'refs/heads/$(BUILD_BRANCH)'
         patterns: |
-            target/debs/bullseye/libyang*.deb
-            target/debs/bullseye/libnl*.deb
-            target/python-wheels/bullseye/sonic_yang_models*.whl
-      displayName: "Download bullseye debs"
+            target/debs/bookworm/libyang*.deb
+            target/debs/bookworm/libnl*.deb
+            target/python-wheels/bookworm/sonic_yang_models*.whl
+      displayName: "Download bookworm debs"
 
     - script: |
         # PYTEST
         sudo pip3 install -U pytest
+        sudo pip3 install -U jsonpatch
 
         # REDIS
         sudo apt-get update
@@ -98,39 +101,39 @@ stages:
         sudo service redis-server start
 
         # LIBYANG
-        sudo dpkg -i ../target/debs/bullseye/libyang*1.0.73*.deb
+        sudo dpkg -i ../target/debs/bookworm/libyang*1.0.73*.deb
       displayName: "Install dependency"
 
     - script: |
         # SONIC YANGS
         set -ex
-        sudo pip3 install ../target/python-wheels/bullseye/sonic_yang_models-1.0-py3-none-any.whl
+        sudo pip3 install ../target/python-wheels/bookworm/sonic_yang_models-1.0-py3-none-any.whl
       displayName: "Install sonic yangs"
 
     - script: |
         # LIBSWSSCOMMON
         sudo apt-get -y purge libnl-3-dev libnl-route-3-dev
-        sudo dpkg -i ../target/debs/bullseye/libnl-3-200_*.deb
-        sudo dpkg -i ../target/debs/bullseye/libnl-genl-3-200_*.deb
-        sudo dpkg -i ../target/debs/bullseye/libnl-route-3-200_*.deb
-        sudo dpkg -i ../target/debs/bullseye/libnl-nf-3-200_*.deb
+        sudo dpkg -i ../target/debs/bookworm/libnl-3-200_*.deb
+        sudo dpkg -i ../target/debs/bookworm/libnl-genl-3-200_*.deb
+        sudo dpkg -i ../target/debs/bookworm/libnl-route-3-200_*.deb
+        sudo dpkg -i ../target/debs/bookworm/libnl-nf-3-200_*.deb
       displayName: "Install libswsscommon dependencies"
 
     - script: |
         set -ex
         # Install .NET CORE
         curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-        sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
+        sudo apt-add-repository https://packages.microsoft.com/debian/12/prod
         sudo apt-get update
-        sudo apt-get install -y dotnet-sdk-5.0
+        sudo apt-get install -y dotnet-sdk-8.0
       displayName: "Install .NET CORE"
 
     - task: DownloadPipelineArtifact@2
       inputs:
         source: specific
         project: build
         pipeline: Azure.sonic-swss-common
-        artifact: sonic-swss-common
+        artifact: sonic-swss-common-bookworm
         runVersion: 'latestFromBranch'
         runBranch: 'refs/heads/$(BUILD_BRANCH)'
       displayName: "Download sonic-swss-common"
@@ -166,8 +169,13 @@ stages:
 
     - script: |
         pushd sonic-gnmi
-        make check_gotest ENABLE_TRANSLIB_WRITE=y
-      displayName: "Test"
+        make all && make check_memleak $(UNIT_TEST_FLAG)
+      displayName: "Check memory leak"
+
+    - script: |
+        pushd sonic-gnmi
+        make all && make check_gotest $(UNIT_TEST_FLAG)
+      displayName: "Run unit test"
 
     - publish: $(Build.ArtifactStagingDirectory)/
       artifact: sonic-gnmi

common_utils/context.go

@@ -6,11 +6,10 @@ import (
 	"sync/atomic"
 )
 
-
 // AuthInfo holds data about the authenticated user
 type AuthInfo struct {
 	// Username
-	User string
+	User        string
 	AuthEnabled bool
 	// Roles
 	Roles []string
@@ -37,6 +36,7 @@ const requestContextKey contextkey = 0
 var requestCounter uint64
 
 type CounterType int
+
 const (
 	GNMI_GET CounterType = iota
 	GNMI_GET_FAIL
@@ -51,6 +51,13 @@ const (
 	DBUS_DELETE_CHECKPOINT
 	DBUS_CONFIG_SAVE
 	DBUS_CONFIG_RELOAD
+	DBUS_STOP_SERVICE
+	DBUS_RESTART_SERVICE
+	DBUS_FILE_STAT
+	DBUS_HALT_SYSTEM
+	DBUS_IMAGE_DOWNLOAD
+	DBUS_IMAGE_INSTALL
+	DBUS_IMAGE_LIST
 	COUNTER_SIZE
 )
 
@@ -82,14 +89,27 @@ func (c CounterType) String() string {
 		return "DBUS config save"
 	case DBUS_CONFIG_RELOAD:
 		return "DBUS config reload"
+	case DBUS_STOP_SERVICE:
+		return "DBUS stop service"
+	case DBUS_RESTART_SERVICE:
+		return "DBUS restart service"
+	case DBUS_FILE_STAT:
+		return "DBUS file stat"
+	case DBUS_HALT_SYSTEM:
+		return "DBUS halt system"
+	case DBUS_IMAGE_DOWNLOAD:
+		return "DBUS image download"
+	case DBUS_IMAGE_INSTALL:
+		return "DBUS image install"
+	case DBUS_IMAGE_LIST:
+		return "DBUS image list"
 	default:
 		return ""
 	}
 }
 
 var globalCounters [COUNTER_SIZE]uint64
 
-
 // GetContext function returns the RequestContext object for a
 // gRPC request. RequestContext is maintained as a context value of
 // the request. Creates a new RequestContext object is not already
@@ -125,4 +145,3 @@ func IncCounter(cnt CounterType) {
 	atomic.AddUint64(&globalCounters[cnt], 1)
 	SetMemCounters(&globalCounters)
 }
-