Merge branch 'sonic-net:master' into fenpan_pubsubfix

.github/workflows/semgrep.yml

@@ -0,0 +1,22 @@
+name: Semgrep
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - master
+      - '201[7-9][0-1][0-9]'
+      - '202[0-9][0-1][0-9]'
+
+jobs:
+  semgrep:
+    if: github.repository_owner == 'sonic-net'
+    name: Semgrep
+    runs-on: ubuntu-latest
+    container:
+      image: returntocorp/semgrep
+    steps:
+      - uses: actions/checkout@v3
+      - run: semgrep ci
+        env:
+           SEMGREP_RULES: p/default

Makefile

@@ -82,17 +82,25 @@ endif
 swsscommon_wrap:
 	make -C swsscommon
 
-check_gotest:
+DBCONFG = $(DBDIR)/database_config.json
+ENVFILE = build/test/env.txt
+TESTENV = $(shell cat $(ENVFILE))
+
+$(DBCONFG): testdata/database_config.json
 	sudo mkdir -p ${DBDIR}
 	sudo cp ./testdata/database_config.json ${DBDIR}
-	sudo mkdir -p /usr/models/yang || true
-	sudo find $(MGMT_COMMON_DIR)/models -name '*.yang' -exec cp {} /usr/models/yang/ \;
+
+$(ENVFILE):
+	mkdir -p $(@D)
+	tools/test/env.sh | grep -v DB_CONFIG_PATH | tee $@
+
+check_gotest: $(DBCONFG) $(ENVFILE)
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-config.txt -covermode=atomic -v github.com/sonic-net/sonic-gnmi/sonic_db_config
-	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-gnmi.txt -covermode=atomic -mod=vendor $(BLD_FLAGS) -v github.com/sonic-net/sonic-gnmi/gnmi_server -coverpkg ../...
-	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-dialcout.txt -covermode=atomic -mod=vendor $(BLD_FLAGS) -v github.com/sonic-net/sonic-gnmi/dialout/dialout_client
+	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(TESTENV) $(GO) test -race -timeout 20m -coverprofile=coverage-gnmi.txt -covermode=atomic -mod=vendor $(BLD_FLAGS) -v github.com/sonic-net/sonic-gnmi/gnmi_server -coverpkg ../...
+	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(TESTENV) $(GO) test -coverprofile=coverage-dialcout.txt -covermode=atomic -mod=vendor $(BLD_FLAGS) -v github.com/sonic-net/sonic-gnmi/dialout/dialout_client
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-data.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_data_client
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-dbus.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_service_client
-	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-translutils.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/transl_utils
+	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(TESTENV) $(GO) test -race -coverprofile=coverage-translutils.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/transl_utils
 	$(GO) get github.com/axw/gocov/...
 	$(GO) get github.com/AlekSi/gocov-xml
 	$(GO) mod vendor

azure-pipelines.yml

@@ -154,7 +154,7 @@ stages:
 
     - script: |
         pushd sonic-gnmi
-        make check_gotest
+        make check_gotest ENABLE_TRANSLIB_WRITE=y
       displayName: "Test"
 
     - publish: $(Build.ArtifactStagingDirectory)/

gnmi_server/client_subscribe.go

@@ -5,11 +5,13 @@ import (
 	"io"
 	"net"
 	"sync"
+	"strings"
 
 	"github.com/Workiva/go-datastructures/queue"
 	log "github.com/golang/glog"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	sdc "github.com/sonic-net/sonic-gnmi/sonic_data_client"
 	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
 )
@@ -128,23 +130,23 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 		return grpc.Errorf(codes.InvalidArgument, "first message must be SubscriptionList: %q", query)
 	}
 
-	var target string
 	prefix := c.subscribe.GetPrefix()
-	if prefix == nil {
-		return grpc.Errorf(codes.Unimplemented, "No target specified in prefix")
-	} else {
-		target = prefix.GetTarget()
-		// TODO: add data client support for fetching non-db data
-		if target == "" {
-			return grpc.Errorf(codes.Unimplemented, "Empty target data not supported yet")
-		}
-	}
+	origin := prefix.GetOrigin()
+	target := prefix.GetTarget()
 
 	paths, err := c.populateDbPathSubscrition(c.subscribe)
 	if err != nil {
 		return grpc.Errorf(codes.NotFound, "Invalid subscription path: %v %q", err, query)
 	}
 
+	if o, err := ParseOrigin(paths); err != nil {
+		return err // origin conflict within paths
+	} else if len(origin) == 0 {
+		origin = o // Use origin from paths if not given in prefix
+	} else if len(o) != 0 && o != origin {
+		return status.Error(codes.InvalidArgument, "Origin conflict between prefix and paths")
+	}
+
 	if connectionKey, valid = connectionManager.Add(c.addr, query.String()); !valid {
 		return grpc.Errorf(codes.Unavailable, "Server connections are at capacity.")
 	}
@@ -155,15 +157,26 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 
 	mode := c.subscribe.GetMode()
 
-	if target == "OTHERS" {
+	log.V(3).Infof("mode=%v, origin=%q, target=%q", mode, origin, target)
+
+	if origin == "openconfig" {
+		dc, err = sdc.NewTranslClient(prefix, paths, ctx, extensions, sdc.TranslWildcardOption{})
+	} else if len(origin) != 0 {
+		return grpc.Errorf(codes.Unimplemented, "Unsupported origin: %s", origin)
+	} else if target == "" {
+		// This and subsequent conditions handle target based path identification
+		// when origin == "". As per the spec it should have been treated as "openconfig".
+		// But we take a deviation and stick to legacy logic for backward compatibility
+		return grpc.Errorf(codes.Unimplemented, "Empty target data not supported")
+	} else if target == "OTHERS" {
 		dc, err = sdc.NewNonDbClient(paths, prefix)
 	} else if ((target == "EVENTS") && (mode == gnmipb.SubscriptionList_STREAM)) {
 		dc, err = sdc.NewEventClient(paths, prefix, c.logLevel)
 	} else if _, ok, _, _ := sdc.IsTargetDb(target); ok {
 		dc, err = sdc.NewDbClient(paths, prefix)
 	} else {
 		/* For any other target or no target create new Transl Client. */
-		dc, err = sdc.NewTranslClient(prefix, paths, ctx, extensions)
+		dc, err = sdc.NewTranslClient(prefix, paths, ctx, extensions, sdc.TranslWildcardOption{})
 	}
 
 	if err != nil {
@@ -195,6 +208,9 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 	c.Close()
 	// Wait until all child go routines exited
 	c.w.Wait()
+	if strings.Contains(err.Error(), "i/o timeout") {
+		return grpc.Errorf(codes.Internal, "%s", err)
+	}
 	return grpc.Errorf(codes.InvalidArgument, "%s", err)
 }