fixed golang license extraction

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Upcoming changes...
 
+## [0.7.2] - 2023-11-27
+### Added
+- Added SQL tracing support for advanced debug
+### Fixed
+- Fixed issue with golang license reporting for certain components
+
 ## [0.7.1] - 2023-11-20
 ### Added
 - Added Open Telemetry spans/traces/metrics
@@ -50,3 +56,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [0.6.0]: https://github.com/scanoss/papi/compare/v0.5.0...v0.6.0
 [0.7.0]: https://github.com/scanoss/papi/compare/v0.6.0...v0.7.0
 [0.7.1]: https://github.com/scanoss/papi/compare/v0.7.0...v0.7.1
+[0.7.2]: https://github.com/scanoss/papi/compare/v0.7.1...v0.7.2

config/app-config-dev.json

@@ -15,7 +15,8 @@
     "Host":   "localhost",
     "User":   "scanoss",
     "Passwd": "secret123!",
-    "Schema": "scanoss"
+    "Schema": "scanoss",
+    "Trace": false
   },
   "Components": {
     "CommitMissing": false

config/app-config-docker-local-dev.json

@@ -11,7 +11,8 @@
     "Host":   "host.docker.internal",
     "User":   "scanoss",
     "Passwd": "secret123!",
-    "Schema": "scanoss"
+    "Schema": "scanoss",
+    "Trace": false
   },
   "Components": {
     "CommitMissing": false

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/package-url/packageurl-go v0.1.2
-	github.com/scanoss/go-grpc-helper v0.2.0
+	github.com/scanoss/go-grpc-helper v0.3.0
 	github.com/scanoss/go-purl-helper v0.2.0
 	github.com/scanoss/papi v0.2.0
 	github.com/scanoss/zap-logging-helper v0.3.0

go.sum

@@ -641,8 +641,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
-github.com/scanoss/go-grpc-helper v0.2.0 h1:Dy5tTU2T/7cti65m8UJNN5xIB+VXl8KXFQhg4ILe31I=
-github.com/scanoss/go-grpc-helper v0.2.0/go.mod h1:bUkHdBfMhZnkYXTbELr0FUe7GntzcDIWM5Q7b9kuFaQ=
+github.com/scanoss/go-grpc-helper v0.3.0 h1:4FeNOUX1X1uxZCd8pWz0iYwoPsH7cm4jK0yM3Mbuuvk=
+github.com/scanoss/go-grpc-helper v0.3.0/go.mod h1:bUkHdBfMhZnkYXTbELr0FUe7GntzcDIWM5Q7b9kuFaQ=
 github.com/scanoss/go-purl-helper v0.2.0 h1:kJBVv3euNhKQunoVFxiHxZLNHgSPAwTbz823oa3wcgw=
 github.com/scanoss/go-purl-helper v0.2.0/go.mod h1:lAtt1KcKIUylaj8YW7Qn/jq5Vvwh8fV2huqfvT9SKLM=
 github.com/scanoss/ipfilter/v2 v2.0.2 h1:GaB9i8kVJg9JQZm5XGStYkEpiaCVdsrj7ezI2wV/oh8=

pkg/models/all_urls.go

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Copyright (C) 2018-2022 SCANOSS.COM
+ * Copyright (C) 2018-2023 SCANOSS.COM
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -23,6 +23,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/scanoss/go-grpc-helper/pkg/grpc/database"
+
 	"github.com/Masterminds/semver/v3"
 	"github.com/jmoiron/sqlx"
 	purlutils "github.com/scanoss/go-purl-helper/pkg"
@@ -35,6 +37,7 @@ type AllUrlsModel struct {
 	conn       *sqlx.Conn
 	project    *ProjectModel
 	golangProj *GolangProjects
+	q          *database.DBQueryContext
 }
 
 type AllURL struct {
@@ -59,8 +62,8 @@ const (
 )
 
 // NewAllURLModel creates a new instance of the 'All URL' Model.
-func NewAllURLModel(ctx context.Context, s *zap.SugaredLogger, conn *sqlx.Conn, project *ProjectModel, golangProj *GolangProjects) *AllUrlsModel {
-	return &AllUrlsModel{ctx: ctx, s: s, conn: conn, project: project, golangProj: golangProj}
+func NewAllURLModel(ctx context.Context, s *zap.SugaredLogger, conn *sqlx.Conn, project *ProjectModel, golangProj *GolangProjects, q *database.DBQueryContext) *AllUrlsModel {
+	return &AllUrlsModel{ctx: ctx, s: s, conn: conn, project: project, golangProj: golangProj, q: q}
 }
 
 // GetURLsByPurlString searches for component details of the specified Purl string (and optional requirement).
@@ -91,9 +94,13 @@ func (m *AllUrlsModel) GetURLsByPurlString(purlString, purlReq string) (AllURL,
 	}
 	if purl.Type == "golang" {
 		allURL, err := m.golangProj.GetGoLangURLByPurl(purl, purlName, purlReq) // Search a separate table for golang dependencies
-		// If no golang package is found, but it's a GitHub component, search GitHub for it
-		if err == nil && allURL.Component == "" && strings.HasPrefix(purlString, "pkg:golang/github.com/") {
-			m.s.Debugf("Didn't find golang component in projects table for %v. Checking all urls...", purlString)
+		// If no golang package/license is found, but it's a GitHub component, search GitHub for it
+		if err == nil && (len(allURL.Component) == 0 || len(allURL.License) == 0) && strings.HasPrefix(purlString, "pkg:golang/github.com/") {
+			if len(allURL.Component) == 0 {
+				m.s.Debugf("Didn't find component in golang projects table for %v. Checking all urls...", purlString)
+			} else if len(allURL.License) == 0 {
+				m.s.Debugf("Didn't find license in golang projects table for %v. Checking all urls...", purlString)
+			}
 			purlString = purlutils.ConvertGoPurlStringToGithub(purlString) // Convert to GitHub purl
 			purl, err = purlutils.PurlFromString(purlString)
 			if err != nil {
@@ -124,17 +131,11 @@ func (m *AllUrlsModel) GetURLsByPurlNameType(purlName, purlType, purlReq string)
 		m.s.Errorf("Please specify a valid Purl Type to query: %v", purlName)
 		return AllURL{}, errors.New("please specify a valid Purl Type to query")
 	}
+	query := purlSQLQuerySelect + licSpdxSQLQuery + " purl_name, mine_id FROM all_urls u" +
+		mineLeftJoinSQL + licLeftJoinSQL + verLeftJoinSQL +
+		" WHERE m.purl_type = $1 AND u.purl_name = $2 ORDER BY date DESC"
 	var allUrls []AllURL
-	err := m.conn.SelectContext(m.ctx, &allUrls,
-		purlSQLQuerySelect+
-			licSpdxSQLQuery+
-			" purl_name, mine_id FROM all_urls u"+
-			mineLeftJoinSQL+
-			licLeftJoinSQL+
-			verLeftJoinSQL+
-			" WHERE m.purl_type = $1 AND u.purl_name = $2"+
-			" ORDER BY date DESC",
-		purlType, purlName)
+	err := m.q.SelectContext(m.ctx, &allUrls, query, purlType, purlName)
 	if err != nil {
 		m.s.Errorf("Failed to query all urls table for %v - %v: %v", purlType, purlName, err)
 		return AllURL{}, fmt.Errorf("failed to query the all urls table: %v", err)
@@ -158,17 +159,11 @@ func (m *AllUrlsModel) GetURLsByPurlNameTypeVersion(purlName, purlType, purlVers
 		m.s.Error("Please specify a valid Purl Version to query")
 		return AllURL{}, errors.New("please specify a valid Purl Version to query")
 	}
+	query := purlSQLQuerySelect + licSpdxSQLQuery + " purl_name, mine_id FROM all_urls u" +
+		mineLeftJoinSQL + licLeftJoinSQL + verLeftJoinSQL +
+		" WHERE m.purl_type = $1 AND u.purl_name = $2 AND v.version_name = $3 ORDER BY date DESC"
 	var allUrls []AllURL
-	err := m.conn.SelectContext(m.ctx, &allUrls,
-		purlSQLQuerySelect+
-			licSpdxSQLQuery+
-			" purl_name, mine_id FROM all_urls u"+
-			mineLeftJoinSQL+
-			licLeftJoinSQL+
-			verLeftJoinSQL+
-			" WHERE m.purl_type = $1 AND u.purl_name = $2 AND v.version_name = $3"+
-			" ORDER BY date DESC",
-		purlType, purlName, purlVersion)
+	err := m.q.SelectContext(m.ctx, &allUrls, query, purlType, purlName, purlVersion)
 	if err != nil {
 		m.s.Errorf("Failed to query all urls table for %v - %v: %v", purlType, purlName, err)
 		return AllURL{}, fmt.Errorf("failed to query the all urls table: %v", err)

pkg/models/all_urls_test.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/scanoss/go-grpc-helper/pkg/grpc/database"
+
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
 	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
 	myconfig "scanoss.com/dependencies/pkg/config"
@@ -48,7 +50,8 @@ func TestAllUrlsSearch(t *testing.T) {
 	}
 	myConfig.Components.CommitMissing = true
 	myConfig.Database.Trace = true
-	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), NewGolangProjectModel(ctx, s, conn, myConfig))
+	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn),
+		NewGolangProjectModel(ctx, s, conn, myConfig), database.NewDBSelectContext(s, conn, myConfig.Database.Trace))
 
 	allUrls, err := allUrlsModel.GetURLsByPurlNameType("tablestyle", "gem", "")
 	if err != nil {
@@ -143,7 +146,8 @@ func TestAllUrlsSearchVersion(t *testing.T) {
 	}
 	myConfig.Components.CommitMissing = true
 	myConfig.Database.Trace = true
-	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), NewGolangProjectModel(ctx, s, conn, myConfig))
+	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn),
+		NewGolangProjectModel(ctx, s, conn, myConfig), database.NewDBSelectContext(s, conn, myConfig.Database.Trace))
 
 	allUrls, err := allUrlsModel.GetURLsByPurlNameTypeVersion("tablestyle", "gem", "0.0.12")
 	if err != nil {
@@ -210,7 +214,8 @@ func TestAllUrlsSearchVersionRequirement(t *testing.T) {
 	}
 	myConfig.Components.CommitMissing = true
 	myConfig.Database.Trace = true
-	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), NewGolangProjectModel(ctx, s, conn, myConfig))
+	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn),
+		NewGolangProjectModel(ctx, s, conn, myConfig), database.NewDBSelectContext(s, conn, myConfig.Database.Trace))
 
 	allUrls, err := allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", ">0.0.4")
 	if err != nil {
@@ -252,7 +257,7 @@ func TestAllUrlsSearchNoProject(t *testing.T) {
 	}
 	myConfig.Components.CommitMissing = true
 	myConfig.App.Trace = true
-	allUrlsModel := NewAllURLModel(ctx, s, conn, nil, NewGolangProjectModel(ctx, s, conn, myConfig))
+	allUrlsModel := NewAllURLModel(ctx, s, conn, nil, NewGolangProjectModel(ctx, s, conn, myConfig), database.NewDBSelectContext(s, conn, myConfig.Database.Trace))
 
 	allUrls, err := allUrlsModel.GetURLsByPurlNameType("tablestyle", "gem", "0.0.8")
 	if err != nil {
@@ -286,7 +291,8 @@ func TestAllUrlsSearchNoLicense(t *testing.T) {
 	}
 	myConfig.Components.CommitMissing = true
 	myConfig.App.Trace = true
-	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), NewGolangProjectModel(ctx, s, conn, myConfig))
+	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn),
+		NewGolangProjectModel(ctx, s, conn, myConfig), database.NewDBSelectContext(s, conn, myConfig.Database.Trace))
 
 	allUrls, err := allUrlsModel.GetURLsByPurlString("pkg:gem/[email protected]", "")
 	if err != nil {
@@ -316,7 +322,8 @@ func TestAllUrlsSearchBadSql(t *testing.T) {
 	}
 	myConfig.Components.CommitMissing = true
 	myConfig.App.Trace = true
-	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), NewGolangProjectModel(ctx, s, conn, myConfig))
+	allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn),
+		NewGolangProjectModel(ctx, s, conn, myConfig), database.NewDBSelectContext(s, conn, myConfig.Database.Trace))
 	_, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", "")
 	if err == nil {
 		t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error")

pkg/models/common.go

@@ -22,11 +22,8 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"regexp"
 	"testing"
 
-	"go.uber.org/zap"
-
 	"github.com/jmoiron/sqlx"
 	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
 )
@@ -116,19 +113,3 @@ func CloseRows(rows *sqlx.Rows) {
 		}
 	}
 }
-
-var sqlRegex = regexp.MustCompile(`\$\d+`) // regex to check for SQL parameters
-
-// sqlQueryTrace logs the given SQL query if trace and debug are enabled.
-func sqlQueryTrace(trace bool, s *zap.SugaredLogger, query string, args ...interface{}) {
-	if trace {
-		s.Debugf("SQL Query: "+sqlRegex.ReplaceAllString(query, "%v"), args...)
-	}
-}
-
-// sqlResultsTrace logs the given SQL result if trace and debug are enabled.
-func sqlResultsTrace(trace bool, s *zap.SugaredLogger, results interface{}) {
-	if trace {
-		s.Debugf("SQL Results: %#v", results)
-	}
-}

pkg/models/golang_projects.go

@@ -21,7 +21,8 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
-	"regexp"
+
+	"github.com/scanoss/go-grpc-helper/pkg/grpc/database"
 
 	pkggodevclient "github.com/guseggert/pkggodev-client"
 	"github.com/jmoiron/sqlx"
@@ -32,21 +33,23 @@ import (
 )
 
 type GolangProjects struct {
-	ctx    context.Context
-	s      *zap.SugaredLogger
-	conn   *sqlx.Conn
-	config *myconfig.ServerConfig
-	ver    *VersionModel
-	lic    *LicenseModel
-	mine   *MineModel
+	ctx     context.Context
+	s       *zap.SugaredLogger
+	conn    *sqlx.Conn
+	config  *myconfig.ServerConfig
+	q       *database.DBQueryContext
+	ver     *VersionModel
+	lic     *LicenseModel
+	mine    *MineModel
+	project *ProjectModel // TODO Do we add golang component to the projects table?
 }
 
-var vRegex = regexp.MustCompile(`^v\d+\.\d+\.\d+-\d+-\w+$`) // regex to check for commit based version
-
 // NewGolangProjectModel creates a new instance of Golang Project Model.
 func NewGolangProjectModel(ctx context.Context, s *zap.SugaredLogger, conn *sqlx.Conn, config *myconfig.ServerConfig) *GolangProjects {
 	return &GolangProjects{ctx: ctx, s: s, conn: conn, config: config,
+		q:   database.NewDBSelectContext(s, conn, config.Database.Trace),
 		ver: NewVersionModel(ctx, s, conn), lic: NewLicenseModel(ctx, s, conn), mine: NewMineModel(ctx, s, conn),
+		project: NewProjectModel(ctx, s, conn),
 	}
 }
 
@@ -100,17 +103,26 @@ func (m *GolangProjects) GetGolangUrlsByPurlNameType(purlName, purlType, purlReq
 		" LEFT JOIN versions v ON u.version_id = v.id" +
 		" WHERE m.purl_type = $1 AND u.purl_name = $2 AND is_indexed = True" +
 		" ORDER BY version_date DESC"
-	sqlQueryTrace(m.config.Database.Trace, m.s, query, purlType, purlName)
-	var allUrls []AllURL
-	err := m.conn.SelectContext(m.ctx, &allUrls, query, purlType, purlName)
+	var allURLs []AllURL
+	err := m.q.SelectContext(m.ctx, &allURLs, query, purlType, purlName)
 	if err != nil {
 		m.s.Errorf("Failed to query golang projects table for %v - %v: %v", purlType, purlName, err)
 		return AllURL{}, fmt.Errorf("failed to query the golang projects table: %v", err)
 	}
-	m.s.Debugf("Found %v results for %v, %v.", len(allUrls), purlType, purlName)
-	sqlResultsTrace(m.config.Database.Trace, m.s, allUrls)
+	m.s.Debugf("Found %v results for %v, %v.", len(allURLs), purlType, purlName)
+	if len(allURLs) == 0 { // Check pkg.go.dev for the latest data
+		m.s.Debugf("Checking PkgGoDev for live info...")
+		allURL, err := m.getLatestPkgGoDev(purlName, purlType, "")
+		if err == nil {
+			m.s.Debugf("Retrieved golang data from pkg.go.dev: %#v", allURL)
+			allURLs = append(allURLs, allURL)
+		} else {
+			m.s.Infof("Ran into an issue looking up pkg.go.dev for: %v. Ignoring", purlName)
+		}
+	}
+
 	// Pick the most appropriate version to return
-	return pickOneUrl(m.s, nil, allUrls, purlName, purlType, purlReq)
+	return pickOneUrl(m.s, m.project, allURLs, purlName, purlType, purlReq)
 }
 
 // GetGolangUrlsByPurlNameTypeVersion searches Golang Projects for specified Purl, Type and Version.
@@ -135,14 +147,22 @@ func (m *GolangProjects) GetGolangUrlsByPurlNameTypeVersion(purlName, purlType,
 		" LEFT JOIN versions v ON u.version_id = v.id" +
 		" WHERE m.purl_type = $1 AND u.purl_name = $2 AND v.version_name = $3 AND is_indexed = True" +
 		" ORDER BY version_date DESC"
-	sqlQueryTrace(m.config.Database.Trace, m.s, query, purlType, purlName, purlVersion)
 	var allURLs []AllURL
-	err := m.conn.SelectContext(m.ctx, &allURLs, query, purlType, purlName, purlVersion)
+	err := m.q.SelectContext(m.ctx, &allURLs, query, purlType, purlName, purlVersion)
 	if err != nil {
 		m.s.Errorf("Failed to query golang projects table for %v - %v: %v", purlType, purlName, err)
 		return AllURL{}, fmt.Errorf("failed to query the golang projects table: %v", err)
 	}
 	m.s.Debugf("Found %v results for %v, %v.", len(allURLs), purlType, purlName)
+	if len(allURLs) > 0 { // We found an entry. Let's check if it has license data
+		allURL, err2 := pickOneUrl(m.s, m.project, allURLs, purlName, purlType, "")
+		if len(allURL.License) == 0 { // No license data found. Need to search for live info
+			m.s.Debugf("Couldn't find license data for component. Need to search live data")
+			allURLs = allURLs[:0]
+		} else {
+			return allURL, err2 // Return the component details
+		}
+	}
 	if len(allURLs) == 0 { // Check pkg.go.dev for the latest data
 		m.s.Debugf("Checking PkgGoDev for live info...")
 		allURL, err := m.getLatestPkgGoDev(purlName, purlType, purlVersion)
@@ -153,9 +173,8 @@ func (m *GolangProjects) GetGolangUrlsByPurlNameTypeVersion(purlName, purlType,
 			m.s.Infof("Ran into an issue looking up pkg.go.dev for: %v - %v. Ignoring", purlName, purlVersion)
 		}
 	}
-	sqlResultsTrace(m.config.Database.Trace, m.s, allURLs)
 	// Pick the most appropriate version to return
-	return pickOneUrl(m.s, nil, allURLs, purlName, purlType, "")
+	return pickOneUrl(m.s, m.project, allURLs, purlName, purlType, "")
 }
 
 // savePkg writes the given package details to the Golang Projects table.
@@ -286,7 +305,7 @@ func (m *GolangProjects) queryPkgGoDev(purlName, purlVersion string) (AllURL, *p
 	latest := false
 	m.s.Debugf("Checking pkg.go.dev for the latest info: %v", pkg)
 	comp, err := client.DescribePackage(pkggodevclient.DescribePackageRequest{Package: pkg})
-	if err != nil && len(purlVersion) > 0 && vRegex.MatchString(purlVersion) {
+	if err != nil && len(purlVersion) > 0 {
 		// We have a version zero search, so look for the latest one
 		m.s.Debugf("Failed to query pkg.go.dev for %v: %v. Trying without version...", pkg, err)
 		comp, err = client.DescribePackage(pkggodevclient.DescribePackageRequest{Package: purlName})