From bdb1a2d4d085e76fc5b12629e4bbc35f8f31da2c Mon Sep 17 00:00:00 2001 From: deo002 Date: Fri, 9 Aug 2024 12:26:09 +0530 Subject: [PATCH] feat(external_ref): Make external_ref fields configurable Signed-off-by: deo002 --- .github/workflows/api-swagger.yml | 6 +- .github/workflows/codeql.yml | 2 +- .github/workflows/go.yml | 2 +- .github/workflows/golangci.yml | 3 + .gitignore | 4 ++ Dockerfile | 5 +- README.md | 14 ++++ cmd/laas/gen.go | 8 +++ cmd/laas/gen_external_ref_schema.go | 87 ++++++++++++++++++++++ cmd/laas/main.go | 1 + external_ref_fields.example.yaml | 16 +++++ go.mod | 1 + go.sum | 2 + pkg/api/licenses.go | 108 +++++++++++++++++++--------- pkg/models/external_ref_structs.go | 13 ---- 15 files changed, 219 insertions(+), 53 deletions(-) create mode 100644 cmd/laas/gen.go create mode 100644 cmd/laas/gen_external_ref_schema.go create mode 100644 external_ref_fields.example.yaml delete mode 100644 pkg/models/external_ref_structs.go diff --git a/.github/workflows/api-swagger.yml b/.github/workflows/api-swagger.yml index 1447c4d..a963b81 100644 --- a/.github/workflows/api-swagger.yml +++ b/.github/workflows/api-swagger.yml @@ -23,6 +23,10 @@ jobs: check-latest: true cache: true + - name: Code Generation + run: | + cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./... + - name: Install swag run: | go install github.com/swaggo/swag/cmd/swag@latest @@ -33,7 +37,7 @@ jobs: - name: Check doc diff run: | - diff swag/docs/docs.go cmd/laas/docs/docs.go > swagger-diff.txt + diff swag/docs/docs.go cmd/laas/docs/docs.go > swagger-diff.txt || true # Check if file swagger-diff.txt is empty if [ -s swagger-diff.txt ] then diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3b0e20a..281c83b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -85,7 +85,7 @@ jobs: - if: matrix.build-mode == 'manual' shell: bash run: | - go build ./cmd/laas + cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./... && go build ./cmd/laas - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ddce0bb..03dc834 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -34,7 +34,7 @@ jobs: fi - name: Build - run: go build -v ./... + run: cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./... && go build -v ./... # - name: Test # run: go test -v ./... diff --git a/.github/workflows/golangci.yml b/.github/workflows/golangci.yml index 23a75ea..0b05719 100644 --- a/.github/workflows/golangci.yml +++ b/.github/workflows/golangci.yml @@ -27,6 +27,9 @@ jobs: check-latest: true cache: true + - name: Code Generation + run: cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./... + - name: lint uses: golangci/golangci-lint-action@v3 with: diff --git a/.gitignore b/.gitignore index 1bf2e63..4b67bae 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ licenseRef.json # IDE related files /.idea /LicenseDb.iml + +external_ref_fields.yaml + +pkg/models/external_ref_structs.go diff --git a/Dockerfile b/Dockerfile index c3e13f0..d7675ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,11 @@ RUN go mod download COPY cmd/ cmd/ COPY pkg/ pkg/ COPY docs/ docs/ +COPY external_ref_fields.example.yaml external_ref_fields.yaml RUN wget https://raw.githubusercontent.com/fossology/fossology/master/install/db/licenseRef.json -O licenseRef.json -RUN CGO_ENABLED=0 GOOS=linux go build -a -o laas ./cmd/laas +RUN CGO_ENABLED=0 GOOS=linux go generate ./cmd/laas && go build -a -o laas ./cmd/laas # Release Stage FROM alpine:3.20 AS build-release @@ -22,7 +23,7 @@ WORKDIR /app COPY entrypoint.sh /app/entrypoint.sh -RUN apk add --no-cache openssl bash \ +RUN apk add --no-cache openssl bash libc6-compat \ && addgroup -S noroot \ && adduser -S noroot -G noroot diff --git a/README.md b/README.md index e942516..3dc1b11 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,20 @@ git clone https://github.com/fossology/LicenseDb.git cd LicenseDb ``` +- Create the `external_ref_fields.yaml` file in the root directory of the project and change the + values of the extra license json keys as per your requirement. + +```bash +cp external_ref_fields.yaml.example external_ref_fields.yaml +vim external_ref_fields.yaml +``` + +- Generate Go struct for the extra fields listed in the external_ref_fields.yaml. + +```bash +go generate ./... +``` + - Build the project using following command. ```bash diff --git a/cmd/laas/gen.go b/cmd/laas/gen.go new file mode 100644 index 0000000..25362be --- /dev/null +++ b/cmd/laas/gen.go @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2024 Dearsh Oberoi +// SPDX-FileCopyrightText: 2024 Siemens AG +// +// SPDX-License-Identifier: GPL-2.0-only + +package main + +//go:generate go run gen_external_ref_schema.go diff --git a/cmd/laas/gen_external_ref_schema.go b/cmd/laas/gen_external_ref_schema.go new file mode 100644 index 0000000..d6c62ed --- /dev/null +++ b/cmd/laas/gen_external_ref_schema.go @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2024 Dearsh Oberoi +// SPDX-FileCopyrightText: 2024 Siemens AG +// +// SPDX-License-Identifier: GPL-2.0-only + +//go:build ignore + +package main + +import ( + "errors" + "fmt" + "log" + "os" + + "path/filepath" + + "github.com/dave/jennifer/jen" + "gopkg.in/yaml.v3" +) + +var ( + PATH_EXTERNAL_REF_CONFIG_FILE = filepath.FromSlash("../../external_ref_fields.yaml") + PATH_EXTERNAL_REF_STRUCT_FILE = filepath.FromSlash("../../pkg/models/external_ref_structs.go") +) + +// ExternalRefFieldMetadata is the metadata about the extra fields saved as json in the license external_ref column +type ExternalRefFieldMetaData struct { + StructFieldName string `yaml:"struct_field_name"` + Type string `yaml:"type"` + Name string `yaml:"name"` +} + +// ExternalRefFields is the list of metadata of all extra fields +type ExternalRefFields struct { + Fields []ExternalRefFieldMetaData `yaml:"fields"` +} + +func main() { + externalRefFields := ExternalRefFields{} + + fieldsMetadata, err := os.ReadFile(PATH_EXTERNAL_REF_CONFIG_FILE) + if err != nil { + log.Fatalf("Failed to instantiate json schema for external ref in license: %v", err) + } + + err = yaml.Unmarshal(fieldsMetadata, &externalRefFields) + if err != nil { + log.Fatalf("Failed to instantiate json schema for external ref in license: %v", err) + } + + // REUSE-IgnoreStart + + f := jen.NewFile("models") + f.Comment("SPDX-FileCopyrightText: FOSSology Community\nSPDX-License-Identifier: GPL-2.0-only\n\nThis file is autogenerated. Please do not edit it.\n") + + // REUSE-IgnoreStart + + var fields []jen.Code + + for _, f := range externalRefFields.Fields { + field := jen.Id(f.StructFieldName).Op("*") + if f.StructFieldName == "" { + err = errors.New("field struct_field_name is missing in external_ref_fields.yaml") + } + switch f.Type { + case "boolean": + field = field.Bool() + case "string": + field = field.String() + case "int": + field = field.Int64() + default: + err = fmt.Errorf("type %s in external_ref_fields.yaml is not supported", f.Type) + } + if err != nil { + log.Fatalf("Failed to instantiate json schema for external ref in license: %v", err) + return + } + field = field.Tag(map[string]string{"json": fmt.Sprintf("%s,omitempty", f.Name)}) + fields = append(fields, field) + } + + f.Type().Id("LicenseDBSchemaExtension").Struct(fields...) + f.Save(PATH_EXTERNAL_REF_STRUCT_FILE) + +} diff --git a/cmd/laas/main.go b/cmd/laas/main.go index 78c1b3e..6382f4c 100644 --- a/cmd/laas/main.go +++ b/cmd/laas/main.go @@ -12,6 +12,7 @@ import ( "github.com/joho/godotenv" + _ "github.com/dave/jennifer/jen" _ "github.com/fossology/LicenseDb/cmd/laas/docs" "github.com/fossology/LicenseDb/pkg/api" "github.com/fossology/LicenseDb/pkg/db" diff --git a/external_ref_fields.example.yaml b/external_ref_fields.example.yaml new file mode 100644 index 0000000..5d12c83 --- /dev/null +++ b/external_ref_fields.example.yaml @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# SPDX-FileCopyrightText: FOSSology contributors + +fields: + - name: "license_suffix" + type: "string" + struct_field_name: "LicenseSuffix" + label: "License Suffix" + formComponentPath: "../components/dynamic/inputField" + componentType: "input" + - name: "license_explanation" + type: "string" + struct_field_name: "LicenseExplanation" + label: "License Explanation" + formComponentPath: "../components/dynamic/inputField" + componentType: "input" \ No newline at end of file diff --git a/go.mod b/go.mod index 5012f52..88ca4d6 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/dave/jennifer v1.7.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index a9233eb..3243479 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.7.0 h1:uRbSBH9UTS64yXbh4FrMHfgfY762RD+C7bUPKODpSJE= +github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/api/licenses.go b/pkg/api/licenses.go index db047f7..cf34723 100644 --- a/pkg/api/licenses.go +++ b/pkg/api/licenses.go @@ -12,6 +12,7 @@ import ( "fmt" "net/http" "path/filepath" + "reflect" "strconv" "strings" "time" @@ -720,41 +721,78 @@ func addChangelogsForLicenseUpdate(tx *gorm.DB, username string, UpdatedValue: &newVal, }) } - if (oldLicense.ExternalRef.Data().InternalComment == nil && newLicense.ExternalRef.Data().InternalComment != nil) || - (oldLicense.ExternalRef.Data().InternalComment != nil && newLicense.ExternalRef.Data().InternalComment == nil) || - ((oldLicense.ExternalRef.Data().InternalComment != nil && newLicense.ExternalRef.Data().InternalComment != nil) && (*oldLicense.ExternalRef.Data().InternalComment != *newLicense.ExternalRef.Data().InternalComment)) { - changes = append(changes, models.ChangeLog{ - Field: "ExternalRef.InternalComment", - OldValue: oldLicense.ExternalRef.Data().InternalComment, - UpdatedValue: newLicense.ExternalRef.Data().InternalComment, - }) - } - if (oldLicense.ExternalRef.Data().LicenseExplanation == nil && newLicense.ExternalRef.Data().LicenseExplanation != nil) || - (oldLicense.ExternalRef.Data().LicenseExplanation != nil && newLicense.ExternalRef.Data().LicenseExplanation == nil) || - ((oldLicense.ExternalRef.Data().LicenseExplanation != nil && newLicense.ExternalRef.Data().LicenseExplanation != nil) && (*oldLicense.ExternalRef.Data().LicenseExplanation != *newLicense.ExternalRef.Data().LicenseExplanation)) { - changes = append(changes, models.ChangeLog{ - Field: "ExternalRef.LicenseExplanation", - OldValue: oldLicense.ExternalRef.Data().LicenseExplanation, - UpdatedValue: newLicense.ExternalRef.Data().LicenseExplanation, - }) - } - if (oldLicense.ExternalRef.Data().LicenseSuffix == nil && newLicense.ExternalRef.Data().LicenseSuffix != nil) || - (oldLicense.ExternalRef.Data().LicenseSuffix != nil && newLicense.ExternalRef.Data().LicenseSuffix == nil) || - ((oldLicense.ExternalRef.Data().LicenseSuffix != nil && newLicense.ExternalRef.Data().LicenseSuffix != nil) && (*oldLicense.ExternalRef.Data().LicenseSuffix != *newLicense.ExternalRef.Data().LicenseSuffix)) { - changes = append(changes, models.ChangeLog{ - Field: "ExternalRef.LicenseSuffix", - OldValue: oldLicense.ExternalRef.Data().LicenseSuffix, - UpdatedValue: newLicense.ExternalRef.Data().LicenseSuffix, - }) - } - if (oldLicense.ExternalRef.Data().LicenseVersion == nil && newLicense.ExternalRef.Data().LicenseVersion != nil) || - (oldLicense.ExternalRef.Data().LicenseVersion != nil && newLicense.ExternalRef.Data().LicenseVersion == nil) || - ((oldLicense.ExternalRef.Data().LicenseVersion != nil && newLicense.ExternalRef.Data().LicenseVersion != nil) && (*oldLicense.ExternalRef.Data().LicenseVersion != *newLicense.ExternalRef.Data().LicenseVersion)) { - changes = append(changes, models.ChangeLog{ - Field: "ExternalRef.LicenseVersion", - OldValue: oldLicense.ExternalRef.Data().LicenseVersion, - UpdatedValue: newLicense.ExternalRef.Data().LicenseVersion, - }) + + oldLicenseExternalRef := oldLicense.ExternalRef.Data() + oldExternalRefVal := reflect.ValueOf(oldLicenseExternalRef) + typesOf := oldExternalRefVal.Type() + + newLicenseExternalRef := newLicense.ExternalRef.Data() + newExternalRefVal := reflect.ValueOf(newLicenseExternalRef) + + for i := 0; i < oldExternalRefVal.NumField(); i++ { + fieldName := typesOf.Field(i).Name + + switch typesOf.Field(i).Type.String() { + case "*boolean": + oldFieldPtr, _ := oldExternalRefVal.Field(i).Interface().(*bool) + newFieldPtr, _ := newExternalRefVal.Field(i).Interface().(*bool) + if (oldFieldPtr == nil && newFieldPtr != nil) || (oldFieldPtr != nil && newFieldPtr == nil) || + ((oldFieldPtr != nil && newFieldPtr != nil) && (*oldFieldPtr != *newFieldPtr)) { + var oldVal, newVal *string + oldVal, newVal = nil, nil + + if oldFieldPtr != nil { + _oldVal := fmt.Sprintf("%t", *oldFieldPtr) + oldVal = &_oldVal + } + + if newFieldPtr != nil { + _newVal := fmt.Sprintf("%t", *newFieldPtr) + newVal = &_newVal + } + + changes = append(changes, models.ChangeLog{ + Field: fmt.Sprintf("ExternalRef.%s", fieldName), + OldValue: oldVal, + UpdatedValue: newVal, + }) + } + case "*string": + oldFieldPtr, _ := oldExternalRefVal.Field(i).Interface().(*string) + newFieldPtr, _ := newExternalRefVal.Field(i).Interface().(*string) + if (oldFieldPtr == nil && newFieldPtr != nil) || (oldFieldPtr != nil && newFieldPtr == nil) || + ((oldFieldPtr != nil && newFieldPtr != nil) && (*oldFieldPtr != *newFieldPtr)) { + changes = append(changes, models.ChangeLog{ + Field: fmt.Sprintf("ExternalRef.%s", fieldName), + OldValue: oldFieldPtr, + UpdatedValue: newFieldPtr, + }) + } + case "*int": + oldFieldPtr, _ := oldExternalRefVal.Field(i).Interface().(*int) + newFieldPtr, _ := newExternalRefVal.Field(i).Interface().(*int) + if (oldFieldPtr == nil && newFieldPtr != nil) || (oldFieldPtr != nil && newFieldPtr == nil) || + ((oldFieldPtr != nil && newFieldPtr != nil) && (*oldFieldPtr != *newFieldPtr)) { + var oldVal, newVal *string + oldVal, newVal = nil, nil + + if oldFieldPtr != nil { + _oldVal := fmt.Sprintf("%d", *oldFieldPtr) + oldVal = &_oldVal + } + + if newFieldPtr != nil { + _newVal := fmt.Sprintf("%d", *newFieldPtr) + newVal = &_newVal + } + + changes = append(changes, models.ChangeLog{ + Field: fmt.Sprintf("ExternalRef.%s", fieldName), + OldValue: oldVal, + UpdatedValue: newVal, + }) + } + } } if len(changes) != 0 { diff --git a/pkg/models/external_ref_structs.go b/pkg/models/external_ref_structs.go deleted file mode 100644 index fe32c35..0000000 --- a/pkg/models/external_ref_structs.go +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Siemens AG -// SPDX-FileContributor: Dearsh Oberoi -// -// SPDX-License-Identifier: GPL-2.0-only - -package models - -type LicenseDBSchemaExtension struct { - InternalComment *string `json:"internal_comment,omitempty" example:"comment"` - LicenseExplanation *string `json:"license_explanation,omitempty" example:"explanation of license"` - LicenseSuffix *string `json:"license_suffix,omitempty" example:"license suffix"` - LicenseVersion *string `json:"license_version,omitempty"` -}