From 351b83df5b8f8e0a7c92454a51289a6da349cb61 Mon Sep 17 00:00:00 2001 From: Shurkys Date: Wed, 15 Jan 2025 01:02:30 +0200 Subject: [PATCH 01/10] add terraform state package --- models/packages/descriptor.go | 3 + models/packages/package.go | 4 + modules/packages/terraform/metadata.go | 88 ++++ modules/packages/terraform/metadata_test.go | 161 +++++++ modules/setting/packages.go | 2 + routers/api/packages/api.go | 21 + routers/api/packages/terraform/terraform.go | 396 ++++++++++++++++++ routers/api/v1/packages/package.go | 2 +- services/forms/package_form.go | 2 +- services/packages/packages.go | 2 + templates/package/content/terraform.tmpl | 30 ++ templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + .../api_packages_terraform_test.go | 154 +++++++ 14 files changed, 866 insertions(+), 2 deletions(-) create mode 100644 modules/packages/terraform/metadata.go create mode 100644 modules/packages/terraform/metadata_test.go create mode 100644 routers/api/packages/terraform/terraform.go create mode 100644 templates/package/content/terraform.tmpl create mode 100644 tests/integration/api_packages_terraform_test.go diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 803b73c968995..bdb2361239b5a 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -31,6 +31,7 @@ import ( "code.gitea.io/gitea/modules/packages/rpm" "code.gitea.io/gitea/modules/packages/rubygems" "code.gitea.io/gitea/modules/packages/swift" + "code.gitea.io/gitea/modules/packages/terraform" "code.gitea.io/gitea/modules/packages/vagrant" "code.gitea.io/gitea/modules/util" @@ -191,6 +192,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &rubygems.Metadata{} case TypeSwift: metadata = &swift.Metadata{} + case TypeTerraform: + metadata = &terraform.Metadata{} case TypeVagrant: metadata = &vagrant.Metadata{} default: diff --git a/models/packages/package.go b/models/packages/package.go index 31e1277a6e37b..b09672663a89e 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -51,6 +51,7 @@ const ( TypeRpm Type = "rpm" TypeRubyGems Type = "rubygems" TypeSwift Type = "swift" + TypeTerraform Type = "terraform" TypeVagrant Type = "vagrant" ) @@ -76,6 +77,7 @@ var TypeList = []Type{ TypeRpm, TypeRubyGems, TypeSwift, + TypeTerraform, TypeVagrant, } @@ -175,6 +177,8 @@ func (pt Type) SVGName() string { return "gitea-rubygems" case TypeSwift: return "gitea-swift" + case TypeTerraform: + return "gitea-terraform" case TypeVagrant: return "gitea-vagrant" } diff --git a/modules/packages/terraform/metadata.go b/modules/packages/terraform/metadata.go new file mode 100644 index 0000000000000..14df7cf548d3c --- /dev/null +++ b/modules/packages/terraform/metadata.go @@ -0,0 +1,88 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package terraform + +import ( + "archive/tar" + "compress/gzip" + "errors" + "io" + + "code.gitea.io/gitea/modules/json" +) + +const ( + PropertyTerraformState = "terraform.state" +) + +// Metadata represents the Terraform backend metadata +// Updated to align with TerraformState structure +// Includes additional metadata fields like Description, Author, and URLs +type Metadata struct { + Version int `json:"version"` + TerraformVersion string `json:"terraform_version,omitempty"` + Serial uint64 `json:"serial"` + Lineage string `json:"lineage"` + Outputs map[string]any `json:"outputs,omitempty"` + Resources []ResourceState `json:"resources,omitempty"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` +} + +// ResourceState represents the state of a resource +type ResourceState struct { + Mode string `json:"mode"` + Type string `json:"type"` + Name string `json:"name"` + Provider string `json:"provider"` + Instances []InstanceState `json:"instances"` +} + +// InstanceState represents the state of a resource instance +type InstanceState struct { + SchemaVersion int `json:"schema_version"` + Attributes map[string]any `json:"attributes"` +} + +// ParseMetadataFromState retrieves metadata from the archive with Terraform state +func ParseMetadataFromState(r io.Reader) (*Metadata, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + // Looking for the state.json file + if hd.Name == "state.json" { + return ParseStateFile(tr) + } + } + + return nil, errors.New("state.json not found in archive") +} + +// ParseStateFile parses the state.json file and returns Terraform metadata +func ParseStateFile(r io.Reader) (*Metadata, error) { + var stateData Metadata + if err := json.NewDecoder(r).Decode(&stateData); err != nil { + return nil, err + } + return &stateData, nil +} diff --git a/modules/packages/terraform/metadata_test.go b/modules/packages/terraform/metadata_test.go new file mode 100644 index 0000000000000..b2d5164f5e2c7 --- /dev/null +++ b/modules/packages/terraform/metadata_test.go @@ -0,0 +1,161 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package terraform + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestParseMetadataFromState tests the ParseMetadataFromState function +func TestParseMetadataFromState(t *testing.T) { + tests := []struct { + name string + input []byte + expectedError bool + }{ + { + name: "valid state file", + input: createValidStateArchive(), + expectedError: false, + }, + { + name: "missing state.json file", + input: createInvalidStateArchive(), + expectedError: true, + }, + { + name: "corrupt archive", + input: []byte("invalid archive data"), + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := bytes.NewReader(tt.input) + metadata, err := ParseMetadataFromState(r) + + if tt.expectedError { + assert.Error(t, err) + assert.Nil(t, metadata) + } else { + assert.NoError(t, err) + assert.NotNil(t, metadata) + // Optionally, check if certain fields are populated correctly + assert.NotEmpty(t, metadata.Lineage) + } + }) + } +} + +// createValidStateArchive creates a valid TAR.GZ archive with a sample state.json +func createValidStateArchive() []byte { + metadata := `{ + "version": 4, + "terraform_version": "1.2.0", + "serial": 1, + "lineage": "abc123", + "resources": [], + "description": "Test project", + "author": "Test Author", + "project_url": "http://example.com", + "repository_url": "http://repo.com" + }` + + // Create a gzip writer and tar writer + buf := new(bytes.Buffer) + gz := gzip.NewWriter(buf) + tw := tar.NewWriter(gz) + + // Add the state.json file to the tar + hdr := &tar.Header{ + Name: "state.json", + Size: int64(len(metadata)), + Mode: 0o600, + } + if err := tw.WriteHeader(hdr); err != nil { + panic(err) + } + if _, err := tw.Write([]byte(metadata)); err != nil { + panic(err) + } + + // Close the writers + if err := tw.Close(); err != nil { + panic(err) + } + if err := gz.Close(); err != nil { + panic(err) + } + + return buf.Bytes() +} + +// createInvalidStateArchive creates an invalid TAR.GZ archive (missing state.json) +func createInvalidStateArchive() []byte { + // Create a tar archive without the state.json file + buf := new(bytes.Buffer) + gz := gzip.NewWriter(buf) + tw := tar.NewWriter(gz) + + // Add an empty file to the tar (but not state.json) + hdr := &tar.Header{ + Name: "other_file.txt", + Size: 0, + Mode: 0o600, + } + if err := tw.WriteHeader(hdr); err != nil { + panic(err) + } + + // Close the writers + if err := tw.Close(); err != nil { + panic(err) + } + if err := gz.Close(); err != nil { + panic(err) + } + + return buf.Bytes() +} + +// TestParseStateFile tests the ParseStateFile function directly +func TestParseStateFile(t *testing.T) { + tests := []struct { + name string + input string + expectedError bool + }{ + { + name: "valid state.json", + input: `{"version":4,"terraform_version":"1.2.0","serial":1,"lineage":"abc123"}`, + expectedError: false, + }, + { + name: "invalid JSON", + input: `{"version":4,"terraform_version"}`, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := bytes.NewReader([]byte(tt.input)) + metadata, err := ParseStateFile(r) + + if tt.expectedError { + assert.Error(t, err) + assert.Nil(t, metadata) + } else { + assert.NoError(t, err) + assert.NotNil(t, metadata) + } + }) + } +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3f618cfd64115..6eff4f1b5c9eb 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -42,6 +42,7 @@ var ( LimitSizeRpm int64 LimitSizeRubyGems int64 LimitSizeSwift int64 + LimitSizeTerraform int64 LimitSizeVagrant int64 DefaultRPMSignEnabled bool @@ -100,6 +101,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { Packages.LimitSizeRpm = mustBytes(sec, "LIMIT_SIZE_RPM") Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS") Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT") + Packages.LimitSizeTerraform = mustBytes(sec, "LIMIT_SIZE_TERRAFORM") Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT") Packages.DefaultRPMSignEnabled = sec.Key("DEFAULT_RPM_SIGN_ENABLED").MustBool(false) return nil diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 41c3eb95e9011..2dd53198d515e 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -34,6 +34,7 @@ import ( "code.gitea.io/gitea/routers/api/packages/rpm" "code.gitea.io/gitea/routers/api/packages/rubygems" "code.gitea.io/gitea/routers/api/packages/swift" + "code.gitea.io/gitea/routers/api/packages/terraform" "code.gitea.io/gitea/routers/api/packages/vagrant" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/context" @@ -674,6 +675,26 @@ func CommonRoutes() *web.Router { }) }) }, reqPackageAccess(perm.AccessModeRead)) + // Define routes for Terraform HTTP backend API + r.Group("/terraform/state", func() { + // Routes for specific state identified by {statename} + r.Group("/{statename}", func() { + // Fetch the current state + r.Get("", reqPackageAccess(perm.AccessModeRead), terraform.GetState) + // Update the state (supports both POST and PUT methods) + r.Post("", reqPackageAccess(perm.AccessModeWrite), terraform.UpdateState) + r.Put("", reqPackageAccess(perm.AccessModeWrite), terraform.UpdateState) + // Delete the state + r.Delete("", reqPackageAccess(perm.AccessModeWrite), terraform.DeleteState) + // Lock and unlock operations for the state + r.Group("/lock", func() { + // Lock the state + r.Post("", reqPackageAccess(perm.AccessModeWrite), terraform.LockState) + // Unlock the state + r.Delete("", reqPackageAccess(perm.AccessModeWrite), terraform.UnlockState) + }) + }) + }, reqPackageAccess(perm.AccessModeRead)) }, context.UserAssignmentWeb(), context.PackageAssignment()) return r diff --git a/routers/api/packages/terraform/terraform.go b/routers/api/packages/terraform/terraform.go new file mode 100644 index 0000000000000..f0948fbbd84c3 --- /dev/null +++ b/routers/api/packages/terraform/terraform.go @@ -0,0 +1,396 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package terraform + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "sync" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/routers/api/packages/helper" + "code.gitea.io/gitea/services/context" + packages_service "code.gitea.io/gitea/services/packages" + "github.com/google/uuid" +) + +type TFState struct { + Version int `json:"version"` + TerraformVersion string `json:"terraform_version"` + Serial uint64 `json:"serial"` + Lineage string `json:"lineage"` + Outputs map[string]any `json:"outputs"` + Resources []ResourceState `json:"resources"` +} + +type ResourceState struct { + Mode string `json:"mode"` + Type string `json:"type"` + Name string `json:"name"` + Provider string `json:"provider"` + Instances []InstanceState `json:"instances"` +} + +type InstanceState struct { + SchemaVersion int `json:"schema_version"` + Attributes map[string]any `json:"attributes"` +} + +var ( + stateStorage = make(map[string]*TFState) + stateLocks = make(map[string]string) + storeMutex sync.Mutex +) + +func apiError(ctx *context.Context, status int, obj any) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + type Error struct { + Status int `json:"status"` + Message string `json:"message"` + } + ctx.JSON(status, struct { + Errors []Error `json:"errors"` + }{ + Errors: []Error{ + {Status: status, Message: message}, + }, + }) + }) +} + +func GetState(ctx *context.Context) { + stateName := ctx.PathParam("statename") + log.Info("Function GetState called with parameters: stateName=%s", stateName) + + // Find the package version + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeTerraform, + Name: packages_model.SearchValue{ + ExactMatch: true, + Value: stateName, + }, + HasFileWithName: stateName, + IsInternal: optional.Some(false), + }) + if err != nil { + log.Error("Failed to search package versions for state %s: %v", stateName, err) + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // If no version is found, return 204 + if len(pvs) == 0 { + log.Info("No existing state found for %s, returning 204 No Content", stateName) + ctx.Resp.WriteHeader(http.StatusNoContent) + return + } + + // Get the latest package version + stateVersion := pvs[0] + if stateVersion == nil { + log.Error("State version is nil for state %s", stateName) + apiError(ctx, http.StatusInternalServerError, "Invalid state version") + return + } + log.Info("Fetching file stream for state %s with version %s", stateName, stateVersion.Version) + + // Log the parameters of GetFileStreamByPackageNameAndVersion call + log.Info("Fetching file stream with params: Owner=%v, PackageType=%v, Name=%v, Version=%v, Filename=%v", + ctx.Package.Owner, + packages_model.TypeTerraform, + stateName, + stateVersion.Version, + stateName, + ) + + // Fetch the file stream + s, _, _, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeTerraform, + Name: stateName, + Version: stateVersion.Version, + }, + &packages_service.PackageFileInfo{ + Filename: stateName, + }, + ) + if err != nil { + log.Error("Error fetching file stream for state %s: %v", stateName, err) + if errors.Is(err, packages_model.ErrPackageNotExist) { + log.Error("Package does not exist: %v", err) + apiError(ctx, http.StatusNotFound, "Package not found") + return + } + if errors.Is(err, packages_model.ErrPackageFileNotExist) { + log.Error("Package file does not exist: %v", err) + apiError(ctx, http.StatusNotFound, "File not found") + return + } + apiError(ctx, http.StatusInternalServerError, "Failed to fetch file stream") + return + } + defer s.Close() + + // Read the file contents + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, s); err != nil { + log.Error("Failed to read state file for %s: %v", stateName, err) + apiError(ctx, http.StatusInternalServerError, "Failed to read state file") + return + } + + // Deserialize the state + var state TFState + if err := json.Unmarshal(buf.Bytes(), &state); err != nil { + log.Error("Failed to unmarshal state file for %s: %v", stateName, err) + apiError(ctx, http.StatusInternalServerError, "Invalid state file format") + return + } + + // Ensure lineage is set + if state.Lineage == "" { + state.Lineage = uuid.NewString() + log.Info("Generated new lineage for state %s: %s", stateName, state.Lineage) + } + + // Send the state in the response + ctx.Resp.Header().Set("Content-Type", "application/json") + ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", stateName)) + ctx.Resp.WriteHeader(http.StatusOK) + if _, writeErr := ctx.Resp.Write(buf.Bytes()); writeErr != nil { + log.Error("Failed to write response for state %s: %v", stateName, writeErr) + } +} + +// UpdateState updates or creates a new Terraform state and interacts with Gitea packages. +func UpdateState(ctx *context.Context) { + stateName := ctx.PathParam("statename") + log.Info("UpdateState called for stateName: %s", stateName) + + storeMutex.Lock() + defer storeMutex.Unlock() + + // Check for the presence of a lock ID + requestLockID := ctx.Req.URL.Query().Get("ID") + if requestLockID == "" { + apiError(ctx, http.StatusBadRequest, "Missing ID query parameter") + return + } + + // Check for blocking state + if lockID, locked := stateLocks[stateName]; locked && lockID != requestLockID { + apiError(ctx, http.StatusConflict, fmt.Sprintf("State %s is locked", stateName)) + return + } + + // Read the request body + body, err := io.ReadAll(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to read request body") + return + } + + var newState TFState + if err := json.Unmarshal(body, &newState); err != nil { + apiError(ctx, http.StatusBadRequest, "Invalid JSON") + return + } + + // Getting the current serial + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeTerraform, + Name: packages_model.SearchValue{ExactMatch: true, Value: stateName}, + IsInternal: optional.Some(false), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to search package versions") + return + } + + serial := uint64(1) // Start from 1 + if len(pvs) > 0 { + lastSerial, _ := strconv.ParseUint(pvs[0].Version, 10, 64) + serial = lastSerial + 1 + } + log.Info("State %s updated to serial %d", stateName, serial) + + // Create package information + packageVersion := fmt.Sprintf("%d", serial) + packageInfo := &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeTerraform, + Name: stateName, + Version: packageVersion, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: newState, + } + + buffer, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(body)) + if err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to create buffer") + return + } + + // Create/update package + if _, _, err = packages_service.CreatePackageOrAddFileToExisting( + ctx, + packageInfo, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: stateName, + }, + Creator: ctx.Doer, + Data: buffer, + IsLead: true, + }, + ); err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to update package") + return + } + + log.Info("State %s updated successfully with version %s", stateName, packageVersion) + + ctx.JSON(http.StatusOK, map[string]string{ + "message": "State updated successfully", + "statename": stateName, + }) +} + +// LockState locks a Terraform state to prevent updates. +func LockState(ctx *context.Context) { + stateName := ctx.PathParam("statename") + log.Info("LockState called for state: %s", stateName) + + // Read the request body + body, err := io.ReadAll(ctx.Req.Body) + if err != nil { + log.Error("Failed to read request body: %v", err) + apiError(ctx, http.StatusInternalServerError, "Failed to read request body") + return + } + + // Decode JSON and check lockID + var lockRequest struct { + ID string `json:"ID"` + } + if err := json.Unmarshal(body, &lockRequest); err != nil || lockRequest.ID == "" { + log.Error("Invalid lock request body: %v", err) + apiError(ctx, http.StatusBadRequest, "Invalid or missing lock ID") + return + } + + storeMutex.Lock() + defer storeMutex.Unlock() + + // Check if the state is locked + if _, locked := stateLocks[stateName]; locked { + log.Warn("State %s is already locked", stateName) + apiError(ctx, http.StatusConflict, fmt.Sprintf("State %s is already locked", stateName)) + return + } + + // Set the lock + stateLocks[stateName] = lockRequest.ID + log.Info("State %s locked with ID %s", stateName, lockRequest.ID) + + ctx.JSON(http.StatusOK, map[string]string{ + "message": "State locked successfully", + "statename": stateName, + }) +} + +// UnlockState unlocks a Terraform state. +func UnlockState(ctx *context.Context) { + stateName := ctx.PathParam("statename") + log.Info("UnlockState called for state: %s", stateName) + + // Extract lockID from request body or parameters + var unlockRequest struct { + ID string `json:"ID"` + } + + // Trying to read the request body + body, _ := io.ReadAll(ctx.Req.Body) + if len(body) > 0 { + _ = json.Unmarshal(body, &unlockRequest) // The error can be ignored, since the ID can also be in the query + } + + // If the ID is not found in the body, look in the query parameters + if unlockRequest.ID == "" { + unlockRequest.ID = ctx.Query("ID").(string) + } + + // Check for ID presence + if unlockRequest.ID == "" { + log.Error("Missing lock ID in both query and request body") + apiError(ctx, http.StatusBadRequest, "Missing lock ID") + return + } + + log.Info("Extracted lockID: %s", unlockRequest.ID) + + storeMutex.Lock() + defer storeMutex.Unlock() + + // Check the lock status + currentLockID, locked := stateLocks[stateName] + if !locked || currentLockID != unlockRequest.ID { + log.Warn("Unlock attempt failed for state %s with lock ID %s", stateName, unlockRequest.ID) + apiError(ctx, http.StatusConflict, fmt.Sprintf("State %s is not locked or lock ID mismatch", stateName)) + return + } + + // Remove the lock + delete(stateLocks, stateName) + log.Info("State %s unlocked successfully", stateName) + + ctx.JSON(http.StatusOK, map[string]string{ + "message": "State unlocked successfully", + "statename": stateName, + }) +} + +// DeleteState deletes the Terraform state for a given name. +func DeleteState(ctx *context.Context) { + stateName := ctx.PathParam("statename") + log.Info("Attempting to delete state: %s", stateName) + + storeMutex.Lock() + defer storeMutex.Unlock() + + // Check if a state or lock exists + _, stateExists := stateStorage[stateName] + _, lockExists := stateLocks[stateName] + + if !stateExists && !lockExists { + log.Warn("State %s does not exist or is not locked", stateName) + apiError(ctx, http.StatusNotFound, "State not found") + return + } + + // Delete the state and lock + delete(stateStorage, stateName) + delete(stateLocks, stateName) + + log.Info("State %s deleted successfully", stateName) + ctx.JSON(http.StatusOK, map[string]string{ + "message": "State deleted successfully", + "statename": stateName, + }) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index b38aa131676e1..c9af7e8db0aad 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [alpine, cargo, chef, composer, conan, conda, container, cran, debian, generic, go, helm, maven, npm, nuget, pub, pypi, rpm, rubygems, swift, vagrant] + // enum: [alpine, cargo, chef, composer, conan, conda, container, cran, debian, generic, go, helm, maven, npm, nuget, pub, pypi, rpm, rubygems, swift, terraform, vagrant] // - name: q // in: query // description: name filter diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 9b6f9071647bc..d1a2b8587ccf5 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` + Type string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,terraform,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/packages.go b/services/packages/packages.go index bd1d460fd3ba8..0736ef4b56e2d 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -393,6 +393,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p typeSpecificSize = setting.Packages.LimitSizeRubyGems case packages_model.TypeSwift: typeSpecificSize = setting.Packages.LimitSizeSwift + case packages_model.TypeTerraform: + typeSpecificSize = setting.Packages.LimitSizeTerraform case packages_model.TypeVagrant: typeSpecificSize = setting.Packages.LimitSizeVagrant } diff --git a/templates/package/content/terraform.tmpl b/templates/package/content/terraform.tmpl new file mode 100644 index 0000000000000..e7e8f11d495ac --- /dev/null +++ b/templates/package/content/terraform.tmpl @@ -0,0 +1,30 @@ +{{if eq .PackageDescriptor.Package.Type "terraform"}} +

{{ctx.Locale.Tr "packages.installation"}}

+
+
+
+ + 

+export GITEA_USER_PASSWORD=<YOUR-USER-PASSWORD>
+export TF_STATE_NAME=your-state.tfstate
+terraform init \
+    -backend-config="address= \
+    -backend-config="lock_address= \
+    -backend-config="unlock_address= \
+    -backend-config="username={{.PackageDescriptor.Owner.Name}}" \
+    -backend-config="password=$GITEA_USER_PASSWORD" \
+    -backend-config="lock_method=POST" \
+    -backend-config="unlock_method=DELETE" \
+    -backend-config="retry_wait_min=5"
+
+
+
+ +
+
+
+ {{if .PackageDescriptor.Metadata.Description}} +

{{ctx.Locale.Tr "packages.about"}}

+
{{.PackageDescriptor.Metadata.Description}}
+ {{end}} +{{end}} \ No newline at end of file diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 9e92207466d96..5c5305cd09d8a 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -37,6 +37,7 @@ {{template "package/content/rpm" .}} {{template "package/content/rubygems" .}} {{template "package/content/swift" .}} + {{template "package/content/terraform" .}} {{template "package/content/vagrant" .}}
@@ -68,6 +69,7 @@ {{template "package/metadata/rpm" .}} {{template "package/metadata/rubygems" .}} {{template "package/metadata/swift" .}} + {{template "package/metadata/terraform" .}} {{template "package/metadata/vagrant" .}} {{if not (and (eq .PackageDescriptor.Package.Type "container") .PackageDescriptor.Metadata.Manifests)}}
{{svg "octicon-database"}} {{FileSize .PackageDescriptor.CalculateBlobSize}}
diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 8082fc594ac02..17cb8c1cc9f20 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -3275,6 +3275,7 @@ "rpm", "rubygems", "swift", + "terraform", "vagrant" ], "type": "string", diff --git a/tests/integration/api_packages_terraform_test.go b/tests/integration/api_packages_terraform_test.go new file mode 100644 index 0000000000000..0e7df0104d05f --- /dev/null +++ b/tests/integration/api_packages_terraform_test.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "net/http" + "strings" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + terraform_module "code.gitea.io/gitea/modules/packages/terraform" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageTerraform(t *testing.T) { + defer tests.PrepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + token := "Bearer " + getUserToken(t, user.Name, auth_model.AccessTokenScopeWritePackage) + + packageName := "test_module" + packageVersion := "1.0.1" + packageDescription := "Test Terraform Module" + + filename := "terraform_module.tar.gz" + + infoContent, _ := json.Marshal(map[string]string{ + "description": packageDescription, + }) + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + archive := tar.NewWriter(zw) + archive.WriteHeader(&tar.Header{ + Name: "info.json", + Mode: 0o600, + Size: int64(len(infoContent)), + }) + archive.Write(infoContent) + archive.Close() + zw.Close() + content := buf.Bytes() + + root := fmt.Sprintf("/api/packages/%s/terraform", user.Name) + + t.Run("Authenticate", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + authenticateURL := fmt.Sprintf("%s/authenticate", root) + + req := NewRequest(t, "GET", authenticateURL) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "GET", authenticateURL). + AddTokenAuth(token) + MakeRequest(t, req, http.StatusOK) + }) + + moduleURL := fmt.Sprintf("%s/%s", root, packageName) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "HEAD", moduleURL) + MakeRequest(t, req, http.StatusNotFound) + + uploadURL := fmt.Sprintf("%s/%s/%s", moduleURL, packageVersion, filename) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)). + AddTokenAuth(token) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "HEAD", moduleURL) + resp := MakeRequest(t, req, http.StatusOK) + assert.True(t, strings.HasPrefix(resp.Header().Get("Content-Type"), "application/json")) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeTerraform) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &terraform_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)). + AddTokenAuth(token) + MakeRequest(t, req, http.StatusConflict) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", moduleURL, packageVersion, filename)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, content, resp.Body.Bytes()) + }) + + t.Run("EnumeratePackageVersions", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", moduleURL) + resp := MakeRequest(t, req, http.StatusOK) + + type versionMetadata struct { + Version string `json:"version"` + Status string `json:"status"` + } + + type packageMetadata struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Versions []*versionMetadata `json:"versions"` + } + + var result packageMetadata + DecodeJSON(t, resp, &result) + + assert.Equal(t, packageName, result.Name) + assert.Equal(t, packageDescription, result.Description) + assert.Len(t, result.Versions, 1) + version := result.Versions[0] + assert.Equal(t, packageVersion, version.Version) + assert.Equal(t, "active", version.Status) + }) +} From 0457597f82553e8f3c12dd89272cc99d22c93ed5 Mon Sep 17 00:00:00 2001 From: Shurkys Date: Wed, 15 Jan 2025 01:07:14 +0200 Subject: [PATCH 02/10] fix lint --- templates/package/content/terraform.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/package/content/terraform.tmpl b/templates/package/content/terraform.tmpl index e7e8f11d495ac..8d6ddd57d21ed 100644 --- a/templates/package/content/terraform.tmpl +++ b/templates/package/content/terraform.tmpl @@ -27,4 +27,4 @@ terraform init \

{{ctx.Locale.Tr "packages.about"}}

{{.PackageDescriptor.Metadata.Description}}
{{end}} -{{end}} \ No newline at end of file +{{end}} From 5ad2c19e7b96a3a67e5de585be76e11f0773c1e8 Mon Sep 17 00:00:00 2001 From: Shurkys Date: Wed, 15 Jan 2025 01:33:23 +0200 Subject: [PATCH 03/10] fix lint --- routers/api/packages/terraform/terraform.go | 5 ----- templates/package/content/terraform.tmpl | 16 ++++++++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/routers/api/packages/terraform/terraform.go b/routers/api/packages/terraform/terraform.go index f0948fbbd84c3..e213d76ef1feb 100644 --- a/routers/api/packages/terraform/terraform.go +++ b/routers/api/packages/terraform/terraform.go @@ -331,11 +331,6 @@ func UnlockState(ctx *context.Context) { _ = json.Unmarshal(body, &unlockRequest) // The error can be ignored, since the ID can also be in the query } - // If the ID is not found in the body, look in the query parameters - if unlockRequest.ID == "" { - unlockRequest.ID = ctx.Query("ID").(string) - } - // Check for ID presence if unlockRequest.ID == "" { log.Error("Missing lock ID in both query and request body") diff --git a/templates/package/content/terraform.tmpl b/templates/package/content/terraform.tmpl index 8d6ddd57d21ed..c59713c0dbc0c 100644 --- a/templates/package/content/terraform.tmpl +++ b/templates/package/content/terraform.tmpl @@ -8,14 +8,14 @@ export GITEA_USER_PASSWORD=<YOUR-USER-PASSWORD> export TF_STATE_NAME=your-state.tfstate terraform init \ - -backend-config="address= \ - -backend-config="lock_address= \ - -backend-config="unlock_address= \ - -backend-config="username={{.PackageDescriptor.Owner.Name}}" \ - -backend-config="password=$GITEA_USER_PASSWORD" \ - -backend-config="lock_method=POST" \ - -backend-config="unlock_method=DELETE" \ - -backend-config="retry_wait_min=5" + -backend-config="address= \ + -backend-config="lock_address= \ + -backend-config="unlock_address= \ + -backend-config="username={{.PackageDescriptor.Owner.Name}}" \ + -backend-config="password=$GITEA_USER_PASSWORD" \ + -backend-config="lock_method=POST" \ + -backend-config="unlock_method=DELETE" \ + -backend-config="retry_wait_min=5"
From 64f08862c0dd4992356215f6d208791c3bc0dc10 Mon Sep 17 00:00:00 2001 From: Shurkys Date: Wed, 15 Jan 2025 02:00:07 +0200 Subject: [PATCH 04/10] fix view --- models/packages/package.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/packages/package.go b/models/packages/package.go index b09672663a89e..ab5deeaec161d 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -126,6 +126,8 @@ func (pt Type) Name() string { return "RubyGems" case TypeSwift: return "Swift" + case TypeTerraform: + return "Terraform" case TypeVagrant: return "Vagrant" } From e58fa368c62fe221cfc660d4a3f8215bb51e32fc Mon Sep 17 00:00:00 2001 From: Shurkys Date: Wed, 15 Jan 2025 02:17:38 +0200 Subject: [PATCH 05/10] fix view --- templates/package/metadata/terraform.tmpl | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 templates/package/metadata/terraform.tmpl diff --git a/templates/package/metadata/terraform.tmpl b/templates/package/metadata/terraform.tmpl new file mode 100644 index 0000000000000..87fdf2c2f9404 --- /dev/null +++ b/templates/package/metadata/terraform.tmpl @@ -0,0 +1,5 @@ +{{if eq .PackageDescriptor.Package.Type "terrafomr"}} + {{if .PackageDescriptor.Metadata.Author}}
{{svg "octicon-person"}} {{.PackageDescriptor.Metadata.Author}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external"}} {{ctx.Locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external"}} {{ctx.Locale.Tr "packages.details.repository_site"}}
{{end}} +{{end}} From 0d3d5c7393ba7f8f45fd4217c319bf1e1a9cf18c Mon Sep 17 00:00:00 2001 From: Shurkys Date: Wed, 15 Jan 2025 02:43:14 +0200 Subject: [PATCH 06/10] make fmt --- routers/api/packages/terraform/terraform.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/api/packages/terraform/terraform.go b/routers/api/packages/terraform/terraform.go index e213d76ef1feb..3945d7172593c 100644 --- a/routers/api/packages/terraform/terraform.go +++ b/routers/api/packages/terraform/terraform.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/routers/api/packages/helper" "code.gitea.io/gitea/services/context" packages_service "code.gitea.io/gitea/services/packages" + "github.com/google/uuid" ) From 86fa672d1fe6dbc8035db9e74c654cdb4ca90132 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 14 Jan 2025 21:55:34 -0500 Subject: [PATCH 07/10] Apply suggestions from code review Co-authored-by: Lunny Xiao --- modules/packages/terraform/metadata.go | 2 +- modules/packages/terraform/metadata_test.go | 2 +- routers/api/packages/terraform/terraform.go | 2 +- tests/integration/api_packages_terraform_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/packages/terraform/metadata.go b/modules/packages/terraform/metadata.go index 14df7cf548d3c..6dfc0d66cee1b 100644 --- a/modules/packages/terraform/metadata.go +++ b/modules/packages/terraform/metadata.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package terraform diff --git a/modules/packages/terraform/metadata_test.go b/modules/packages/terraform/metadata_test.go index b2d5164f5e2c7..657c32588d428 100644 --- a/modules/packages/terraform/metadata_test.go +++ b/modules/packages/terraform/metadata_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package terraform diff --git a/routers/api/packages/terraform/terraform.go b/routers/api/packages/terraform/terraform.go index 3945d7172593c..f69ed138215ea 100644 --- a/routers/api/packages/terraform/terraform.go +++ b/routers/api/packages/terraform/terraform.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package terraform diff --git a/tests/integration/api_packages_terraform_test.go b/tests/integration/api_packages_terraform_test.go index 0e7df0104d05f..9c067a99b37e4 100644 --- a/tests/integration/api_packages_terraform_test.go +++ b/tests/integration/api_packages_terraform_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package integration From b5dd7ea6e1fd484d3bc75f553c0255ba0da32dae Mon Sep 17 00:00:00 2001 From: Shurkys <1240425+shurkys@users.noreply.github.com> Date: Fri, 17 Jan 2025 02:23:11 +0200 Subject: [PATCH 08/10] Some refactoring --- routers/api/packages/terraform/terraform.go | 360 +++++++------------- 1 file changed, 123 insertions(+), 237 deletions(-) diff --git a/routers/api/packages/terraform/terraform.go b/routers/api/packages/terraform/terraform.go index f69ed138215ea..61cd4c27c39c6 100644 --- a/routers/api/packages/terraform/terraform.go +++ b/routers/api/packages/terraform/terraform.go @@ -10,16 +10,15 @@ import ( "io" "net/http" "strconv" - "sync" + "time" packages_model "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" packages_module "code.gitea.io/gitea/modules/packages" - "code.gitea.io/gitea/routers/api/packages/helper" "code.gitea.io/gitea/services/context" - packages_service "code.gitea.io/gitea/services/packages" + "code.gitea.io/gitea/services/packages" "github.com/google/uuid" ) @@ -46,157 +45,105 @@ type InstanceState struct { Attributes map[string]any `json:"attributes"` } -var ( - stateStorage = make(map[string]*TFState) - stateLocks = make(map[string]string) - storeMutex sync.Mutex -) +type LockInfo struct { + ID string `json:"id"` + Created string `json:"created"` +} + +var stateLocks = make(map[string]LockInfo) + +func apiError(ctx *context.Context, status int, message string) { + log.Error("Terraform API Error: %d - %s", status, message) + ctx.JSON(status, map[string]string{"error": message}) +} + +func getLockID(ctx *context.Context) (string, error) { + var lock struct { + ID string `json:"ID"` + } -func apiError(ctx *context.Context, status int, obj any) { - helper.LogAndProcessError(ctx, status, obj, func(message string) { - type Error struct { - Status int `json:"status"` - Message string `json:"message"` + // Read the body of the request and try to parse the JSON + body, err := io.ReadAll(ctx.Req.Body) + if err == nil && len(body) > 0 { + if err := json.Unmarshal(body, &lock); err != nil { + log.Error("Failed to unmarshal request body: %v", err) + return "", err } - ctx.JSON(status, struct { - Errors []Error `json:"errors"` - }{ - Errors: []Error{ - {Status: status, Message: message}, - }, - }) - }) + } + + // We check the presence of lock ID in the request body or request parameters + if lock.ID == "" { + lock.ID = ctx.Req.URL.Query().Get("ID") + } + + if lock.ID == "" { + apiError(ctx, http.StatusBadRequest, "Missing lock ID") + return "", fmt.Errorf("missing lock ID") + } + + log.Info("Extracted lockID: %s", lock.ID) + return lock.ID, nil } func GetState(ctx *context.Context) { stateName := ctx.PathParam("statename") - log.Info("Function GetState called with parameters: stateName=%s", stateName) - - // Find the package version - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: packages_model.TypeTerraform, - Name: packages_model.SearchValue{ - ExactMatch: true, - Value: stateName, - }, + log.Info("GetState called for: %s", stateName) + + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeTerraform, + Name: packages_model.SearchValue{ExactMatch: true, Value: stateName}, HasFileWithName: stateName, IsInternal: optional.Some(false), + Sort: packages_model.SortCreatedDesc, }) if err != nil { - log.Error("Failed to search package versions for state %s: %v", stateName, err) - apiError(ctx, http.StatusInternalServerError, err) + apiError(ctx, http.StatusInternalServerError, "Failed to fetch latest versions") return } - // If no version is found, return 204 if len(pvs) == 0 { - log.Info("No existing state found for %s, returning 204 No Content", stateName) - ctx.Resp.WriteHeader(http.StatusNoContent) + apiError(ctx, http.StatusNoContent, "No content available") return } - // Get the latest package version - stateVersion := pvs[0] - if stateVersion == nil { - log.Error("State version is nil for state %s", stateName) - apiError(ctx, http.StatusInternalServerError, "Invalid state version") - return - } - log.Info("Fetching file stream for state %s with version %s", stateName, stateVersion.Version) - - // Log the parameters of GetFileStreamByPackageNameAndVersion call - log.Info("Fetching file stream with params: Owner=%v, PackageType=%v, Name=%v, Version=%v, Filename=%v", - ctx.Package.Owner, - packages_model.TypeTerraform, - stateName, - stateVersion.Version, - stateName, - ) - - // Fetch the file stream - s, _, _, err := packages_service.GetFileStreamByPackageNameAndVersion( - ctx, - &packages_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: packages_model.TypeTerraform, - Name: stateName, - Version: stateVersion.Version, - }, - &packages_service.PackageFileInfo{ - Filename: stateName, - }, - ) + stream, _, _, err := packages.GetFileStreamByPackageNameAndVersion(ctx, &packages.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeTerraform, + Name: stateName, + Version: pvs[0].Version, + }, &packages.PackageFileInfo{Filename: stateName}) if err != nil { - log.Error("Error fetching file stream for state %s: %v", stateName, err) - if errors.Is(err, packages_model.ErrPackageNotExist) { - log.Error("Package does not exist: %v", err) + switch { + case errors.Is(err, packages_model.ErrPackageNotExist): apiError(ctx, http.StatusNotFound, "Package not found") - return - } - if errors.Is(err, packages_model.ErrPackageFileNotExist) { - log.Error("Package file does not exist: %v", err) + case errors.Is(err, packages_model.ErrPackageFileNotExist): apiError(ctx, http.StatusNotFound, "File not found") - return + default: + apiError(ctx, http.StatusInternalServerError, err.Error()) } - apiError(ctx, http.StatusInternalServerError, "Failed to fetch file stream") return } - defer s.Close() + defer stream.Close() - // Read the file contents - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, s); err != nil { - log.Error("Failed to read state file for %s: %v", stateName, err) - apiError(ctx, http.StatusInternalServerError, "Failed to read state file") - return - } - - // Deserialize the state var state TFState - if err := json.Unmarshal(buf.Bytes(), &state); err != nil { - log.Error("Failed to unmarshal state file for %s: %v", stateName, err) - apiError(ctx, http.StatusInternalServerError, "Invalid state file format") + if err := json.NewDecoder(stream).Decode(&state); err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to parse state file") return } - // Ensure lineage is set if state.Lineage == "" { state.Lineage = uuid.NewString() - log.Info("Generated new lineage for state %s: %s", stateName, state.Lineage) + log.Info("Generated new lineage for state: %s", state.Lineage) } - // Send the state in the response ctx.Resp.Header().Set("Content-Type", "application/json") ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", stateName)) - ctx.Resp.WriteHeader(http.StatusOK) - if _, writeErr := ctx.Resp.Write(buf.Bytes()); writeErr != nil { - log.Error("Failed to write response for state %s: %v", stateName, writeErr) - } + ctx.JSON(http.StatusOK, state) } -// UpdateState updates or creates a new Terraform state and interacts with Gitea packages. func UpdateState(ctx *context.Context) { stateName := ctx.PathParam("statename") - log.Info("UpdateState called for stateName: %s", stateName) - - storeMutex.Lock() - defer storeMutex.Unlock() - - // Check for the presence of a lock ID - requestLockID := ctx.Req.URL.Query().Get("ID") - if requestLockID == "" { - apiError(ctx, http.StatusBadRequest, "Missing ID query parameter") - return - } - - // Check for blocking state - if lockID, locked := stateLocks[stateName]; locked && lockID != requestLockID { - apiError(ctx, http.StatusConflict, fmt.Sprintf("State %s is locked", stateName)) - return - } - - // Read the request body body, err := io.ReadAll(ctx.Req.Body) if err != nil { apiError(ctx, http.StatusInternalServerError, "Failed to read request body") @@ -209,37 +156,36 @@ func UpdateState(ctx *context.Context) { return } - // Getting the current serial - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: packages_model.TypeTerraform, - Name: packages_model.SearchValue{ExactMatch: true, Value: stateName}, - IsInternal: optional.Some(false), + pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeTerraform, + Name: packages_model.SearchValue{ExactMatch: true, Value: stateName}, + HasFileWithName: stateName, + IsInternal: optional.Some(false), + Sort: packages_model.SortCreatedDesc, }) if err != nil { - apiError(ctx, http.StatusInternalServerError, "Failed to search package versions") + apiError(ctx, http.StatusInternalServerError, err.Error()) return } - - serial := uint64(1) // Start from 1 + serial := uint64(0) if len(pvs) > 0 { - lastSerial, _ := strconv.ParseUint(pvs[0].Version, 10, 64) - serial = lastSerial + 1 + if lastSerial, err := strconv.ParseUint(pvs[0].Version, 10, 64); err == nil { + serial = lastSerial + 1 + } } - log.Info("State %s updated to serial %d", stateName, serial) - // Create package information packageVersion := fmt.Sprintf("%d", serial) - packageInfo := &packages_service.PackageCreationInfo{ - PackageInfo: packages_service.PackageInfo{ + + packageInfo := &packages.PackageCreationInfo{ + PackageInfo: packages.PackageInfo{ Owner: ctx.Package.Owner, PackageType: packages_model.TypeTerraform, Name: stateName, Version: packageVersion, }, - SemverCompatible: true, - Creator: ctx.Doer, - Metadata: newState, + Creator: ctx.Doer, + Metadata: newState, } buffer, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(body)) @@ -247,146 +193,86 @@ func UpdateState(ctx *context.Context) { apiError(ctx, http.StatusInternalServerError, "Failed to create buffer") return } - - // Create/update package - if _, _, err = packages_service.CreatePackageOrAddFileToExisting( - ctx, - packageInfo, - &packages_service.PackageFileCreationInfo{ - PackageFileInfo: packages_service.PackageFileInfo{ - Filename: stateName, - }, - Creator: ctx.Doer, - Data: buffer, - IsLead: true, - }, - ); err != nil { + _, _, err = packages.CreatePackageOrAddFileToExisting(ctx, packageInfo, &packages.PackageFileCreationInfo{ + PackageFileInfo: packages.PackageFileInfo{Filename: stateName}, + Creator: ctx.Doer, + Data: buffer, + IsLead: true, + }) + if err != nil { apiError(ctx, http.StatusInternalServerError, "Failed to update package") return } - log.Info("State %s updated successfully with version %s", stateName, packageVersion) - - ctx.JSON(http.StatusOK, map[string]string{ - "message": "State updated successfully", - "statename": stateName, - }) + ctx.JSON(http.StatusOK, map[string]string{"message": "State updated successfully", "statename": stateName}) } -// LockState locks a Terraform state to prevent updates. func LockState(ctx *context.Context) { stateName := ctx.PathParam("statename") - log.Info("LockState called for state: %s", stateName) - - // Read the request body - body, err := io.ReadAll(ctx.Req.Body) + lockID, err := getLockID(ctx) if err != nil { - log.Error("Failed to read request body: %v", err) - apiError(ctx, http.StatusInternalServerError, "Failed to read request body") + apiError(ctx, http.StatusBadRequest, err.Error()) return } - // Decode JSON and check lockID - var lockRequest struct { - ID string `json:"ID"` - } - if err := json.Unmarshal(body, &lockRequest); err != nil || lockRequest.ID == "" { - log.Error("Invalid lock request body: %v", err) - apiError(ctx, http.StatusBadRequest, "Invalid or missing lock ID") - return - } - - storeMutex.Lock() - defer storeMutex.Unlock() - // Check if the state is locked - if _, locked := stateLocks[stateName]; locked { + if lockInfo, locked := stateLocks[stateName]; locked { log.Warn("State %s is already locked", stateName) - apiError(ctx, http.StatusConflict, fmt.Sprintf("State %s is already locked", stateName)) + + // Generate a response for the conflict with information about the current lock + response := lockInfo // Return full information about the lock + ctx.JSON(http.StatusConflict, response) return } // Set the lock - stateLocks[stateName] = lockRequest.ID - log.Info("State %s locked with ID %s", stateName, lockRequest.ID) + stateLocks[stateName] = LockInfo{ + ID: lockID, + Created: time.Now().UTC().Format(time.RFC3339), + } - ctx.JSON(http.StatusOK, map[string]string{ - "message": "State locked successfully", - "statename": stateName, - }) + log.Info("Locked state: %s with ID: %s", stateName, lockID) + ctx.JSON(http.StatusOK, map[string]string{"message": "State locked successfully", "statename": stateName}) } -// UnlockState unlocks a Terraform state. func UnlockState(ctx *context.Context) { stateName := ctx.PathParam("statename") - log.Info("UnlockState called for state: %s", stateName) - - // Extract lockID from request body or parameters - var unlockRequest struct { - ID string `json:"ID"` - } - - // Trying to read the request body - body, _ := io.ReadAll(ctx.Req.Body) - if len(body) > 0 { - _ = json.Unmarshal(body, &unlockRequest) // The error can be ignored, since the ID can also be in the query - } - - // Check for ID presence - if unlockRequest.ID == "" { - log.Error("Missing lock ID in both query and request body") - apiError(ctx, http.StatusBadRequest, "Missing lock ID") + lockID, err := getLockID(ctx) + if err != nil { + apiError(ctx, http.StatusBadRequest, err.Error()) return } - log.Info("Extracted lockID: %s", unlockRequest.ID) - - storeMutex.Lock() - defer storeMutex.Unlock() - // Check the lock status - currentLockID, locked := stateLocks[stateName] - if !locked || currentLockID != unlockRequest.ID { - log.Warn("Unlock attempt failed for state %s with lock ID %s", stateName, unlockRequest.ID) + currentLockInfo, locked := stateLocks[stateName] + if !locked || currentLockInfo.ID != lockID { + log.Warn("Unlock attempt failed for state %s with lock ID %s", stateName, lockID) apiError(ctx, http.StatusConflict, fmt.Sprintf("State %s is not locked or lock ID mismatch", stateName)) return } // Remove the lock delete(stateLocks, stateName) - log.Info("State %s unlocked successfully", stateName) - - ctx.JSON(http.StatusOK, map[string]string{ - "message": "State unlocked successfully", - "statename": stateName, - }) + log.Info("Unlocked state: %s with ID: %s", stateName, lockID) + ctx.JSON(http.StatusOK, map[string]string{"message": "State unlocked successfully"}) } -// DeleteState deletes the Terraform state for a given name. func DeleteState(ctx *context.Context) { stateName := ctx.PathParam("statename") - log.Info("Attempting to delete state: %s", stateName) - - storeMutex.Lock() - defer storeMutex.Unlock() - - // Check if a state or lock exists - _, stateExists := stateStorage[stateName] - _, lockExists := stateLocks[stateName] - - if !stateExists && !lockExists { - log.Warn("State %s does not exist or is not locked", stateName) - apiError(ctx, http.StatusNotFound, "State not found") + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeTerraform, stateName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to fetch package versions") return } - - // Delete the state and lock - delete(stateStorage, stateName) - delete(stateLocks, stateName) - - log.Info("State %s deleted successfully", stateName) - ctx.JSON(http.StatusOK, map[string]string{ - "message": "State deleted successfully", - "statename": stateName, - }) + if len(pvs) == 0 { + ctx.Status(http.StatusNoContent) + return + } + for _, pv := range pvs { + if err := packages.RemovePackageVersion(ctx, ctx.Doer, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, "Failed to delete package version") + return + } + } + ctx.JSON(http.StatusOK, map[string]string{"message": "State deleted successfully"}) } From 98775acb60bd0c10774caab3a59afa5717fd9137 Mon Sep 17 00:00:00 2001 From: Shurkys <1240425+shurkys@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:03:52 +0200 Subject: [PATCH 09/10] Fit TF tests --- .../api_packages_terraform_test.go | 212 ++++++++---------- 1 file changed, 94 insertions(+), 118 deletions(-) diff --git a/tests/integration/api_packages_terraform_test.go b/tests/integration/api_packages_terraform_test.go index 9c067a99b37e4..7e17f3db9e6a2 100644 --- a/tests/integration/api_packages_terraform_test.go +++ b/tests/integration/api_packages_terraform_test.go @@ -4,151 +4,127 @@ package integration import ( - "archive/tar" "bytes" - "compress/gzip" "fmt" + "io" "net/http" "strings" "testing" - auth_model "code.gitea.io/gitea/models/auth" - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" - terraform_module "code.gitea.io/gitea/modules/packages/terraform" "code.gitea.io/gitea/tests" + gouuid "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPackageTerraform(t *testing.T) { defer tests.PrepareTestEnv(t)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - token := "Bearer " + getUserToken(t, user.Name, auth_model.AccessTokenScopeWritePackage) - - packageName := "test_module" - packageVersion := "1.0.1" - packageDescription := "Test Terraform Module" - - filename := "terraform_module.tar.gz" - - infoContent, _ := json.Marshal(map[string]string{ - "description": packageDescription, - }) - - var buf bytes.Buffer - zw := gzip.NewWriter(&buf) - archive := tar.NewWriter(zw) - archive.WriteHeader(&tar.Header{ - Name: "info.json", - Mode: 0o600, - Size: int64(len(infoContent)), - }) - archive.Write(infoContent) - archive.Close() - zw.Close() - content := buf.Bytes() - - root := fmt.Sprintf("/api/packages/%s/terraform", user.Name) - - t.Run("Authenticate", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - authenticateURL := fmt.Sprintf("%s/authenticate", root) - - req := NewRequest(t, "GET", authenticateURL) - MakeRequest(t, req, http.StatusUnauthorized) - - req = NewRequest(t, "GET", authenticateURL). - AddTokenAuth(token) - MakeRequest(t, req, http.StatusOK) - }) - - moduleURL := fmt.Sprintf("%s/%s", root, packageName) - + // Get token for the user + token := "Bearer " + getUserToken(t, user.Name, auth.AccessTokenScopeWritePackage) + + // Define important values + lineage := "bca3c5f6-01dc-cdad-5310-d1b12e02e430" + terraformVersion := "1.10.4" + serial := float64(1) + resourceName := "hello" + resourceType := "null_resource" + id := gouuid.New().String() // Generate a unique ID + + // Build the state JSON + buildState := func() string { + return `{ + "version": 4, + "terraform_version": "` + terraformVersion + `", + "serial": ` + fmt.Sprintf("%.0f", serial) + `, + "lineage": "` + lineage + `", + "outputs": {}, + "resources": [{ + "mode": "managed", + "type": "` + resourceType + `", + "name": "` + resourceName + `", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [{ + "schema_version": 0, + "attributes": { + "id": "3832416504545530133", + "triggers": null + }, + "sensitive_attributes": [] + }] + }], + "check_results": null + }` + } + state := buildState() + content := []byte(state) + root := fmt.Sprintf("/api/packages/%s/terraform/state", user.Name) + stateURL := fmt.Sprintf("%s/providers-gitea.tfstate", root) + + // Upload test t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "HEAD", moduleURL) - MakeRequest(t, req, http.StatusNotFound) - - uploadURL := fmt.Sprintf("%s/%s/%s", moduleURL, packageVersion, filename) - - req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) - MakeRequest(t, req, http.StatusUnauthorized) - - req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)). - AddTokenAuth(token) - MakeRequest(t, req, http.StatusCreated) - - req = NewRequest(t, "HEAD", moduleURL) - resp := MakeRequest(t, req, http.StatusOK) - assert.True(t, strings.HasPrefix(resp.Header().Get("Content-Type"), "application/json")) - - pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeTerraform) - assert.NoError(t, err) - assert.Len(t, pvs, 1) - - pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) - assert.NoError(t, err) - assert.NotNil(t, pd.SemVer) - assert.IsType(t, &terraform_module.Metadata{}, pd.Metadata) - assert.Equal(t, packageName, pd.Package.Name) - assert.Equal(t, packageVersion, pd.Version.Version) - - pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) - assert.NoError(t, err) - assert.Len(t, pfs, 1) - assert.Equal(t, filename, pfs[0].Name) - assert.True(t, pfs[0].IsLead) - - pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) - assert.NoError(t, err) - assert.Equal(t, int64(len(content)), pb.Size) - - req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)). - AddTokenAuth(token) - MakeRequest(t, req, http.StatusConflict) + uploadURL := fmt.Sprintf("%s?ID=%s", stateURL, id) + req := NewRequestWithBody(t, "POST", uploadURL, bytes.NewReader(content)).AddTokenAuth(token) + resp := MakeRequest(t, req, http.StatusOK) // Expecting 200 OK + assert.Equal(t, http.StatusOK, resp.Code) + assert.Contains(t, resp.Header().Get("Content-Type"), "application/json") + bodyBytes, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NotEmpty(t, bodyBytes) }) + // Download test t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", moduleURL, packageVersion, filename)) + downloadURL := fmt.Sprintf("%s?ID=%s", stateURL, id) + req := NewRequest(t, "GET", downloadURL) resp := MakeRequest(t, req, http.StatusOK) + assert.True(t, strings.HasPrefix(resp.Header().Get("Content-Type"), "application/json")) - assert.Equal(t, content, resp.Body.Bytes()) + bodyBytes, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NotEmpty(t, bodyBytes) + + var jsonResponse map[string]any + err = json.Unmarshal(bodyBytes, &jsonResponse) + require.NoError(t, err) + + // Validate the response + assert.Equal(t, lineage, jsonResponse["lineage"]) + assert.Equal(t, terraformVersion, jsonResponse["terraform_version"]) + assert.Equal(t, serial, jsonResponse["serial"]) + resource := jsonResponse["resources"].([]any)[0].(map[string]any) + assert.Equal(t, resourceName, resource["name"]) + assert.Equal(t, resourceType, resource["type"]) + assert.NotContains(t, resource, "sensitive_attributes") }) - t.Run("EnumeratePackageVersions", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + // Lock state test + t.Run("LockState", func(t *testing.T) { + lockURL := fmt.Sprintf("%s/lock?ID=%s", stateURL, id) + req := NewRequestWithBody(t, "POST", lockURL, bytes.NewReader(content)).AddTokenAuth(token) + resp := MakeRequest(t, req, http.StatusOK) // Expecting 200 OK + assert.Equal(t, http.StatusOK, resp.Code) + }) - req := NewRequest(t, "GET", moduleURL) - resp := MakeRequest(t, req, http.StatusOK) + // Unlock state test + t.Run("UnlockState", func(t *testing.T) { + unlockURL := fmt.Sprintf("%s/lock?ID=%s", stateURL, id) + req := NewRequestWithBody(t, "DELETE", unlockURL, bytes.NewReader(content)).AddTokenAuth(token) + resp := MakeRequest(t, req, http.StatusOK) // Expecting 200 OK + assert.Equal(t, http.StatusOK, resp.Code) + }) - type versionMetadata struct { - Version string `json:"version"` - Status string `json:"status"` - } - - type packageMetadata struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Versions []*versionMetadata `json:"versions"` - } - - var result packageMetadata - DecodeJSON(t, resp, &result) - - assert.Equal(t, packageName, result.Name) - assert.Equal(t, packageDescription, result.Description) - assert.Len(t, result.Versions, 1) - version := result.Versions[0] - assert.Equal(t, packageVersion, version.Version) - assert.Equal(t, "active", version.Status) + // Download not found test + t.Run("DownloadNotFound", func(t *testing.T) { + invalidStateURL := fmt.Sprintf("%s/invalid-state.tfstate?ID=%s", root, id) + req := NewRequest(t, "GET", invalidStateURL) + resp := MakeRequest(t, req, http.StatusNoContent) // Expecting 204 No Content + assert.Equal(t, http.StatusNoContent, resp.Code) }) } From 0dc7c5509c0bea6a13cec98f794c71dd65f44391 Mon Sep 17 00:00:00 2001 From: Shurkys <1240425+shurkys@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:34:06 +0200 Subject: [PATCH 10/10] fix lint api_packages_terraform_test --- tests/integration/api_packages_terraform_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_packages_terraform_test.go b/tests/integration/api_packages_terraform_test.go index 7e17f3db9e6a2..424b4034617ca 100644 --- a/tests/integration/api_packages_terraform_test.go +++ b/tests/integration/api_packages_terraform_test.go @@ -97,7 +97,7 @@ func TestPackageTerraform(t *testing.T) { // Validate the response assert.Equal(t, lineage, jsonResponse["lineage"]) assert.Equal(t, terraformVersion, jsonResponse["terraform_version"]) - assert.Equal(t, serial, jsonResponse["serial"]) + assert.InEpsilon(t, serial, jsonResponse["serial"].(float64), 0.0001) resource := jsonResponse["resources"].([]any)[0].(map[string]any) assert.Equal(t, resourceName, resource["name"]) assert.Equal(t, resourceType, resource["type"])