feat: support a new package type pypi

.github/workflows/wc-integration-test.yaml

@@ -65,6 +65,15 @@ jobs:
       - name: Test aqua gr cargo
         run: aqua gr crates.io/skim
 
+      - name: aqua which pypi
+        run: aqua which pre-commit
+      - name: Test the package type "pypi"
+        run: pre-commit --version
+      - name: Test search versions of the package type "pypi"
+        run: aqua -c tests/main/aqua-global.yaml g local,pypi.org/pre-commit
+      - name: Test aqua gr pypi
+        run: aqua gr pypi.org/pre-commit
+
       - name: test cosign
         run: aqua i
         working-directory: tests/cosign

.github/workflows/windows-test.yaml

@@ -82,6 +82,22 @@ jobs:
       - run: helm version
       - run: terrafmt version
 
+      - name: Test the package type "cargo"
+        run: sk --version
+      - name: Test search versions of the package type "cargo"
+        run: aqua -c tests/main/aqua-global.yaml g local,crates.io/skim
+      - name: Test aqua gr cargo
+        run: aqua gr crates.io/skim
+
+      - name: aqua which pypi
+        run: aqua which pre-commit
+      - name: Test the package type "pypi"
+        run: pre-commit --version
+      - name: Test search versions of the package type "pypi"
+        run: aqua -c tests/main/aqua-global.yaml g local,pypi.org/pre-commit
+      - name: Test aqua gr pypi
+        run: aqua gr pypi.org/pre-commit
+
       - run: aqua g -i suzuki-shunsuke/tfcmt
         working-directory: tests/main
       - run: git diff aqua.yaml

json-schema/aqua-yaml.json

@@ -65,6 +65,12 @@
             "type": "string"
           },
           "type": "array"
+        },
+        "description": {
+          "type": "string"
+        },
+        "link": {
+          "type": "string"
         }
       },
       "additionalProperties": false,

json-schema/registry.json

@@ -277,6 +277,9 @@
         "cargo": {
           "$ref": "#/$defs/Cargo"
         },
+        "pypi_name": {
+          "type": "string"
+        },
         "files": {
           "items": {
             "$ref": "#/$defs/File"
@@ -370,6 +373,9 @@
         "path": {
           "type": "string"
         },
+        "pypi_name": {
+          "type": "string"
+        },
         "format": {
           "type": "string",
           "examples": [
@@ -569,6 +575,9 @@
         "url": {
           "type": "string"
         },
+        "pypi_name": {
+          "type": "string"
+        },
         "files": {
           "items": {
             "$ref": "#/$defs/File"

pkg/config/package.go

@@ -91,6 +91,9 @@ func (cpkg *Package) GetPkgPath(rootDir string, rt *runtime.Runtime) (string, er
 	case PkgInfoTypeCargo:
 		registry := "crates.io"
 		return filepath.Join(rootDir, "pkgs", pkgInfo.GetType(), registry, *pkgInfo.Crate, pkg.Version), nil
+	case PkgInfoTypePypi:
+		registry := "pypi.org"
+		return filepath.Join(rootDir, "pkgs", pkgInfo.GetType(), registry, *pkgInfo.PypiName, pkg.Version), nil
 	case PkgInfoTypeGitHubContent, PkgInfoTypeGitHubRelease:
 		if pkgInfo.RepoOwner == "aquaproj" && (pkgInfo.RepoName == "aqua" || pkgInfo.RepoName == "aqua-proxy") {
 			return filepath.Join(rootDir, "internal", "pkgs", pkgInfo.GetType(), "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
@@ -193,6 +196,9 @@ func (cpkg *Package) getFileSrcWithoutWindowsExt(file *registry.File, rt *runtim
 	if pkgInfo.Type == "cargo" {
 		return filepath.Join("bin", file.Name), nil
 	}
+	if pkgInfo.Type == "pypi" {
+		return filepath.Join("bin", file.Name), nil
+	}
 	assetName, err := cpkg.RenderAsset(rt)
 	if err != nil {
 		return "", fmt.Errorf("render the asset name: %w", err)
@@ -218,6 +224,7 @@ const (
 	PkgInfoTypeGoInstall     = "go_install"
 	PkgInfoTypeGoBuild       = "go_build"
 	PkgInfoTypeCargo         = "cargo"
+	PkgInfoTypePypi          = "pypi"
 )
 
 type Param struct {

pkg/config/registry/error.go

@@ -8,6 +8,7 @@ var (
 	errGitHubContentRequirePath = errors.New("github_content package requires path")
 	errGoInstallRequirePath     = errors.New("go_install package requires path")
 	errCargoRequireCrate        = errors.New("cargo package requires crate")
+	errPypiRequirePypiName      = errors.New("pypi package requires pypi_name")
 	errAssetRequired            = errors.New("github_release package requires asset")
 	errURLRequired              = errors.New("http package requires url")
 	errInvalidPackageType       = errors.New("package type is invalid")

pkg/config/registry/package_info.go

@@ -18,6 +18,7 @@ const (
 	PkgInfoTypeGoInstall     = "go_install"
 	PkgInfoTypeGoBuild       = "go_build"
 	PkgInfoTypeCargo         = "cargo"
+	PkgInfoTypePypi          = "pypi"
 )
 
 type PackageInfo struct {
@@ -34,6 +35,7 @@ type PackageInfo struct {
 	Cargo              *Cargo             `json:"cargo,omitempty"`
 	URL                *string            `json:"url,omitempty" yaml:",omitempty"`
 	Path               *string            `json:"path,omitempty" yaml:",omitempty"`
+	PypiName           *string            `json:"pypi_name,omitempty" yaml:"pypi_name,omitempty"`
 	Format             string             `json:"format,omitempty" jsonschema:"example=tar.gz,example=raw,example=zip,example=dmg" yaml:",omitempty"`
 	Overrides          []*Override        `json:"overrides,omitempty" yaml:",omitempty"`
 	FormatOverrides    []*FormatOverride  `yaml:"format_overrides,omitempty" json:"format_overrides,omitempty"`
@@ -66,6 +68,7 @@ type VersionOverride struct {
 	Cargo              *Cargo          `json:"cargo,omitempty"`
 	Path               *string         `yaml:",omitempty" json:"path,omitempty"`
 	URL                *string         `yaml:",omitempty" json:"url,omitempty"`
+	PypiName           *string         `json:"pypi_name,omitempty" yaml:"pypi_name,omitempty"`
 	Files              []*File         `yaml:",omitempty" json:"files,omitempty"`
 	Format             string          `yaml:",omitempty" json:"format,omitempty" jsonschema:"example=tar.gz,example=raw,example=zip"`
 	FormatOverrides    FormatOverrides `yaml:"format_overrides,omitempty" json:"format_overrides,omitempty"`
@@ -93,6 +96,7 @@ type Override struct {
 	Asset              *string         `yaml:",omitempty" json:"asset,omitempty"`
 	Crate              *string         `json:"crate,omitempty" yaml:",omitempty"`
 	Cargo              *Cargo          `json:"cargo,omitempty"`
+	PypiName           *string         `json:"pypi_name,omitempty" yaml:"pypi_name,omitempty"`
 	Files              []*File         `yaml:",omitempty" json:"files,omitempty"`
 	URL                *string         `yaml:",omitempty" json:"url,omitempty"`
 	CompleteWindowsExt *bool           `json:"complete_windows_ext,omitempty" yaml:"complete_windows_ext,omitempty"`
@@ -113,6 +117,7 @@ func (pkgInfo *PackageInfo) Copy() *PackageInfo {
 		Crate:              pkgInfo.Crate,
 		Cargo:              pkgInfo.Cargo,
 		Path:               pkgInfo.Path,
+		PypiName:           pkgInfo.PypiName,
 		Format:             pkgInfo.Format,
 		Files:              pkgInfo.Files,
 		URL:                pkgInfo.URL,
@@ -145,26 +150,33 @@ func (pkgInfo *PackageInfo) resetByPkgType(typ string) { //nolint:funlen
 	switch typ {
 	case PkgInfoTypeGitHubRelease:
 		pkgInfo.URL = nil
+		pkgInfo.PypiName = nil
 		pkgInfo.Path = nil
 		pkgInfo.Crate = nil
 		pkgInfo.Cargo = nil
 	case PkgInfoTypeGitHubContent:
 		pkgInfo.URL = nil
+		pkgInfo.PypiName = nil
 		pkgInfo.Asset = nil
 		pkgInfo.Crate = nil
 		pkgInfo.Cargo = nil
 	case PkgInfoTypeGitHubArchive:
+		pkgInfo.PypiName = nil
 		pkgInfo.URL = nil
 		pkgInfo.Path = nil
 		pkgInfo.Asset = nil
 		pkgInfo.Crate = nil
 		pkgInfo.Cargo = nil
 		pkgInfo.Format = ""
 	case PkgInfoTypeHTTP:
+		pkgInfo.PypiName = nil
 		pkgInfo.Path = nil
 		pkgInfo.Asset = nil
+		pkgInfo.Crate = nil
+		pkgInfo.Cargo = nil
 	case PkgInfoTypeGoInstall:
 		pkgInfo.URL = nil
+		pkgInfo.PypiName = nil
 		pkgInfo.Asset = nil
 		pkgInfo.Crate = nil
 		pkgInfo.Cargo = nil
@@ -176,6 +188,7 @@ func (pkgInfo *PackageInfo) resetByPkgType(typ string) { //nolint:funlen
 		pkgInfo.Rosetta2 = nil
 	case PkgInfoTypeGoBuild:
 		pkgInfo.URL = nil
+		pkgInfo.PypiName = nil
 		pkgInfo.Asset = nil
 		pkgInfo.Crate = nil
 		pkgInfo.Cargo = nil
@@ -186,15 +199,29 @@ func (pkgInfo *PackageInfo) resetByPkgType(typ string) { //nolint:funlen
 		pkgInfo.Format = ""
 		pkgInfo.Rosetta2 = nil
 	case PkgInfoTypeCargo:
+		pkgInfo.URL = nil
+		pkgInfo.PypiName = nil
+		pkgInfo.Asset = nil
+		pkgInfo.Path = nil
+		pkgInfo.WindowsExt = ""
+		pkgInfo.CompleteWindowsExt = nil
+		pkgInfo.Cosign = nil
+		pkgInfo.SLSAProvenance = nil
+		pkgInfo.Format = ""
+		pkgInfo.Rosetta2 = nil
+	case PkgInfoTypePypi:
 		pkgInfo.URL = nil
 		pkgInfo.Asset = nil
 		pkgInfo.Path = nil
 		pkgInfo.WindowsExt = ""
 		pkgInfo.CompleteWindowsExt = nil
 		pkgInfo.Cosign = nil
 		pkgInfo.SLSAProvenance = nil
+		pkgInfo.Checksum = nil
 		pkgInfo.Format = ""
 		pkgInfo.Rosetta2 = nil
+		pkgInfo.Crate = nil
+		pkgInfo.Cargo = nil
 	}
 }
 
@@ -276,6 +303,9 @@ func (pkgInfo *PackageInfo) overrideVersion(child *VersionOverride) *PackageInfo
 	if child.NoAsset != nil {
 		pkg.NoAsset = child.NoAsset
 	}
+	if child.PypiName != nil {
+		pkg.PypiName = child.PypiName
+	}
 	return pkg
 }
 
@@ -350,6 +380,9 @@ func (pkgInfo *PackageInfo) OverrideByRuntime(rt *runtime.Runtime) { //nolint:cy
 	if ov.SLSAProvenance != nil {
 		pkgInfo.SLSAProvenance = ov.SLSAProvenance
 	}
+	if ov.PypiName != nil {
+		pkgInfo.PypiName = ov.PypiName
+	}
 }
 
 type FormatOverrides []*FormatOverride
@@ -427,6 +460,12 @@ func (pkgInfo *PackageInfo) GetName() string {
 	if pkgInfo.Name != "" {
 		return pkgInfo.Name
 	}
+	if pkgInfo.Type == PkgInfoTypePypi {
+		return fmt.Sprintf("pypi.org/" + *pkgInfo.PypiName)
+	}
+	if pkgInfo.Type == PkgInfoTypeCargo {
+		return fmt.Sprintf("crates.io/" + *pkgInfo.Crate)
+	}
 	if pkgInfo.HasRepo() {
 		return pkgInfo.RepoOwner + "/" + pkgInfo.RepoName
 	}
@@ -450,6 +489,9 @@ func (pkgInfo *PackageInfo) GetLink() string {
 	if pkgInfo.Link != "" {
 		return pkgInfo.Link
 	}
+	if pkgInfo.Type == PkgInfoTypePypi {
+		return fmt.Sprintf("https://pypi.org/project/%s/", *pkgInfo.PypiName)
+	}
 	if pkgInfo.HasRepo() {
 		return "https://github.com/" + pkgInfo.RepoOwner + "/" + pkgInfo.RepoName
 	}
@@ -517,6 +559,11 @@ func (pkgInfo *PackageInfo) Validate() error { //nolint:cyclop
 			return errCargoRequireCrate
 		}
 		return nil
+	case PkgInfoTypePypi:
+		if pkgInfo.PypiName == nil {
+			return errPypiRequirePypiName
+		}
+		return nil
 	case PkgInfoTypeGitHubContent:
 		if !pkgInfo.HasRepo() {
 			return errRepoRequired

pkg/controller/exec/exec.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"runtime"
 	"time"
 
@@ -65,7 +66,7 @@ func New(param *config.Param, pkgInstaller installpackage.Installer, whichCtrl w
 	}
 }
 
-func (ctrl *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string, args ...string) (gErr error) {
+func (ctrl *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string, args ...string) (gErr error) { //nolint:cyclop
 	logE = logE.WithField("exe_name", exeName)
 	defer func() {
 		if gErr != nil {
@@ -88,7 +89,7 @@ func (ctrl *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *con
 		return err //nolint:wrapcheck
 	}
 	if findResult.Package == nil {
-		return ctrl.execCommandWithRetry(ctx, logE, findResult.ExePath, args...)
+		return ctrl.execCommandWithRetry(ctx, logE, findResult.ExePath, args, nil)
 	}
 
 	logE = logE.WithFields(logrus.Fields{
@@ -109,7 +110,16 @@ func (ctrl *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *con
 	if err := ctrl.install(ctx, logE, findResult, policyCfgs); err != nil {
 		return err
 	}
-	return ctrl.execCommandWithRetry(ctx, logE, findResult.ExePath, args...)
+	var envs []string
+	if findResult.Package.PackageInfo.Type == "pypi" {
+		pkgPath := filepath.Dir(filepath.Dir(findResult.ExePath))
+		if pythonPath, ok := os.LookupEnv("PYTHONPATH"); ok {
+			envs = []string{fmt.Sprintf("PYTHONPATH=%s:%s", pkgPath, pythonPath)}
+		} else {
+			envs = []string{fmt.Sprintf("PYTHONPATH=%s", pkgPath)}
+		}
+	}
+	return ctrl.execCommandWithRetry(ctx, logE, findResult.ExePath, args, envs)
 }
 
 func (ctrl *Controller) install(ctx context.Context, logE *logrus.Entry, findResult *which.FindResult, policies []*policy.Config) error {
@@ -167,14 +177,14 @@ func wait(ctx context.Context, duration time.Duration) error {
 
 var errFailedToStartProcess = errors.New("it failed to start the process")
 
-func (ctrl *Controller) execCommand(ctx context.Context, exePath string, args ...string) (bool, error) {
+func (ctrl *Controller) execCommand(ctx context.Context, exePath string, args, envs []string) (bool, error) {
 	if ctrl.enabledXSysExec {
-		if err := ctrl.executor.ExecXSys(exePath, args...); err != nil {
+		if err := ctrl.executor.ExecXSysWithEnvs(exePath, args, envs); err != nil {
 			return true, fmt.Errorf("call execve(2): %w", err)
 		}
 		return false, nil
 	}
-	if exitCode, err := ctrl.executor.Exec(ctx, exePath, args...); err != nil {
+	if exitCode, err := ctrl.executor.ExecWithEnvs(ctx, exePath, args, envs); err != nil {
 		// https://pkg.go.dev/os#ProcessState.ExitCode
 		// > ExitCode returns the exit code of the exited process,
 		// > or -1 if the process hasn't exited or was terminated by a signal.
@@ -186,10 +196,10 @@ func (ctrl *Controller) execCommand(ctx context.Context, exePath string, args ..
 	return false, nil
 }
 
-func (ctrl *Controller) execCommandWithRetry(ctx context.Context, logE *logrus.Entry, exePath string, args ...string) error {
+func (ctrl *Controller) execCommandWithRetry(ctx context.Context, logE *logrus.Entry, exePath string, args, envs []string) error {
 	for i := 0; i < 10; i++ {
 		logE.Debug("execute the command")
-		retried, err := ctrl.execCommand(ctx, exePath, args...)
+		retried, err := ctrl.execCommand(ctx, exePath, args, envs)
 		if !retried {
 			return err
 		}

pkg/controller/exec/exec_internal_test.go

@@ -36,7 +36,7 @@ func TestController_execCommand(t *testing.T) {
 				stderr:   os.Stderr,
 				executor: d.executor,
 			}
-			err := ctrl.execCommandWithRetry(ctx, logE, d.exePath, d.args...)
+			err := ctrl.execCommandWithRetry(ctx, logE, d.exePath, d.args, nil)
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/controller/exec/exec_test.go

@@ -152,7 +152,7 @@ packages:
 			whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker)
 			downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient))
 			executor := &exec.Mock{}
-			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &policy.Checker{}, &cosign.MockVerifier{}, &slsa.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{})
+			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &policy.Checker{}, &cosign.MockVerifier{}, &slsa.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockPypiInstaller{}, &installpackage.MockCargoPackageInstaller{})
 			policyFinder := policy.NewConfigFinder(fs)
 			ctrl := execCtrl.New(d.param, pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)), policyFinder)
 			if err := ctrl.Exec(ctx, logE, d.param, d.exeName, d.args...); err != nil {
@@ -248,7 +248,7 @@ packages:
 			whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, afero.NewOsFs(), d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker)
 			downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient))
 			executor := &exec.Mock{}
-			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &policy.Checker{}, &cosign.MockVerifier{}, &slsa.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{})
+			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &policy.Checker{}, &cosign.MockVerifier{}, &slsa.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockPypiInstaller{}, &installpackage.MockCargoPackageInstaller{})
 			ctrl := execCtrl.New(d.param, pkgInstaller, whichCtrl, executor, osEnv, fs, &policy.MockReader{}, policy.NewConfigFinder(fs))
 			b.ResetTimer()
 			for i := 0; i < b.N; i++ {

pkg/controller/exec/interface.go

@@ -3,6 +3,6 @@ package exec
 import "context"
 
 type Executor interface {
-	Exec(ctx context.Context, exePath string, args ...string) (int, error)
-	ExecXSys(exePath string, args ...string) error
+	ExecXSysWithEnvs(exePath string, args, envs []string) error
+	ExecWithEnvs(ctx context.Context, exePath string, args, envs []string) (int, error)
 }