From f2bbb56d36671284d0457c533647ea6a46ff2d8a Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Thu, 6 Feb 2025 10:39:15 +1100 Subject: [PATCH 1/7] feat: register Maven transitive extractor --- .../language/java/pomxmlnet/pomxmlnet.go | 20 ++++++++++++++++--- extractor/filesystem/list/list.go | 16 +++++++++++++-- internal/resolution/client/client.go | 4 ---- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go index 2869789e..a5d77748 100644 --- a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go +++ b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go @@ -42,6 +42,21 @@ type Extractor struct { *datasource.MavenRegistryAPIClient } +// New makes a new pom.xml transitive extractor with required clients. +// Clients are assuming Maven Central as the registry. +func New() *Extractor { + // No need to check errors since we are using the default Maven Central URL. + depClient, _ := client.NewMavenRegistryClient(datasource.MavenCentral) + mavenClient, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{ + URL: datasource.MavenCentral, + ReleasesEnabled: true, + }) + return &Extractor{ + DependencyClient: depClient, + MavenRegistryAPIClient: mavenClient, + } +} + // Name of the extractor. func (e Extractor) Name() string { return "java/pomxmlnet" } @@ -51,12 +66,11 @@ func (e Extractor) Version() int { return 0 } // Requirements of the extractor. func (e Extractor) Requirements() *plugin.Capabilities { return &plugin.Capabilities{ - Network: true, - DirectFS: true, + Network: true, } } -// FileRequired never returns true, as this is for the osv-scanner json output. +// FileRequired returns true if the specified file matches Maven POM lockfile patterns. func (e Extractor) FileRequired(fapi filesystem.FileAPI) bool { return filepath.Base(fapi.Path()) == "pom.xml" } diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index 8abbf9e3..34efbfdb 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -42,6 +42,7 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradlelockfile" "github.com/google/osv-scalibr/extractor/filesystem/language/java/gradleverificationmetadataxml" "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxmlnet" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagejson" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" @@ -93,6 +94,13 @@ var ( javaarchive.New(javaarchive.DefaultConfig()), pomxml.Extractor{}, } + // JavaNet extractors requiring network access. + JavaNet []filesystem.Extractor = []filesystem.Extractor{ + gradlelockfile.Extractor{}, + gradleverificationmetadataxml.Extractor{}, + javaarchive.New(javaarchive.DefaultConfig()), + pomxmlnet.New(), + } // Javascript extractors. Javascript []filesystem.Extractor = []filesystem.Extractor{ packagejson.New(packagejson.DefaultConfig()), @@ -172,6 +180,9 @@ var ( // Default extractors that are recommended to be enabled. Default []filesystem.Extractor = slices.Concat(Java, Javascript, Python, Go, OS) + // DefaultNet defines the list of recommended extractors that require network access. + DefaultNet []filesystem.Extractor = slices.Concat(JavaNet, Javascript, Python, Go, OS) + // All extractors available from SCALIBR. All []filesystem.Extractor = slices.Concat( Cpp, @@ -217,8 +228,9 @@ var ( "containers": Containers, // Collections. - "default": Default, - "all": All, + "default": Default, + "defaultnet": DefaultNet, + "all": All, } ) diff --git a/internal/resolution/client/client.go b/internal/resolution/client/client.go index 227b0580..c723ddd9 100644 --- a/internal/resolution/client/client.go +++ b/internal/resolution/client/client.go @@ -22,10 +22,6 @@ import ( // DependencyClient is the interface of the client required by dependency resolution. type DependencyClient interface { resolve.Client - // WriteCache writes a manifest-specific resolution cache. - WriteCache(filepath string) error - // LoadCache loads a manifest-specific resolution cache. - LoadCache(filepath string) error // AddRegistries adds the specified registries to fetch data. AddRegistries(registries []Registry) error } From 1dbc456bd5340067000da93fc1424c4169b488d5 Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Thu, 6 Feb 2025 14:54:13 +1100 Subject: [PATCH 2/7] comment --- extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go index a5d77748..b6f08dc3 100644 --- a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go +++ b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go @@ -75,7 +75,7 @@ func (e Extractor) FileRequired(fapi filesystem.FileAPI) bool { return filepath.Base(fapi.Path()) == "pom.xml" } -// Extract extracts packages from yarn.lock files passed through the scan input. +// Extract extracts packages from pom.xml files passed through the scan input. func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { var project maven.Project if err := datasource.NewMavenDecoder(input.Reader).Decode(&project); err != nil { From 4bde8dbc4b07b0985a385dfc8572921992850b64 Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Thu, 6 Feb 2025 16:47:03 +1100 Subject: [PATCH 3/7] config --- .../language/java/pomxmlnet/pomxmlnet.go | 35 +++++++++++++------ extractor/filesystem/list/list.go | 2 +- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go index b6f08dc3..35e9db61 100644 --- a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go +++ b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go @@ -38,25 +38,38 @@ import ( // Extractor extracts Maven packages with transitive dependency resolution. type Extractor struct { + depClient client.DependencyClient + mavenClient *datasource.MavenRegistryAPIClient +} + +// Config is the configuration for the pomxmlnet Extractor. +type Config struct { client.DependencyClient *datasource.MavenRegistryAPIClient } -// New makes a new pom.xml transitive extractor with required clients. -// Clients are assuming Maven Central as the registry. -func New() *Extractor { +// DefaultConfig returns the default configuration for the pomxmlnet extractor. +func DefaultConfig() Config { // No need to check errors since we are using the default Maven Central URL. depClient, _ := client.NewMavenRegistryClient(datasource.MavenCentral) mavenClient, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{ URL: datasource.MavenCentral, ReleasesEnabled: true, }) - return &Extractor{ + return Config{ DependencyClient: depClient, MavenRegistryAPIClient: mavenClient, } } +// New makes a new pom.xml transitive extractor with the given config. +func New(c Config) *Extractor { + return &Extractor{ + depClient: c.DependencyClient, + mavenClient: c.MavenRegistryAPIClient, + } +} + // Name of the extractor. func (e Extractor) Name() string { return "java/pomxmlnet" } @@ -85,8 +98,10 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] if err := project.MergeProfiles("", maven.ActivationOS{}); err != nil { return nil, fmt.Errorf("failed to merge profiles: %w", err) } + // Clear the registries that may be from other extraction. + e.mavenClient = e.mavenClient.WithoutRegistries() for _, repo := range project.Repositories { - if err := e.MavenRegistryAPIClient.AddRegistry(datasource.MavenRegistry{ + if err := e.mavenClient.AddRegistry(datasource.MavenRegistry{ URL: string(repo.URL), ID: string(repo.ID), ReleasesEnabled: repo.Releases.Enabled.Boolean(), @@ -96,7 +111,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] } } // Merging parents data by parsing local parent pom.xml or fetching from upstream. - if err := mavenutil.MergeParents(ctx, input, e.MavenRegistryAPIClient, &project, project.Parent, 1, true); err != nil { + if err := mavenutil.MergeParents(ctx, input, e.mavenClient, &project, project.Parent, 1, true); err != nil { return nil, fmt.Errorf("failed to merge parents: %w", err) } // Process the dependencies: @@ -104,20 +119,20 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] // - import dependency management // - fill in missing dependency version requirement project.ProcessDependencies(func(groupID, artifactID, version maven.String) (maven.DependencyManagement, error) { - return mavenutil.GetDependencyManagement(ctx, e.MavenRegistryAPIClient, groupID, artifactID, version) + return mavenutil.GetDependencyManagement(ctx, e.mavenClient, groupID, artifactID, version) }) - if registries := e.MavenRegistryAPIClient.GetRegistries(); len(registries) > 0 { + if registries := e.mavenClient.GetRegistries(); len(registries) > 0 { clientRegs := make([]client.Registry, len(registries)) for i, reg := range registries { clientRegs[i] = reg } - if err := e.DependencyClient.AddRegistries(clientRegs); err != nil { + if err := e.depClient.AddRegistries(clientRegs); err != nil { return nil, err } } - overrideClient := client.NewOverrideClient(e.DependencyClient) + overrideClient := client.NewOverrideClient(e.depClient) resolver := mavenresolve.NewResolver(overrideClient) // Resolve the dependencies. diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index 34efbfdb..c7fff242 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -99,7 +99,7 @@ var ( gradlelockfile.Extractor{}, gradleverificationmetadataxml.Extractor{}, javaarchive.New(javaarchive.DefaultConfig()), - pomxmlnet.New(), + pomxmlnet.New(pomxmlnet.DefaultConfig()), } // Javascript extractors. Javascript []filesystem.Extractor = []filesystem.Extractor{ From f53f1b6f0d67373538c27c4ad106bd7fba114a08 Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Thu, 6 Feb 2025 16:50:13 +1100 Subject: [PATCH 4/7] fix test --- .../filesystem/language/java/pomxmlnet/pomxmlnet_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go index 8e580058..61d466ed 100644 --- a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go +++ b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go @@ -242,10 +242,10 @@ func TestExtractor_Extract(t *testing.T) { for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") - extr := pomxmlnet.Extractor{ + extr := pomxmlnet.New(pomxmlnet.Config{ DependencyClient: resolutionClient, MavenRegistryAPIClient: &datasource.MavenRegistryAPIClient{}, - } + }) scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) defer extracttest.CloseTestScanInput(t, scanInput) @@ -350,10 +350,10 @@ func TestExtractor_Extract_WithMockServer(t *testing.T) { } resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") - extr := pomxmlnet.Extractor{ + extr := pomxmlnet.New(pomxmlnet.Config{ DependencyClient: resolutionClient, MavenRegistryAPIClient: apiClient, - } + }) scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) defer extracttest.CloseTestScanInput(t, scanInput) From 335667ea1bdcd1545d3278c6ff537fdb577ee3e3 Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Fri, 7 Feb 2025 14:10:10 +1100 Subject: [PATCH 5/7] public clients --- .../resolution => clients}/clienttest/mock_http.go | 0 .../clienttest/mock_resolution_client.go | 8 +++----- {internal => clients}/datasource/cache.go | 0 {internal => clients}/datasource/cache_test.go | 2 +- {internal => clients}/datasource/http_auth.go | 0 {internal => clients}/datasource/http_auth_test.go | 2 +- {internal => clients}/datasource/insights.go | 0 {internal => clients}/datasource/insights_cache.go | 0 {internal => clients}/datasource/maven_registry.go | 0 .../datasource/maven_registry_cache.go | 0 .../datasource/maven_registry_test.go | 4 ++-- {internal => clients}/datasource/maven_settings.go | 0 .../datasource/maven_settings_test.go | 2 +- .../testdata/maven_settings/settings.xml | 0 .../client => clients/resolution}/client.go | 4 ++-- .../resolution}/depsdev_client.go | 4 ++-- .../resolution}/maven_registry_client.go | 4 ++-- .../resolution}/override_client.go | 2 +- .../language/java/pomxmlnet/pomxmlnet.go | 14 +++++++------- .../language/java/pomxmlnet/pomxmlnet_test.go | 4 ++-- internal/mavenutil/maven.go | 2 +- 21 files changed, 25 insertions(+), 27 deletions(-) rename {internal/resolution => clients}/clienttest/mock_http.go (100%) rename {internal/resolution => clients}/clienttest/mock_resolution_client.go (84%) rename {internal => clients}/datasource/cache.go (100%) rename {internal => clients}/datasource/cache_test.go (97%) rename {internal => clients}/datasource/http_auth.go (100%) rename {internal => clients}/datasource/http_auth_test.go (99%) rename {internal => clients}/datasource/insights.go (100%) rename {internal => clients}/datasource/insights_cache.go (100%) rename {internal => clients}/datasource/maven_registry.go (100%) rename {internal => clients}/datasource/maven_registry_cache.go (100%) rename {internal => clients}/datasource/maven_registry_test.go (98%) rename {internal => clients}/datasource/maven_settings.go (100%) rename {internal => clients}/datasource/maven_settings_test.go (98%) rename {internal => clients}/datasource/testdata/maven_settings/settings.xml (100%) rename {internal/resolution/client => clients/resolution}/client.go (91%) rename {internal/resolution/client => clients/resolution}/depsdev_client.go (96%) rename {internal/resolution/client => clients/resolution}/maven_registry_client.go (98%) rename {internal/resolution/client => clients/resolution}/override_client.go (99%) diff --git a/internal/resolution/clienttest/mock_http.go b/clients/clienttest/mock_http.go similarity index 100% rename from internal/resolution/clienttest/mock_http.go rename to clients/clienttest/mock_http.go diff --git a/internal/resolution/clienttest/mock_resolution_client.go b/clients/clienttest/mock_resolution_client.go similarity index 84% rename from internal/resolution/clienttest/mock_resolution_client.go rename to clients/clienttest/mock_resolution_client.go index 55aafb46..037ddaba 100644 --- a/internal/resolution/clienttest/mock_resolution_client.go +++ b/clients/clienttest/mock_resolution_client.go @@ -21,7 +21,7 @@ import ( "deps.dev/util/resolve" "deps.dev/util/resolve/schema" - "github.com/google/osv-scalibr/internal/resolution/client" + "github.com/google/osv-scalibr/clients/resolution" "gopkg.in/yaml.v3" ) @@ -35,12 +35,10 @@ type mockDependencyClient struct { *resolve.LocalClient } -func (mdc mockDependencyClient) LoadCache(string) error { return nil } -func (mdc mockDependencyClient) WriteCache(string) error { return nil } -func (mdc mockDependencyClient) AddRegistries(_ []client.Registry) error { return nil } +func (mdc mockDependencyClient) AddRegistries(_ []resolution.Registry) error { return nil } // NewMockResolutionClient creates a new mock resolution client from the given universe YAML. -func NewMockResolutionClient(t *testing.T, universeYAML string) client.DependencyClient { +func NewMockResolutionClient(t *testing.T, universeYAML string) resolution.DependencyClient { t.Helper() f, err := os.Open(universeYAML) if err != nil { diff --git a/internal/datasource/cache.go b/clients/datasource/cache.go similarity index 100% rename from internal/datasource/cache.go rename to clients/datasource/cache.go diff --git a/internal/datasource/cache_test.go b/clients/datasource/cache_test.go similarity index 97% rename from internal/datasource/cache_test.go rename to clients/datasource/cache_test.go index fbec8638..2c6074b3 100644 --- a/internal/datasource/cache_test.go +++ b/clients/datasource/cache_test.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "testing" - "github.com/google/osv-scalibr/internal/datasource" + "github.com/google/osv-scalibr/clients/datasource" ) func TestRequestCache(t *testing.T) { diff --git a/internal/datasource/http_auth.go b/clients/datasource/http_auth.go similarity index 100% rename from internal/datasource/http_auth.go rename to clients/datasource/http_auth.go diff --git a/internal/datasource/http_auth_test.go b/clients/datasource/http_auth_test.go similarity index 99% rename from internal/datasource/http_auth_test.go rename to clients/datasource/http_auth_test.go index e7416420..8665f9f1 100644 --- a/internal/datasource/http_auth_test.go +++ b/clients/datasource/http_auth_test.go @@ -19,7 +19,7 @@ import ( "net/http" "testing" - "github.com/google/osv-scalibr/internal/datasource" + "github.com/google/osv-scalibr/clients/datasource" ) // mockTransport is used to inspect the requests being made by HTTPAuthentications diff --git a/internal/datasource/insights.go b/clients/datasource/insights.go similarity index 100% rename from internal/datasource/insights.go rename to clients/datasource/insights.go diff --git a/internal/datasource/insights_cache.go b/clients/datasource/insights_cache.go similarity index 100% rename from internal/datasource/insights_cache.go rename to clients/datasource/insights_cache.go diff --git a/internal/datasource/maven_registry.go b/clients/datasource/maven_registry.go similarity index 100% rename from internal/datasource/maven_registry.go rename to clients/datasource/maven_registry.go diff --git a/internal/datasource/maven_registry_cache.go b/clients/datasource/maven_registry_cache.go similarity index 100% rename from internal/datasource/maven_registry_cache.go rename to clients/datasource/maven_registry_cache.go diff --git a/internal/datasource/maven_registry_test.go b/clients/datasource/maven_registry_test.go similarity index 98% rename from internal/datasource/maven_registry_test.go rename to clients/datasource/maven_registry_test.go index de03cd4a..6ff665cd 100644 --- a/internal/datasource/maven_registry_test.go +++ b/clients/datasource/maven_registry_test.go @@ -20,8 +20,8 @@ import ( "testing" "deps.dev/util/maven" - "github.com/google/osv-scalibr/internal/datasource" - "github.com/google/osv-scalibr/internal/resolution/clienttest" + "github.com/google/osv-scalibr/clients/clienttest" + "github.com/google/osv-scalibr/clients/datasource" ) func TestGetProject(t *testing.T) { diff --git a/internal/datasource/maven_settings.go b/clients/datasource/maven_settings.go similarity index 100% rename from internal/datasource/maven_settings.go rename to clients/datasource/maven_settings.go diff --git a/internal/datasource/maven_settings_test.go b/clients/datasource/maven_settings_test.go similarity index 98% rename from internal/datasource/maven_settings_test.go rename to clients/datasource/maven_settings_test.go index debd45f0..47e1fdbc 100644 --- a/internal/datasource/maven_settings_test.go +++ b/clients/datasource/maven_settings_test.go @@ -19,7 +19,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/osv-scalibr/internal/datasource" + "github.com/google/osv-scalibr/clients/datasource" ) func TestParseMavenSettings(t *testing.T) { diff --git a/internal/datasource/testdata/maven_settings/settings.xml b/clients/datasource/testdata/maven_settings/settings.xml similarity index 100% rename from internal/datasource/testdata/maven_settings/settings.xml rename to clients/datasource/testdata/maven_settings/settings.xml diff --git a/internal/resolution/client/client.go b/clients/resolution/client.go similarity index 91% rename from internal/resolution/client/client.go rename to clients/resolution/client.go index c723ddd9..f19fc3ba 100644 --- a/internal/resolution/client/client.go +++ b/clients/resolution/client.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package client provides clients required by dependency resolution. -package client +// Package resolution provides clients required by dependency resolution. +package resolution import ( "deps.dev/util/resolve" diff --git a/internal/resolution/client/depsdev_client.go b/clients/resolution/depsdev_client.go similarity index 96% rename from internal/resolution/client/depsdev_client.go rename to clients/resolution/depsdev_client.go index 04dd8563..163f0cbd 100644 --- a/internal/resolution/client/depsdev_client.go +++ b/clients/resolution/depsdev_client.go @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package client +package resolution import ( "encoding/gob" "os" "deps.dev/util/resolve" - "github.com/google/osv-scalibr/internal/datasource" + "github.com/google/osv-scalibr/clients/datasource" ) const depsDevCacheExt = ".resolve.deps" diff --git a/internal/resolution/client/maven_registry_client.go b/clients/resolution/maven_registry_client.go similarity index 98% rename from internal/resolution/client/maven_registry_client.go rename to clients/resolution/maven_registry_client.go index 8a7c6943..8f432132 100644 --- a/internal/resolution/client/maven_registry_client.go +++ b/clients/resolution/maven_registry_client.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package client +package resolution import ( "context" @@ -23,7 +23,7 @@ import ( "deps.dev/util/maven" "deps.dev/util/resolve" "deps.dev/util/resolve/version" - "github.com/google/osv-scalibr/internal/datasource" + "github.com/google/osv-scalibr/clients/datasource" "github.com/google/osv-scalibr/internal/mavenutil" ) diff --git a/internal/resolution/client/override_client.go b/clients/resolution/override_client.go similarity index 99% rename from internal/resolution/client/override_client.go rename to clients/resolution/override_client.go index 0dd11fc6..43fa33dc 100644 --- a/internal/resolution/client/override_client.go +++ b/clients/resolution/override_client.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package client +package resolution import ( "context" diff --git a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go index 35e9db61..bb56513f 100644 --- a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go +++ b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet.go @@ -26,32 +26,32 @@ import ( "deps.dev/util/maven" "deps.dev/util/resolve" mavenresolve "deps.dev/util/resolve/maven" + "github.com/google/osv-scalibr/clients/datasource" + "github.com/google/osv-scalibr/clients/resolution" "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem" "github.com/google/osv-scalibr/extractor/filesystem/osv" - "github.com/google/osv-scalibr/internal/datasource" "github.com/google/osv-scalibr/internal/mavenutil" - "github.com/google/osv-scalibr/internal/resolution/client" "github.com/google/osv-scalibr/plugin" "github.com/google/osv-scalibr/purl" ) // Extractor extracts Maven packages with transitive dependency resolution. type Extractor struct { - depClient client.DependencyClient + depClient resolution.DependencyClient mavenClient *datasource.MavenRegistryAPIClient } // Config is the configuration for the pomxmlnet Extractor. type Config struct { - client.DependencyClient + resolution.DependencyClient *datasource.MavenRegistryAPIClient } // DefaultConfig returns the default configuration for the pomxmlnet extractor. func DefaultConfig() Config { // No need to check errors since we are using the default Maven Central URL. - depClient, _ := client.NewMavenRegistryClient(datasource.MavenCentral) + depClient, _ := resolution.NewMavenRegistryClient(datasource.MavenCentral) mavenClient, _ := datasource.NewMavenRegistryAPIClient(datasource.MavenRegistry{ URL: datasource.MavenCentral, ReleasesEnabled: true, @@ -123,7 +123,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] }) if registries := e.mavenClient.GetRegistries(); len(registries) > 0 { - clientRegs := make([]client.Registry, len(registries)) + clientRegs := make([]resolution.Registry, len(registries)) for i, reg := range registries { clientRegs[i] = reg } @@ -132,7 +132,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] } } - overrideClient := client.NewOverrideClient(e.depClient) + overrideClient := resolution.NewOverrideClient(e.depClient) resolver := mavenresolve.NewResolver(overrideClient) // Resolve the dependencies. diff --git a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go index 61d466ed..d74a88e1 100644 --- a/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go +++ b/extractor/filesystem/language/java/pomxmlnet/pomxmlnet_test.go @@ -20,12 +20,12 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/clients/clienttest" + "github.com/google/osv-scalibr/clients/datasource" "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxmlnet" "github.com/google/osv-scalibr/extractor/filesystem/osv" "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" - "github.com/google/osv-scalibr/internal/datasource" - "github.com/google/osv-scalibr/internal/resolution/clienttest" "github.com/google/osv-scalibr/testing/extracttest" ) diff --git a/internal/mavenutil/maven.go b/internal/mavenutil/maven.go index c2b08730..e1a3c6cb 100644 --- a/internal/mavenutil/maven.go +++ b/internal/mavenutil/maven.go @@ -22,8 +22,8 @@ import ( "path/filepath" "deps.dev/util/maven" + "github.com/google/osv-scalibr/clients/datasource" "github.com/google/osv-scalibr/extractor/filesystem" - "github.com/google/osv-scalibr/internal/datasource" ) // Origin of the dependencies. From f25978f11a96beec33ab20b9d371f71f9ef052d8 Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Fri, 7 Feb 2025 14:16:35 +1100 Subject: [PATCH 6/7] lint --- clients/resolution/depsdev_client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/clients/resolution/depsdev_client.go b/clients/resolution/depsdev_client.go index 162dcf00..c91d77cc 100644 --- a/clients/resolution/depsdev_client.go +++ b/clients/resolution/depsdev_client.go @@ -15,9 +15,6 @@ package resolution import ( - "encoding/gob" - "os" - "deps.dev/util/resolve" "github.com/google/osv-scalibr/clients/datasource" ) From 017aa6c8b111695f1d8d7125b42509d21f90c60c Mon Sep 17 00:00:00 2001 From: Xueqin Cui Date: Fri, 7 Feb 2025 15:40:17 +1100 Subject: [PATCH 7/7] TODO --- extractor/filesystem/list/list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index 160d0009..1473bd63 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -95,6 +95,7 @@ var ( javaarchive.New(javaarchive.DefaultConfig()), pomxml.Extractor{}, } + // TODO(#441): enable pomxmlnet extractor when network is accesible. // JavaNet extractors requiring network access. JavaNet []filesystem.Extractor = []filesystem.Extractor{ gradlelockfile.Extractor{},