From b48cf34b8e3db937115baa67e5b79b873a6a76df Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Wed, 31 Jan 2024 12:29:54 +0100 Subject: [PATCH] add presave processor to deduplicate profile info Signed-off-by: Matthias Bertschy --- pkg/apis/softwarecomposition/types.go | 27 +++++++++++ pkg/apiserver/apiserver.go | 3 +- pkg/registry/file/processor.go | 64 +++++++++++++++++++++++++++ pkg/registry/file/storage.go | 19 +++++++- 4 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 pkg/registry/file/processor.go diff --git a/pkg/apis/softwarecomposition/types.go b/pkg/apis/softwarecomposition/types.go index d0036b535..b49b0c054 100644 --- a/pkg/apis/softwarecomposition/types.go +++ b/pkg/apis/softwarecomposition/types.go @@ -18,6 +18,7 @@ package softwarecomposition import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -314,11 +315,37 @@ type ExecCalls struct { Envs []string } +const sep = "␟" + +func (e ExecCalls) String() string { + s := strings.Builder{} + s.WriteString(e.Path) + for _, arg := range e.Args { + s.WriteString(sep) + s.WriteString(arg) + } + for _, env := range e.Envs { + s.WriteString(sep) + s.WriteString(env) + } + return s.String() +} + type OpenCalls struct { Path string Flags []string } +func (e OpenCalls) String() string { + s := strings.Builder{} + s.WriteString(e.Path) + for _, arg := range e.Flags { + s.WriteString(sep) + s.WriteString(arg) + } + return s.String() +} + type ApplicationProfileStatus struct { } diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 780fac01d..bdb2d984c 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -149,6 +149,7 @@ func (c completedConfig) New() (*WardleServer, error) { storageImpl := file.NewStorageImpl(osFs, file.DefaultStorageRoot) + applicationProfileStorageImpl := file.NewStorageImplWithCollector(osFs, file.DefaultStorageRoot, &file.ApplicationProfileProcessor{}) configScanStorageImpl := file.NewConfigurationScanSummaryStorage(&storageImpl) vulnerabilitySummaryStorage := file.NewVulnerabilitySummaryStorage(&storageImpl) generatedNetworkPolicyStorage := file.NewGeneratedNetworkPolicyStorage(&storageImpl) @@ -168,7 +169,7 @@ func (c completedConfig) New() (*WardleServer, error) { v1beta1storage["configurationscansummaries"] = sbomregistry.RESTInPeace(configurationscansummary.NewREST(Scheme, configScanStorageImpl, c.GenericConfig.RESTOptionsGetter)) v1beta1storage["vulnerabilitysummaries"] = sbomregistry.RESTInPeace(vsumstorage.NewREST(Scheme, vulnerabilitySummaryStorage, c.GenericConfig.RESTOptionsGetter)) - v1beta1storage["applicationprofiles"] = sbomregistry.RESTInPeace(applicationprofile.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter)) + v1beta1storage["applicationprofiles"] = sbomregistry.RESTInPeace(applicationprofile.NewREST(Scheme, applicationProfileStorageImpl, c.GenericConfig.RESTOptionsGetter)) v1beta1storage["applicationprofilesummaries"] = sbomregistry.RESTInPeace(applicationprofilesummary.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter)) v1beta1storage["applicationactivities"] = sbomregistry.RESTInPeace(applicationactivity.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter)) diff --git a/pkg/registry/file/processor.go b/pkg/registry/file/processor.go new file mode 100644 index 000000000..2961100ab --- /dev/null +++ b/pkg/registry/file/processor.go @@ -0,0 +1,64 @@ +package file + +import ( + "fmt" + sets "github.com/deckarep/golang-set/v2" + "github.com/kubescape/storage/pkg/apis/softwarecomposition" + "k8s.io/apimachinery/pkg/runtime" +) + +type Processor interface { + PreSave(object runtime.Object) error +} + +type DefaultProcessor struct { +} + +var _ Processor = (*DefaultProcessor)(nil) + +func (d DefaultProcessor) PreSave(_ runtime.Object) error { + return nil +} + +type ApplicationProfileProcessor struct { +} + +var _ Processor = (*ApplicationProfileProcessor)(nil) + +func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error { + profile, ok := object.(*softwarecomposition.ApplicationProfile) + if !ok { + return fmt.Errorf("given object is not an ApplicationProfile") + } + for i, container := range profile.Spec.Containers { + profile.Spec.Containers[i] = deflate(container) + } + return nil +} + +func deflate(container softwarecomposition.ApplicationProfileContainer) softwarecomposition.ApplicationProfileContainer { + return softwarecomposition.ApplicationProfileContainer{ + Name: container.Name, + Capabilities: sets.NewThreadUnsafeSet(container.Capabilities...).ToSlice(), + Execs: deflateStringer(container.Execs), + Opens: deflateStringer(container.Opens), + Syscalls: sets.NewThreadUnsafeSet(container.Syscalls...).ToSlice(), + } +} + +type Stringer interface { + String() string +} + +func deflateStringer[T Stringer](in []T) []T { + var out []T + set := sets.NewSet[string]() + for _, item := range in { + if set.Contains(item.String()) { + continue + } + set.Add(item.String()) + out = append(out, item) + } + return out +} diff --git a/pkg/registry/file/storage.go b/pkg/registry/file/storage.go index 8c71c406e..4e47dce49 100644 --- a/pkg/registry/file/storage.go +++ b/pkg/registry/file/storage.go @@ -41,10 +41,11 @@ type objState struct { // hides all the storage-related operations behind it. type StorageImpl struct { appFs afero.Fs - watchDispatcher watchDispatcher locks *Mutex[string] + processor Processor root string versioner storage.Versioner + watchDispatcher watchDispatcher } // StorageQuerier wraps the storage.Interface and adds some extra methods which are used by the storage implementation. @@ -62,10 +63,22 @@ var _ StorageQuerier = &StorageImpl{} func NewStorageImpl(appFs afero.Fs, root string) StorageQuerier { return &StorageImpl{ appFs: appFs, + locks: NewMapMutex[string](), + processor: DefaultProcessor{}, + root: root, + versioner: storage.APIObjectVersioner{}, watchDispatcher: newWatchDispatcher(), + } +} + +func NewStorageImplWithCollector(appFs afero.Fs, root string, processor Processor) StorageQuerier { + return &StorageImpl{ + appFs: appFs, locks: NewMapMutex[string](), + processor: processor, root: root, versioner: storage.APIObjectVersioner{}, + watchDispatcher: newWatchDispatcher(), } } @@ -106,6 +119,10 @@ func isPayloadFile(path string) bool { } func (s *StorageImpl) writeFiles(key string, obj runtime.Object, metaOut runtime.Object) error { + // call processor on object to be saved + if err := s.processor.PreSave(obj); err != nil { + return fmt.Errorf("processor.PreSave: %w", err) + } // set resourceversion if version, _ := s.versioner.ObjectResourceVersion(obj); version == 0 { if err := s.versioner.UpdateObject(obj, 1); err != nil {