From ea8fd0b16ec55af4c4030901ccc5b579934cc7d5 Mon Sep 17 00:00:00 2001 From: Marius Goetze Date: Fri, 19 Apr 2024 14:04:12 +0200 Subject: [PATCH 1/3] fix: don't drop error messages from loading provider-metadata.json previously in case case of trying last resort dns, all other error messages were dropped --- csaf/providermetaloader.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 203f2b35..0c4fc3bd 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -173,6 +173,8 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata // We have a candidate. if wellknownResult.Valid() { wellknownGood = wellknownResult + } else { + pmdl.messages.AppendUnique(wellknownResult.Messages) } // Next load the PMDs from security.txt @@ -220,25 +222,28 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata } } // Take the good well-known. - wellknownGood.Messages.AppendUnique(pmdl.messages) + wellknownGood.Messages = pmdl.messages return wellknownGood } // Don't have well-known. Take first good from security.txt. ignoreExtras() - secGoods[0].Messages.AppendUnique(pmdl.messages) + secGoods[0].Messages = pmdl.messages return secGoods[0] } // If we have a good well-known take it. if wellknownGood != nil { - wellknownGood.Messages.AppendUnique(pmdl.messages) + wellknownGood.Messages = pmdl.messages return wellknownGood } // Last resort: fall back to DNS. dnsURL := "https://csaf.data.security." + domain - return pmdl.loadFromURL(dnsURL) + dnsURLResult := pmdl.loadFromURL(dnsURL) + pmdl.messages.AppendUnique(dnsURLResult.Messages) // keep order of messages consistent (i.e. last occurred message is last element) + dnsURLResult.Messages = pmdl.messages + return dnsURLResult } // loadFromSecurity loads the PMDs mentioned in the security.txt. From a88f326a2b44f6e9e656c517f4674e7db86dd205 Mon Sep 17 00:00:00 2001 From: Marius Goetze Date: Fri, 19 Apr 2024 14:06:56 +0200 Subject: [PATCH 2/3] refactor: deduplicate filtering pmd results from security.txt already done in `loadFromSecurity` --- csaf/providermetaloader.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 0c4fc3bd..b21ddc61 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -178,20 +178,7 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata } // Next load the PMDs from security.txt - secResults := pmdl.loadFromSecurity(domain) - - // Filter out the results which are valid. - var secGoods []*LoadedProviderMetadata - - for _, result := range secResults { - if len(result.Messages) > 0 { - // If there where validation issues append them - // to the overall report - pmdl.messages.AppendUnique(pmdl.messages) - } else { - secGoods = append(secGoods, result) - } - } + secGoods := pmdl.loadFromSecurity(domain) // Mention extra CSAF entries in security.txt. ignoreExtras := func() { @@ -246,7 +233,7 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata return dnsURLResult } -// loadFromSecurity loads the PMDs mentioned in the security.txt. +// loadFromSecurity loads the PMDs mentioned in the security.txt. Only valid PMDs are returned. func (pmdl *ProviderMetadataLoader) loadFromSecurity(domain string) []*LoadedProviderMetadata { // If .well-known fails try legacy location. From f004c47e3c46c74aea94b8e8099d793b28c54981 Mon Sep 17 00:00:00 2001 From: Marius Goetze Date: Mon, 15 Jul 2024 10:52:13 +0200 Subject: [PATCH 3/3] fix: don't require debug level to print error details on failed loading of provider metadata json --- cmd/csaf_aggregator/processor.go | 14 +++++++++----- cmd/csaf_downloader/downloader.go | 13 ++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go index 9f10a774..5cb36283 100644 --- a/cmd/csaf_aggregator/processor.go +++ b/cmd/csaf_aggregator/processor.go @@ -89,17 +89,21 @@ func (w *worker) locateProviderMetadata(domain string) error { lpmd := loader.Load(domain) - if w.processor.cfg.Verbose { + if !lpmd.Valid() { for i := range lpmd.Messages { - w.log.Info( + w.log.Error( "Loading provider-metadata.json", "domain", domain, "message", lpmd.Messages[i].Message) } - } - - if !lpmd.Valid() { return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) + } else if w.processor.cfg.Verbose { + for i := range lpmd.Messages { + w.log.Debug( + "Loading provider-metadata.json", + "domain", domain, + "message", lpmd.Messages[i].Message) + } } w.metadataProvider = lpmd.Document diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index a0cf34e5..e370f55a 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -199,7 +199,14 @@ func (d *downloader) download(ctx context.Context, domain string) error { lpmd := loader.Load(domain) - if d.cfg.verbose() { + if !lpmd.Valid() { + for i := range lpmd.Messages { + slog.Error("Loading provider-metadata.json", + "domain", domain, + "message", lpmd.Messages[i].Message) + } + return fmt.Errorf("no valid provider-metadata.json found for '%s': ", domain) + } else if d.cfg.verbose() { for i := range lpmd.Messages { slog.Debug("Loading provider-metadata.json", "domain", domain, @@ -207,10 +214,6 @@ func (d *downloader) download(ctx context.Context, domain string) error { } } - if !lpmd.Valid() { - return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) - } - base, err := url.Parse(lpmd.URL) if err != nil { return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err)