From ea568dba5861de8b049d5b0771639c6412c1e0c7 Mon Sep 17 00:00:00 2001 From: koplas <54645365+koplas@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:29:46 +0200 Subject: [PATCH] Improve download handler interface --- cmd/csaf_downloader/main.go | 56 +++++++++++++++++++++++++++++++----- lib/downloader/downloader.go | 51 +++++++++++--------------------- 2 files changed, 66 insertions(+), 41 deletions(-) diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index 8011f396..36f4c065 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -11,6 +11,7 @@ package main import ( "context" + "encoding/json" "log/slog" "os" "os/signal" @@ -19,9 +20,12 @@ import ( "strconv" "strings" "sync" + "time" + "github.com/csaf-poc/csaf_distribution/v3/csaf" "github.com/csaf-poc/csaf_distribution/v3/internal/options" "github.com/csaf-poc/csaf_distribution/v3/lib/downloader" + "github.com/csaf-poc/csaf_distribution/v3/util" ) // failedForwardDir is the name of the special sub folder @@ -79,6 +83,35 @@ func mkdirAll(path string, perm os.FileMode) error { return os.MkdirAll(path, perm) } +func extractInitialReleaseDate(doc any) time.Time { + var initialReleaseDate time.Time + dateExtract := util.TimeMatcher(&initialReleaseDate, time.RFC3339) + + eval := util.NewPathEval() + + if err := eval.Extract( + `$.document.tracking.initial_release_date`, dateExtract, false, doc, + ); err != nil { + slog.Warn("Cannot extract initial_release_date from advisory") + initialReleaseDate = time.Now() + } + initialReleaseDate = initialReleaseDate.UTC() + return initialReleaseDate +} + +func extractTLP(doc any) csaf.TLPLabel { + eval := util.NewPathEval() + labelString, err := eval.Eval(`$.document.distribution.tlp.label`, doc) + if err != nil { + return csaf.TLPLabelUnlabeled + } + label, ok := labelString.(string) + if !ok { + return csaf.TLPLabelUnlabeled + } + return csaf.TLPLabel(label) +} + func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error { return func(d downloader.DownloadedDocument) error { if cfg.NoStore { @@ -98,13 +131,22 @@ func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error { newDir = cfg.Directory } - lower := strings.ToLower(string(d.Label)) + var doc any + if err := json.Unmarshal(d.Data, &doc); err != nil { + slog.Error("Could not parse json document", "err", err) + return nil + } + + initialReleaseDate := extractInitialReleaseDate(doc) + label := extractTLP(doc) + + lower := strings.ToLower(string(label)) // Do we have a configured destination folder? if cfg.Folder != "" { newDir = path.Join(newDir, cfg.Folder) } else { - newDir = path.Join(newDir, lower, strconv.Itoa(d.InitialReleaseDate.Year())) + newDir = path.Join(newDir, lower, strconv.Itoa(initialReleaseDate.Year())) } if newDir != lastDir { @@ -121,9 +163,9 @@ func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error { p string d []byte }{ - {filePath, d.Data.Bytes()}, - {filePath + ".sha256", d.S256Data}, - {filePath + ".sha512", d.S512Data}, + {filePath, d.Data}, + {filePath + ".sha256", d.SHA256}, + {filePath + ".sha512", d.SHA512}, {filePath + ".asc", d.SignData}, } { if x.d != nil { @@ -157,8 +199,8 @@ func storeFailedAdvisory(cfg *config) func(filename, doc, sha256, sha512 string) {filename + ".sha512", sha512}, } { if len(x.d) != 0 { - path := filepath.Join(dir, x.p) - if err := os.WriteFile(path, []byte(x.d), 0644); err != nil { + p := filepath.Join(dir, x.p) + if err := os.WriteFile(p, []byte(x.d), 0644); err != nil { return err } } diff --git a/lib/downloader/downloader.go b/lib/downloader/downloader.go index a43118a3..1960d372 100644 --- a/lib/downloader/downloader.go +++ b/lib/downloader/downloader.go @@ -25,7 +25,6 @@ import ( "path/filepath" "strings" "sync" - "time" "github.com/ProtonMail/gopenpgp/v2/crypto" "golang.org/x/time/rate" @@ -48,14 +47,12 @@ type Downloader struct { // DownloadedDocument conatins the document data with additional metadata. type DownloadedDocument struct { - Data bytes.Buffer - S256Data []byte - S512Data []byte - SignData []byte - InitialReleaseDate time.Time - Filename string - ValStatus ValidationStatus - Label csaf.TLPLabel + Data []byte + SHA256 []byte + SHA512 []byte + SignData []byte + Filename string + ValStatus ValidationStatus } // failedValidationDir is the name of the sub folder @@ -252,13 +249,12 @@ func (d *Downloader) download(ctx context.Context, domain string) error { } return afp.Process(func(label csaf.TLPLabel, files []csaf.AdvisoryFile) error { - return d.downloadFiles(ctx, label, files) + return d.downloadFiles(ctx, files) }) } func (d *Downloader) downloadFiles( ctx context.Context, - label csaf.TLPLabel, files []csaf.AdvisoryFile, ) error { var ( @@ -284,7 +280,7 @@ func (d *Downloader) downloadFiles( for i := 0; i < n; i++ { wg.Add(1) - go d.downloadWorker(ctx, &wg, label, advisoryCh, errorCh) + go d.downloadWorker(ctx, &wg, advisoryCh, errorCh) } allFiles: @@ -416,18 +412,15 @@ func (d *Downloader) logValidationIssues(url string, errors []string, err error) func (d *Downloader) downloadWorker( ctx context.Context, wg *sync.WaitGroup, - label csaf.TLPLabel, files <-chan csaf.AdvisoryFile, errorCh chan<- error, ) { defer wg.Done() var ( - client = d.httpClient() - data bytes.Buffer - initialReleaseDate time.Time - dateExtract = util.TimeMatcher(&initialReleaseDate, time.RFC3339) - stats = stats{} + client = d.httpClient() + data bytes.Buffer + stats = stats{} ) // Add collected stats back to total. @@ -649,24 +642,14 @@ nextAdvisory: string(s256Data), string(s512Data)) } - if err := d.eval.Extract( - `$.document.tracking.initial_release_date`, dateExtract, false, doc, - ); err != nil { - slog.Warn("Cannot extract initial_release_date from advisory", - "url", file.URL()) - initialReleaseDate = time.Now() - } - initialReleaseDate = initialReleaseDate.UTC() download := DownloadedDocument{ - Data: data, - S256Data: s256Data, - S512Data: s512Data, - SignData: signData, - InitialReleaseDate: initialReleaseDate, - Filename: filename, - ValStatus: valStatus, - Label: label, + Data: data.Bytes(), + SHA256: s256Data, + SHA512: s512Data, + SignData: signData, + Filename: filename, + ValStatus: valStatus, } err = d.cfg.DownloadHandler(download)