Skip to content

Commit

Permalink
Move downloader to lib/downloader
Browse files Browse the repository at this point in the history
  • Loading branch information
@koplas
koplas committed Jun 18, 2024
1 parent d909e9d commit 88f3bdb
Show file tree
Hide file tree
Showing 7 changed files with 111 additions and 110 deletions.
23 changes: 12 additions & 11 deletions cmd/csaf_downloader/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ package main

import (
"context"
"github.com/csaf-poc/csaf_distribution/v3/lib/downloader"
"os"
"os/signal"

Expand All @@ -19,36 +20,36 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)

func run(cfg *config, domains []string) error {
d, err := newDownloader(cfg)
func run(cfg *downloader.Config, domains []string) error {
d, err := downloader.NewDownloader(cfg)
if err != nil {
return err
}
defer d.close()
defer d.Close()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
defer stop()

if cfg.ForwardURL != "" {
f := newForwarder(cfg)
go f.run()
f := downloader.NewForwarder(cfg)
go f.Run()
defer func() {
f.log()
f.close()
f.Log()
f.Close()
}()
d.forwarder = f
d.Forwarder = f
}

return d.run(ctx, domains)
return d.Run(ctx, domains)
}

func main() {

domains, cfg, err := parseArgsConfig()
domains, cfg, err := downloader.ParseArgsConfig()
options.ErrorCheck(err)
options.ErrorCheck(cfg.prepare())
options.ErrorCheck(cfg.Prepare())

if len(domains) == 0 {
slog.Warn("No domains given.")
Expand Down
46 changes: 23 additions & 23 deletions cmd/csaf_downloader/config.go → lib/downloader/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

package main
package downloader

import (
"crypto/tls"
Expand Down Expand Up @@ -42,7 +42,7 @@ const (
validationUnsafe = validationMode("unsafe")
)

type config struct {
type Config struct {

Check failure on line 45 in lib/downloader/config.go

View workflow job for this annotation

GitHub Actions / build 

exported type Config should have comment or be unexported
Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"`
IgnoreSignatureCheck bool `long:"ignore_sigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignore_sigcheck"`
Expand Down Expand Up @@ -80,25 +80,25 @@ type config struct {
ignorePattern filter.PatternMatcher
}

// configPaths are the potential file locations of the config file.
// configPaths are the potential file locations of the Config file.
var configPaths = []string{
"~/.config/csaf/downloader.toml",
"~/.csaf_downloader.toml",
"csaf_downloader.toml",
}

// parseArgsConfig parses the command line and if need a config file.
func parseArgsConfig() ([]string, *config, error) {
// ParseArgsConfig parses the command line and if need a config file.
func ParseArgsConfig() ([]string, *Config, error) {
var (
logFile = defaultLogFile
logLevel = &options.LogLevel{Level: defaultLogLevel}
)
p := options.Parser[config]{
p := options.Parser[Config]{
DefaultConfigLocations: configPaths,
ConfigLocation: func(cfg *config) string { return cfg.Config },
ConfigLocation: func(cfg *Config) string { return cfg.Config },
Usage: "[OPTIONS] domain...",
HasVersion: func(cfg *config) bool { return cfg.Version },
SetDefaults: func(cfg *config) {
HasVersion: func(cfg *Config) bool { return cfg.Version },
SetDefaults: func(cfg *Config) {
cfg.Worker = defaultWorker
cfg.RemoteValidatorPresets = []string{defaultPreset}
cfg.ValidationMode = defaultValidationMode
Expand All @@ -107,7 +107,7 @@ func parseArgsConfig() ([]string, *config, error) {
cfg.LogLevel = logLevel
},
// Re-establish default values if not set.
EnsureDefaults: func(cfg *config) {
EnsureDefaults: func(cfg *Config) {
if cfg.Worker == 0 {
cfg.Worker = defaultWorker
}
Expand Down Expand Up @@ -152,18 +152,18 @@ func (vm *validationMode) UnmarshalFlag(value string) error {
}

// ignoreFile returns true if the given URL should not be downloaded.
func (cfg *config) ignoreURL(u string) bool {
func (cfg *Config) ignoreURL(u string) bool {
return cfg.ignorePattern.Matches(u)
}

// verbose is considered a log level equal or less debug.
func (cfg *config) verbose() bool {
func (cfg *Config) verbose() bool {
return cfg.LogLevel.Level <= slog.LevelDebug
}

// prepareDirectory ensures that the working directory
// exists and is setup properly.
func (cfg *config) prepareDirectory() error {
func (cfg *Config) prepareDirectory() error {
// If not given use current working directory.
if cfg.Directory == "" {
dir, err := os.Getwd()
Expand Down Expand Up @@ -197,7 +197,7 @@ func dropSubSeconds(_ []string, a slog.Attr) slog.Attr {
}

// prepareLogging sets up the structured logging.
func (cfg *config) prepareLogging() error {
func (cfg *Config) prepareLogging() error {
var w io.Writer
if cfg.LogFile == nil || *cfg.LogFile == "" {
log.Println("using STDERR for logging")
Expand Down Expand Up @@ -230,7 +230,7 @@ func (cfg *config) prepareLogging() error {
}

// compileIgnorePatterns compiles the configure patterns to be ignored.
func (cfg *config) compileIgnorePatterns() error {
func (cfg *Config) compileIgnorePatterns() error {
pm, err := filter.NewPatternMatcher(cfg.IgnorePattern)
if err != nil {
return err
Expand All @@ -240,7 +240,7 @@ func (cfg *config) compileIgnorePatterns() error {
}

// prepareCertificates loads the client side certificates used by the HTTP client.
func (cfg *config) prepareCertificates() error {
func (cfg *Config) prepareCertificates() error {
cert, err := certs.LoadCertificate(
cfg.ClientCert, cfg.ClientKey, cfg.ClientPassphrase)
if err != nil {
Expand All @@ -250,13 +250,13 @@ func (cfg *config) prepareCertificates() error {
return nil
}

// prepare prepares internal state of a loaded configuration.
func (cfg *config) prepare() error {
for _, prepare := range []func(*config) error{
(*config).prepareDirectory,
(*config).prepareLogging,
(*config).prepareCertificates,
(*config).compileIgnorePatterns,
// Prepare prepares internal state of a loaded configuration.
func (cfg *Config) Prepare() error {
for _, prepare := range []func(*Config) error{
(*Config).prepareDirectory,
(*Config).prepareLogging,
(*Config).prepareCertificates,
(*Config).compileIgnorePatterns,
} {
if err := prepare(cfg); err != nil {
return err
Expand Down
58 changes: 29 additions & 29 deletions cmd/csaf_downloader/downloader.go → lib/downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022, 2023 Intevation GmbH <https://intevation.de>

package main
package downloader

import (
"bytes"
Expand Down Expand Up @@ -38,12 +38,12 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/util"
)

type downloader struct {
cfg *config
type Downloader struct {

Check failure on line 41 in lib/downloader/downloader.go

View workflow job for this annotation

GitHub Actions / build 

exported type Downloader should have comment or be unexported
cfg *Config
keys *crypto.KeyRing
eval *util.PathEval
validator csaf.RemoteValidator
forwarder *forwarder
Forwarder *Forwarder
mkdirMu sync.Mutex
statsMu sync.Mutex
stats stats
Expand All @@ -54,7 +54,7 @@ type downloader struct {
// unsafe mode.
const failedValidationDir = "failed_validation"

func newDownloader(cfg *config) (*downloader, error) {
func NewDownloader(cfg *Config) (*Downloader, error) {

Check failure on line 57 in lib/downloader/downloader.go

View workflow job for this annotation

GitHub Actions / build 

exported function NewDownloader should have comment or be unexported

var validator csaf.RemoteValidator

Expand All @@ -72,22 +72,22 @@ func newDownloader(cfg *config) (*downloader, error) {
validator = csaf.SynchronizedRemoteValidator(validator)
}

return &downloader{
return &Downloader{
cfg: cfg,
eval: util.NewPathEval(),
validator: validator,
}, nil
}

func (d *downloader) close() {
func (d *Downloader) Close() {

Check failure on line 82 in lib/downloader/downloader.go

View workflow job for this annotation

GitHub Actions / build 

exported method Downloader.Close should have comment or be unexported
if d.validator != nil {
d.validator.Close()
d.validator = nil
}
}

// addStats add stats to total stats
func (d *downloader) addStats(o *stats) {
func (d *Downloader) addStats(o *stats) {
d.statsMu.Lock()
defer d.statsMu.Unlock()
d.stats.add(o)
Expand All @@ -105,7 +105,7 @@ func logRedirect(req *http.Request, via []*http.Request) error {
return nil
}

func (d *downloader) httpClient() util.Client {
func (d *Downloader) httpClient() util.Client {

hClient := http.Client{}

Expand Down Expand Up @@ -165,7 +165,7 @@ func httpLog(who string) func(string, string) {
}
}

func (d *downloader) download(ctx context.Context, domain string) error {
func (d *Downloader) download(ctx context.Context, domain string) error {
client := d.httpClient()

loader := csaf.NewProviderMetadataLoader(client)
Expand Down Expand Up @@ -215,7 +215,7 @@ func (d *downloader) download(ctx context.Context, domain string) error {
})
}

func (d *downloader) downloadFiles(
func (d *Downloader) downloadFiles(
ctx context.Context,
label csaf.TLPLabel,
files []csaf.AdvisoryFile,
Expand Down Expand Up @@ -264,7 +264,7 @@ allFiles:
return errors.Join(errs...)
}

func (d *downloader) loadOpenPGPKeys(
func (d *Downloader) loadOpenPGPKeys(
client util.Client,
doc any,
base *url.URL,
Expand Down Expand Up @@ -355,7 +355,7 @@ func (d *downloader) loadOpenPGPKeys(
}

// logValidationIssues logs the issues reported by the advisory schema validation.
func (d *downloader) logValidationIssues(url string, errors []string, err error) {
func (d *Downloader) logValidationIssues(url string, errors []string, err error) {
if err != nil {
slog.Error("Failed to validate",
"url", url,
Expand All @@ -375,7 +375,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error)
}
}

func (d *downloader) downloadWorker(
func (d *Downloader) downloadWorker(
ctx context.Context,
wg *sync.WaitGroup,
label csaf.TLPLabel,
Expand Down Expand Up @@ -549,9 +549,9 @@ nextAdvisory:

// Validate against CSAF schema.
schemaCheck := func() error {
if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
if errs, err := csaf.ValidateCSAF(doc); err != nil || len(errs) > 0 {
stats.schemaFailed++
d.logValidationIssues(file.URL(), errors, err)
d.logValidationIssues(file.URL(), errs, err)
return fmt.Errorf("schema validation for %q failed", file.URL())
}
return nil
Expand Down Expand Up @@ -605,9 +605,9 @@ nextAdvisory:
}
valStatus.update(validValidationStatus)

// Send to forwarder
if d.forwarder != nil {
d.forwarder.forward(
// Send to Forwarder
if d.Forwarder != nil {
d.Forwarder.forward(
filename, data.String(),
valStatus,
string(s256Data),
Expand Down Expand Up @@ -655,17 +655,17 @@ nextAdvisory:
}

// Write advisory to file
path := filepath.Join(lastDir, filename)
filePath := filepath.Join(lastDir, filename)

// Write data to disk.
for _, x := range []struct {
p string
d []byte
}{
{path, data.Bytes()},
{path + ".sha256", s256Data},
{path + ".sha512", s512Data},
{path + ".asc", signData},
{filePath, data.Bytes()},
{filePath + ".sha256", s256Data},
{filePath + ".sha512", s512Data},
{filePath + ".asc", signData},
} {
if x.d != nil {
if err := os.WriteFile(x.p, x.d, 0644); err != nil {
Expand All @@ -676,17 +676,17 @@ nextAdvisory:
}

stats.succeeded++
slog.Info("Written advisory", "path", path)
slog.Info("Written advisory", "path", filePath)
}
}

func (d *downloader) mkdirAll(path string, perm os.FileMode) error {
func (d *Downloader) mkdirAll(path string, perm os.FileMode) error {
d.mkdirMu.Lock()
defer d.mkdirMu.Unlock()
return os.MkdirAll(path, perm)
}

func (d *downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error {
func (d *Downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error {
pm := crypto.NewPlainMessage(data)
t := crypto.GetUnixTime()
return d.keys.VerifyDetached(pm, sign, t)
Expand Down Expand Up @@ -732,8 +732,8 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) {
return hash, data.Bytes(), nil
}

// run performs the downloads for all the given domains.
func (d *downloader) run(ctx context.Context, domains []string) error {
// Run performs the downloads for all the given domains.
func (d *Downloader) Run(ctx context.Context, domains []string) error {
defer d.stats.log()
for _, domain := range domains {
if err := d.download(ctx, domain); err != nil {
Expand Down
Loading

0 comments on commit 88f3bdb

Please sign in to comment.