From 8a95aedda542f76d44ad2b99035cb6a1fa1777b9 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Mon, 28 Oct 2024 12:36:15 +0900 Subject: [PATCH] Support application credential auth for Openstack swift (#6255) --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 12 ++++++ docs/blocks-storage/store-gateway.md | 12 ++++++ docs/configuration/config-file-reference.md | 48 +++++++++++++++++++++ pkg/storage/bucket/swift/bucket_client.go | 41 ++++++++++-------- pkg/storage/bucket/swift/config.go | 42 ++++++++++-------- 6 files changed, 119 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 524d7690e6..d4a059b5bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245 * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249 * [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280 +* [ENHANCEMENT] OpenStack Swift: Add application credential configs for Openstack swift object storage backend. #6255 * [ENHANCEMENT] Query Frontend: Add new query stats metrics `cortex_query_samples_scanned_total` and `cortex_query_peak_samples` to track scannedSamples and peakSample per user. #6228 * [ENHANCEMENT] Ingester: Disable chunk trimming. #6270 * [ENHANCEMENT] Ingester: Add `blocks-storage.tsdb.wal-compression-type` to support zstd wal compression type. #6232 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 6e4b5d78ef..bdf1e3243e 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -456,6 +456,18 @@ blocks_storage: # CLI flag: -blocks-storage.swift.auth-url [auth_url: | default = ""] + # OpenStack Swift application credential ID. + # CLI flag: -blocks-storage.swift.application-credential-id + [application_credential_id: | default = ""] + + # OpenStack Swift application credential name. + # CLI flag: -blocks-storage.swift.application-credential-name + [application_credential_name: | default = ""] + + # OpenStack Swift application credential secret. + # CLI flag: -blocks-storage.swift.application-credential-secret + [application_credential_secret: | default = ""] + # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: | default = ""] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 98eaf3c8f3..b7c353a871 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -547,6 +547,18 @@ blocks_storage: # CLI flag: -blocks-storage.swift.auth-url [auth_url: | default = ""] + # OpenStack Swift application credential ID. + # CLI flag: -blocks-storage.swift.application-credential-id + [application_credential_id: | default = ""] + + # OpenStack Swift application credential name. + # CLI flag: -blocks-storage.swift.application-credential-name + [application_credential_name: | default = ""] + + # OpenStack Swift application credential secret. + # CLI flag: -blocks-storage.swift.application-credential-secret + [application_credential_secret: | default = ""] + # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: | default = ""] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index a3e9eb04ab..f8e0ee6369 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -706,6 +706,18 @@ swift: # CLI flag: -alertmanager-storage.swift.auth-url [auth_url: | default = ""] + # OpenStack Swift application credential ID. + # CLI flag: -alertmanager-storage.swift.application-credential-id + [application_credential_id: | default = ""] + + # OpenStack Swift application credential name. + # CLI flag: -alertmanager-storage.swift.application-credential-name + [application_credential_name: | default = ""] + + # OpenStack Swift application credential secret. + # CLI flag: -alertmanager-storage.swift.application-credential-secret + [application_credential_secret: | default = ""] + # OpenStack Swift username. # CLI flag: -alertmanager-storage.swift.username [username: | default = ""] @@ -989,6 +1001,18 @@ swift: # CLI flag: -blocks-storage.swift.auth-url [auth_url: | default = ""] + # OpenStack Swift application credential ID. + # CLI flag: -blocks-storage.swift.application-credential-id + [application_credential_id: | default = ""] + + # OpenStack Swift application credential name. + # CLI flag: -blocks-storage.swift.application-credential-name + [application_credential_name: | default = ""] + + # OpenStack Swift application credential secret. + # CLI flag: -blocks-storage.swift.application-credential-secret + [application_credential_secret: | default = ""] + # OpenStack Swift username. # CLI flag: -blocks-storage.swift.username [username: | default = ""] @@ -4762,6 +4786,18 @@ swift: # CLI flag: -ruler-storage.swift.auth-url [auth_url: | default = ""] + # OpenStack Swift application credential ID. + # CLI flag: -ruler-storage.swift.application-credential-id + [application_credential_id: | default = ""] + + # OpenStack Swift application credential name. + # CLI flag: -ruler-storage.swift.application-credential-name + [application_credential_name: | default = ""] + + # OpenStack Swift application credential secret. + # CLI flag: -ruler-storage.swift.application-credential-secret + [application_credential_secret: | default = ""] + # OpenStack Swift username. # CLI flag: -ruler-storage.swift.username [username: | default = ""] @@ -5053,6 +5089,18 @@ swift: # CLI flag: -runtime-config.swift.auth-url [auth_url: | default = ""] + # OpenStack Swift application credential ID. + # CLI flag: -runtime-config.swift.application-credential-id + [application_credential_id: | default = ""] + + # OpenStack Swift application credential name. + # CLI flag: -runtime-config.swift.application-credential-name + [application_credential_name: | default = ""] + + # OpenStack Swift application credential secret. + # CLI flag: -runtime-config.swift.application-credential-secret + [application_credential_secret: | default = ""] + # OpenStack Swift username. # CLI flag: -runtime-config.swift.username [username: | default = ""] diff --git a/pkg/storage/bucket/swift/bucket_client.go b/pkg/storage/bucket/swift/bucket_client.go index 1a83cddd08..012d80b598 100644 --- a/pkg/storage/bucket/swift/bucket_client.go +++ b/pkg/storage/bucket/swift/bucket_client.go @@ -9,26 +9,29 @@ import ( ) // NewBucketClient creates a new Swift bucket client -func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { +func NewBucketClient(cfg Config, _ string, logger log.Logger) (objstore.Bucket, error) { bucketConfig := swift.Config{ - AuthVersion: cfg.AuthVersion, - AuthUrl: cfg.AuthURL, - Username: cfg.Username, - UserDomainName: cfg.UserDomainName, - UserDomainID: cfg.UserDomainID, - UserId: cfg.UserID, - Password: cfg.Password, - DomainId: cfg.DomainID, - DomainName: cfg.DomainName, - ProjectID: cfg.ProjectID, - ProjectName: cfg.ProjectName, - ProjectDomainID: cfg.ProjectDomainID, - ProjectDomainName: cfg.ProjectDomainName, - RegionName: cfg.RegionName, - ContainerName: cfg.ContainerName, - Retries: cfg.MaxRetries, - ConnectTimeout: model.Duration(cfg.ConnectTimeout), - Timeout: model.Duration(cfg.RequestTimeout), + AuthVersion: cfg.AuthVersion, + AuthUrl: cfg.AuthURL, + ApplicationCredentialID: cfg.ApplicationCredentialID, + ApplicationCredentialName: cfg.ApplicationCredentialName, + ApplicationCredentialSecret: cfg.ApplicationCredentialSecret, + Username: cfg.Username, + UserDomainName: cfg.UserDomainName, + UserDomainID: cfg.UserDomainID, + UserId: cfg.UserID, + Password: cfg.Password, + DomainId: cfg.DomainID, + DomainName: cfg.DomainName, + ProjectID: cfg.ProjectID, + ProjectName: cfg.ProjectName, + ProjectDomainID: cfg.ProjectDomainID, + ProjectDomainName: cfg.ProjectDomainName, + RegionName: cfg.RegionName, + ContainerName: cfg.ContainerName, + Retries: cfg.MaxRetries, + ConnectTimeout: model.Duration(cfg.ConnectTimeout), + Timeout: model.Duration(cfg.RequestTimeout), // Hard-coded defaults. ChunkSize: swift.DefaultConfig.ChunkSize, diff --git a/pkg/storage/bucket/swift/config.go b/pkg/storage/bucket/swift/config.go index 783621f887..110da50308 100644 --- a/pkg/storage/bucket/swift/config.go +++ b/pkg/storage/bucket/swift/config.go @@ -7,24 +7,27 @@ import ( // Config holds the config options for Swift backend type Config struct { - AuthVersion int `yaml:"auth_version"` - AuthURL string `yaml:"auth_url"` - Username string `yaml:"username"` - UserDomainName string `yaml:"user_domain_name"` - UserDomainID string `yaml:"user_domain_id"` - UserID string `yaml:"user_id"` - Password string `yaml:"password"` - DomainID string `yaml:"domain_id"` - DomainName string `yaml:"domain_name"` - ProjectID string `yaml:"project_id"` - ProjectName string `yaml:"project_name"` - ProjectDomainID string `yaml:"project_domain_id"` - ProjectDomainName string `yaml:"project_domain_name"` - RegionName string `yaml:"region_name"` - ContainerName string `yaml:"container_name"` - MaxRetries int `yaml:"max_retries"` - ConnectTimeout time.Duration `yaml:"connect_timeout"` - RequestTimeout time.Duration `yaml:"request_timeout"` + AuthVersion int `yaml:"auth_version"` + AuthURL string `yaml:"auth_url"` + ApplicationCredentialID string `yaml:"application_credential_id"` + ApplicationCredentialName string `yaml:"application_credential_name"` + ApplicationCredentialSecret string `yaml:"application_credential_secret"` + Username string `yaml:"username"` + UserDomainName string `yaml:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id"` + UserID string `yaml:"user_id"` + Password string `yaml:"password"` + DomainID string `yaml:"domain_id"` + DomainName string `yaml:"domain_name"` + ProjectID string `yaml:"project_id"` + ProjectName string `yaml:"project_name"` + ProjectDomainID string `yaml:"project_domain_id"` + ProjectDomainName string `yaml:"project_domain_name"` + RegionName string `yaml:"region_name"` + ContainerName string `yaml:"container_name"` + MaxRetries int `yaml:"max_retries"` + ConnectTimeout time.Duration `yaml:"connect_timeout"` + RequestTimeout time.Duration `yaml:"request_timeout"` } // RegisterFlags registers the flags for Swift storage @@ -47,6 +50,9 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.ProjectName, prefix+"swift.project-name", "", "OpenStack Swift project name (v2,v3 auth only).") f.StringVar(&cfg.ProjectDomainID, prefix+"swift.project-domain-id", "", "ID of the OpenStack Swift project's domain (v3 auth only), only needed if it differs the from user domain.") f.StringVar(&cfg.ProjectDomainName, prefix+"swift.project-domain-name", "", "Name of the OpenStack Swift project's domain (v3 auth only), only needed if it differs from the user domain.") + f.StringVar(&cfg.ApplicationCredentialID, prefix+"swift.application-credential-id", "", "OpenStack Swift application credential ID.") + f.StringVar(&cfg.ApplicationCredentialName, prefix+"swift.application-credential-name", "", "OpenStack Swift application credential name.") + f.StringVar(&cfg.ApplicationCredentialSecret, prefix+"swift.application-credential-secret", "", "OpenStack Swift application credential secret.") f.StringVar(&cfg.RegionName, prefix+"swift.region-name", "", "OpenStack Swift Region to use (v2,v3 auth only).") f.StringVar(&cfg.ContainerName, prefix+"swift.container-name", "", "Name of the OpenStack Swift container to put chunks in.") f.IntVar(&cfg.MaxRetries, prefix+"swift.max-retries", 3, "Max retries on requests error.")