feat: Added feast Go operator db stores support (#4771)

* Add support for db stores in feast go operator Signed-off-by: Theodor Mihalache <[email protected]> * Added CR example for store persistence Signed-off-by: Theodor Mihalache <[email protected]> * Fixed incorrect yaml tag in RegistryConfig struct Signed-off-by: Theodor Mihalache <[email protected]> * Removed leftovers comments from hasAttrib function Signed-off-by: Theodor Mihalache <[email protected]> * Added another check that object parameter type is the same as value type in hasAttrib Signed-off-by: Theodor Mihalache <[email protected]> * Reverted latest commit Signed-off-by: Theodor Mihalache <[email protected]> --------- Signed-off-by: Theodor Mihalache <[email protected]>
feast-dev · Nov 26, 2024 · 3302363 · 3302363
1 parent d2ba4ef
commit 3302363
Show file tree

Hide file tree

Showing 16 changed files with 1,176 additions and 163 deletions.
diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go
@@ -75,11 +75,13 @@ type OfflineStore struct {
 }
 
 // OfflineStorePersistence configures the persistence settings for the offline store service
+// +kubebuilder:validation:XValidation:rule="[has(self.file), has(self.store)].exists_one(c, c)",message="One selection required between file or store."
 type OfflineStorePersistence struct {
-	FilePersistence *OfflineStoreFilePersistence `json:"file,omitempty"`
+	FilePersistence *OfflineStoreFilePersistence    `json:"file,omitempty"`
+	DBPersistence   *OfflineStoreDBStorePersistence `json:"store,omitempty"`
 }
 
-// OfflineStorePersistence configures the file-based persistence for the offline store service
+// OfflineStoreFilePersistence configures the file-based persistence for the offline store service
 type OfflineStoreFilePersistence struct {
 	// +kubebuilder:validation:Enum=dask;duckdb
 	Type      string     `json:"type,omitempty"`
@@ -91,15 +93,35 @@ var ValidOfflineStoreFilePersistenceTypes = []string{
 	"duckdb",
 }
 
+// OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service
+type OfflineStoreDBStorePersistence struct {
+	// +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;feast_trino.trino.TrinoOfflineStore;redis
+	Type          string                       `json:"type,omitempty"`
+	SecretRef     *corev1.LocalObjectReference `json:"secretRef,omitempty"`
+	SecretKeyName string                       `json:"secretKeyName,omitempty"`
+}
+
+var ValidOfflineStoreDBStorePersistenceTypes = []string{
+	"snowflake.offline",
+	"bigquery",
+	"redshift",
+	"spark",
+	"postgres",
+	"feast_trino.trino.TrinoOfflineStore",
+	"redis",
+}
+
 // OnlineStore configures the deployed online store service
 type OnlineStore struct {
 	ServiceConfigs `json:",inline"`
 	Persistence    *OnlineStorePersistence `json:"persistence,omitempty"`
 }
 
 // OnlineStorePersistence configures the persistence settings for the online store service
+// +kubebuilder:validation:XValidation:rule="[has(self.file), has(self.store)].exists_one(c, c)",message="One selection required between file or store."
 type OnlineStorePersistence struct {
-	FilePersistence *OnlineStoreFilePersistence `json:"file,omitempty"`
+	FilePersistence *OnlineStoreFilePersistence    `json:"file,omitempty"`
+	DBPersistence   *OnlineStoreDBStorePersistence `json:"store,omitempty"`
 }
 
 // OnlineStoreFilePersistence configures the file-based persistence for the offline store service
@@ -111,6 +133,28 @@ type OnlineStoreFilePersistence struct {
 	PvcConfig *PvcConfig `json:"pvc,omitempty"`
 }
 
+// OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service
+type OnlineStoreDBStorePersistence struct {
+	// +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore
+	Type          string                       `json:"type,omitempty"`
+	SecretRef     *corev1.LocalObjectReference `json:"secretRef,omitempty"`
+	SecretKeyName string                       `json:"secretKeyName,omitempty"`
+}
+
+var ValidOnlineStoreDBStorePersistenceTypes = []string{
+	"snowflake.online",
+	"redis",
+	"ikv",
+	"datastore",
+	"dynamodb",
+	"bigtable",
+	"postgres",
+	"cassandra",
+	"mysql",
+	"hazelcast",
+	"singlestore",
+}
+
 // LocalRegistryConfig configures the deployed registry service
 type LocalRegistryConfig struct {
 	ServiceConfigs `json:",inline"`
@@ -119,7 +163,8 @@ type LocalRegistryConfig struct {
 
 // RegistryPersistence configures the persistence settings for the registry service
 type RegistryPersistence struct {
-	FilePersistence *RegistryFilePersistence `json:"file,omitempty"`
+	FilePersistence *RegistryFilePersistence    `json:"file,omitempty"`
+	DBPersistence   *RegistryDBStorePersistence `json:"store,omitempty"`
 }
 
 // RegistryFilePersistence configures the file-based persistence for the registry service
@@ -133,6 +178,19 @@ type RegistryFilePersistence struct {
 	S3AdditionalKwargs *map[string]string `json:"s3_additional_kwargs,omitempty"`
 }
 
+// RegistryDBStorePersistence configures the DB store persistence for the registry service
+type RegistryDBStorePersistence struct {
+	// +kubebuilder:validation:Enum=sql;snowflake.registry
+	Type          string                       `json:"type,omitempty"`
+	SecretRef     *corev1.LocalObjectReference `json:"secretRef,omitempty"`
+	SecretKeyName string                       `json:"secretKeyName,omitempty"`
+}
+
+var ValidRegistryDBStorePersistenceTypes = []string{
+	"snowflake.registry",
+	"sql",
+}
+
 // PvcConfig defines the settings for a persistent file store based on PVCs.
 // We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field.
 // +kubebuilder:validation:XValidation:rule="[has(self.ref), has(self.create)].exists_one(c, c)",message="One selection is required between ref and create."

diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go