From 008862c3ad4eb1de0653322f2e630c436cefd932 Mon Sep 17 00:00:00 2001 From: Fabio Burzigotti Date: Mon, 6 Jan 2025 11:32:33 +0100 Subject: [PATCH] Add ODH application, provisioning and tests --- .../jboss/intersmash/IntersmashConfig.java | 23 ++ .../OpenDataHubOperatorApplication.java | 44 +++ ...enDataHubOpenShiftOperatorProvisioner.java | 112 ++++++++ .../OpenDataHubOperatorProvisioner.java | 261 ++++++++++++++++++ ...OpenDataHubOperatorProvisionerFactory.java | 41 +++ .../model/odh/DSCInitializationList.java | 22 ++ .../model/odh/DataScienceClusterList.java | 22 ++ .../model/odh/FeatureTrackerList.java | 22 ++ .../operator/model/odh/MonitoringList.java | 22 ++ ...ss.intersmash.provision.ProvisionerFactory | 3 +- ...taHubOpenShiftOperatorProvisionerTest.java | 140 ++++++++++ .../OpenShiftProvisionerTestBase.java | 39 ++- .../openshift/ProvisionerCleanupTestCase.java | 5 +- 13 files changed, 752 insertions(+), 4 deletions(-) create mode 100644 provisioners/src/main/java/org/jboss/intersmash/application/operator/OpenDataHubOperatorApplication.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/openshift/OpenDataHubOpenShiftOperatorProvisioner.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisioner.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisionerFactory.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DSCInitializationList.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DataScienceClusterList.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/FeatureTrackerList.java create mode 100644 provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/MonitoringList.java create mode 100644 testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenDataHubOpenShiftOperatorProvisionerTest.java diff --git a/core/src/main/java/org/jboss/intersmash/IntersmashConfig.java b/core/src/main/java/org/jboss/intersmash/IntersmashConfig.java index 55a6a0a0..18bc45dc 100644 --- a/core/src/main/java/org/jboss/intersmash/IntersmashConfig.java +++ b/core/src/main/java/org/jboss/intersmash/IntersmashConfig.java @@ -85,6 +85,13 @@ public class IntersmashConfig { private static final String HYPERFOIL_OPERATOR_PACKAGE_MANIFEST = "intersmash.hyperfoil.operators.package_manifest"; private static final String COMMUNITY_HYPERFOIL_OPERATOR_PACKAGE_MANIFEST = "hyperfoil-bundle"; private static final String DEFAULT_HYPERFOIL_OPERATOR_PACKAGE_MANIFEST = COMMUNITY_HYPERFOIL_OPERATOR_PACKAGE_MANIFEST; + private static final String OPEN_DATA_HUB_OPERATOR_CATALOG_SOURCE_NAME = "intersmash.odh.operators.catalog_source"; + private static final String OPEN_DATA_HUB_OPERATOR_INDEX_IMAGE = "intersmash.odh.operators.index_image"; + private static final String OPEN_DATA_HUB_OPERATOR_CHANNEL = "intersmash.odh.operators.channel"; + private static final String OPEN_DATA_HUB_OPERATOR_PACKAGE_MANIFEST = "intersmash.odh.operators.package_manifest"; + private static final String COMMUNITY_OPEN_DATA_HUB_OPERATOR_PACKAGE_MANIFEST = "opendatahub-operator"; + private static final String DEFAULT_OPEN_DATA_HUB_OPERATOR_PACKAGE_MANIFEST = COMMUNITY_OPEN_DATA_HUB_OPERATOR_PACKAGE_MANIFEST; + // Bootable Jar private static final String BOOTABLE_JAR_IMAGE_URL = "intersmash.bootable.jar.image"; @@ -407,4 +414,20 @@ public static String keycloakOperatorChannel() { public static String keycloakOperatorPackageManifest() { return XTFConfig.get(KEYCLOAK_OPERATOR_PACKAGE_MANIFEST, DEFAULT_KEYCLOAK_OPERATOR_PACKAGE_MANIFEST); } + + public static String openDataHubOperatorCatalogSource() { + return XTFConfig.get(OPEN_DATA_HUB_OPERATOR_CATALOG_SOURCE_NAME, defaultOperatorCatalogSourceName()); + } + + public static String openDataHubOperatorIndexImage() { + return XTFConfig.get(OPEN_DATA_HUB_OPERATOR_INDEX_IMAGE); + } + + public static String openDataHubOperatorChannel() { + return XTFConfig.get(OPEN_DATA_HUB_OPERATOR_CHANNEL); + } + + public static String openDataHubOperatorPackageManifest() { + return XTFConfig.get(OPEN_DATA_HUB_OPERATOR_PACKAGE_MANIFEST, DEFAULT_OPEN_DATA_HUB_OPERATOR_PACKAGE_MANIFEST); + } } diff --git a/provisioners/src/main/java/org/jboss/intersmash/application/operator/OpenDataHubOperatorApplication.java b/provisioners/src/main/java/org/jboss/intersmash/application/operator/OpenDataHubOperatorApplication.java new file mode 100644 index 00000000..235872c0 --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/application/operator/OpenDataHubOperatorApplication.java @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.application.operator; + +import io.opendatahub.datasciencecluster.v1.DataScienceCluster; +import io.opendatahub.dscinitialization.v1.DSCInitialization; + +/** + * End user Application interface which presents an OpenShift AI operator application. + * Only relevant model APIs are currently exposed, more would be added on demand + * + * The application will be deployed by: + * + */ +public interface OpenDataHubOperatorApplication extends OperatorApplication { + + /** + * Provides read access to the deployed {@link DataScienceCluster} instance. + * @return A {@link DataScienceCluster} instance that is deployed. + */ + DataScienceCluster getDataScienceCluster(); + + /** + * Provides read access to the {@link DSCInitialization} instance that should be used to initialize the + * deployed {@link DataScienceCluster}, represented by {@link #getDataScienceCluster} getter. + * @return A {@link DSCInitialization} instance that configures the deployed {@link DataScienceCluster} + */ + DSCInitialization getDSCInitialization(); +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/openshift/OpenDataHubOpenShiftOperatorProvisioner.java b/provisioners/src/main/java/org/jboss/intersmash/provision/openshift/OpenDataHubOpenShiftOperatorProvisioner.java new file mode 100644 index 00000000..f08b3952 --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/openshift/OpenDataHubOpenShiftOperatorProvisioner.java @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2023 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.openshift; + +import org.jboss.intersmash.application.operator.OpenDataHubOperatorApplication; +import org.jboss.intersmash.provision.operator.OpenDataHubOperatorProvisioner; +import org.jboss.intersmash.provision.operator.model.odh.DSCInitializationList; +import org.jboss.intersmash.provision.operator.model.odh.DataScienceClusterList; +import org.jboss.intersmash.provision.operator.model.odh.FeatureTrackerList; +import org.jboss.intersmash.provision.operator.model.odh.MonitoringList; + +import cz.xtf.core.openshift.OpenShifts; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionList; +import io.fabric8.kubernetes.client.NamespacedKubernetesClientAdapter; +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext; +import io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperationsImpl; +import io.fabric8.openshift.client.NamespacedOpenShiftClient; +import io.opendatahub.datasciencecluster.v1.DataScienceCluster; +import io.opendatahub.dscinitialization.v1.DSCInitialization; +import io.opendatahub.features.v1.FeatureTracker; +import io.opendatahub.platform.services.v1alpha1.Monitoring; +import lombok.NonNull; + +public class OpenDataHubOpenShiftOperatorProvisioner + // leverage Open Data Hub common Operator based provisioner behavior + extends OpenDataHubOperatorProvisioner + // ... and common OpenShift provisioning logic, too + implements OpenShiftProvisioner { + + private static final String OPENSHIFT_OPERATORS_NAMESPACE = "openshift-operators"; + + public OpenDataHubOpenShiftOperatorProvisioner( + @NonNull OpenDataHubOperatorApplication application) { + super(application); + } + + @Override + protected String getTargetNamespace() { + return OPENSHIFT_OPERATORS_NAMESPACE; + } + + @Override + public NamespacedKubernetesClientAdapter client() { + return OpenShifts.admin(); + } + + @Override + public String execute(String... args) { + return OpenShiftProvisioner.super.execute(args); + } + + // ================================================================================================================= + // Related to generic provisioning behavior + // ================================================================================================================= + @Override + protected void removeClusterServiceVersion() { + this.execute("delete", "csvs", currentCSV, "-n", this.getTargetNamespace(), "--ignore-not-found"); + } + + @Override + protected void removeSubscription() { + this.execute("delete", "subscription", packageManifestName, "-n", this.getTargetNamespace(), "--ignore-not-found"); + } + + // ================================================================================================================= + // Client related + // ================================================================================================================= + @Override + public NonNamespaceOperation> customResourceDefinitionsClient() { + return OpenShifts.admin().apiextensions().v1().customResourceDefinitions(); + } + + @Override + protected HasMetadataOperationsImpl dataScienceClusterCustomResourcesClient( + CustomResourceDefinitionContext crdc) { + return OpenShifts.admin().newHasMetadataOperation(crdc, DataScienceCluster.class, DataScienceClusterList.class); + } + + @Override + protected HasMetadataOperationsImpl dscInitializationCustomResourcesClient( + CustomResourceDefinitionContext crdc) { + return OpenShifts.admin().newHasMetadataOperation(crdc, DSCInitialization.class, DSCInitializationList.class); + } + + @Override + protected HasMetadataOperationsImpl featureTrackerCustomResourcesClient( + CustomResourceDefinitionContext crdc) { + return OpenShifts.admin().newHasMetadataOperation(crdc, FeatureTracker.class, FeatureTrackerList.class); + } + + @Override + protected HasMetadataOperationsImpl monitoringCustomResourcesClient( + CustomResourceDefinitionContext crdc) { + return OpenShifts.admin().newHasMetadataOperation(crdc, Monitoring.class, MonitoringList.class); + } +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisioner.java b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisioner.java new file mode 100644 index 00000000..06193151 --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisioner.java @@ -0,0 +1,261 @@ +/** + * Copyright (C) 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.operator; + +import java.util.List; + +import org.jboss.intersmash.IntersmashConfig; +import org.jboss.intersmash.application.operator.OpenDataHubOperatorApplication; +import org.jboss.intersmash.provision.Provisioner; +import org.jboss.intersmash.provision.operator.model.odh.DSCInitializationList; +import org.jboss.intersmash.provision.operator.model.odh.DataScienceClusterList; +import org.jboss.intersmash.provision.operator.model.odh.FeatureTrackerList; +import org.jboss.intersmash.provision.operator.model.odh.MonitoringList; +import org.slf4j.event.Level; + +import cz.xtf.core.waiting.SimpleWaiter; +import cz.xtf.core.waiting.failfast.FailFastCheck; +import io.fabric8.kubernetes.api.model.StatusDetails; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionList; +import io.fabric8.kubernetes.client.NamespacedKubernetesClient; +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext; +import io.fabric8.kubernetes.client.dsl.internal.HasMetadataOperationsImpl; +import io.opendatahub.datasciencecluster.v1.DataScienceCluster; +import io.opendatahub.dscinitialization.v1.DSCInitialization; +import io.opendatahub.features.v1.FeatureTracker; +import io.opendatahub.platform.services.v1alpha1.Monitoring; +import lombok.extern.slf4j.Slf4j; + +/** + * Defines the contract and default behavior of an Operator based provisioner for the Open Data Hub Operator + */ +@Slf4j +public abstract class OpenDataHubOperatorProvisioner extends + OperatorProvisioner implements Provisioner { + + public OpenDataHubOperatorProvisioner(OpenDataHubOperatorApplication application) { + super(application, OpenDataHubOperatorProvisioner.OPERATOR_ID); + } + + // ================================================================================================================= + // Open Data Hub related + // ================================================================================================================= + + // ================================================================================================================= + // Related to generic provisioning behavior + // ================================================================================================================= + @Override + public String getOperatorCatalogSource() { + return IntersmashConfig.openDataHubOperatorCatalogSource(); + } + + @Override + public String getOperatorIndexImage() { + return IntersmashConfig.openDataHubOperatorIndexImage(); + } + + @Override + public String getOperatorChannel() { + return IntersmashConfig.openDataHubOperatorChannel(); + } + + @Override + public void deploy() { + FailFastCheck ffCheck = getFailFastCheck(); + + subscribe(); + + dataScienceClusterClient().createOrReplace(getApplication().getDataScienceCluster()); + new SimpleWaiter(() -> dataScienceCluster().get().getStatus() != null) + .failFast(ffCheck) + .reason("Wait for status field to be initialized.") + .level(Level.DEBUG) + .waitFor(); + + dscInitializationClient().createOrReplace(getApplication().getDSCInitialization()); + } + + @Override + public void undeploy() { + // remove the CRs + List deletionDetails = dataScienceCluster().delete(); + boolean deleted = deletionDetails.stream().allMatch(d -> d.getCauses().isEmpty()); + if (!deleted) { + log.warn("Wasn't able to remove the 'DataScienceCluster' resource"); + } + new SimpleWaiter(() -> dataScienceCluster().get() == null) + .reason("Waiting for the the 'DataScienceCluster' resource to be removed.") + .level(Level.DEBUG) + .waitFor(); + + if (getApplication().getDSCInitialization() != null) { + final String appName = getApplication().getName(); + deletionDetails = dscInitializationClient().withName(appName).delete(); + deleted = deletionDetails.stream().allMatch(d -> d.getCauses().isEmpty()); + if (!deleted) { + log.warn("Wasn't able to remove the 'DSCInitialization' resources created for '{}' instance!", + appName); + } + new SimpleWaiter(() -> dscInitializationClient().list().getItems().isEmpty()).level(Level.DEBUG).waitFor(); + } + // delete the OLM subscription + unsubscribe(); + new SimpleWaiter(() -> getPods().isEmpty()) + .reason("Waiting for the all the HyperFoil pods to be stopped.") + .level(Level.DEBUG) + .waitFor(); + } + + @Override + public void scale(int replicas, boolean wait) { + throw new UnsupportedOperationException("Scaling is not implemented by Open Data Hub operator based provisioning"); + } + + // ================================================================================================================= + // Client related + // ================================================================================================================= + // this is the packagemanifest for the operator; + // you can get it with command: + // oc get packagemanifest -o template --template='{{ .metadata.name }}' + public static String OPERATOR_ID = IntersmashConfig.openDataHubOperatorPackageManifest(); + + // this is the name of the CustomResourceDefinition(s) + // you can get it with command: + // oc get crd > -o template --template='{{ .metadata.name }}' + protected static String ODH_DATA_SCIENCE_CLUSTER_CRD_NAME = "datascienceclusters.datasciencecluster.opendatahub.io"; + + protected static String ODH_DSC_INITIALIZATION_CRD_NAME = "dscinitializations.dscinitialization.opendatahub.io"; + + protected static String ODH_FEATURE_TRACKER_CRD_NAME = "featuretrackers.features.opendatahub.io"; + + protected static String ODH_MONITORING_CRD_NAME = "monitorings.services.platform.opendatahub.io"; + + /** + * Generic CRD client which is used by client builders default implementation to build the CRDs client + * + * @return A {@link NonNamespaceOperation} instance that represents a + */ + protected abstract NonNamespaceOperation> customResourceDefinitionsClient(); + + // datascienceclusters.datasciencecluster.opendatahub.io + protected abstract HasMetadataOperationsImpl dataScienceClusterCustomResourcesClient( + CustomResourceDefinitionContext crdc); + + // dscinitializations.dscinitialization.opendatahub.io + protected abstract HasMetadataOperationsImpl dscInitializationCustomResourcesClient( + CustomResourceDefinitionContext crdc); + + // featuretrackers.features.opendatahub.io + protected abstract HasMetadataOperationsImpl featureTrackerCustomResourcesClient( + CustomResourceDefinitionContext crdc); + + // monitorings.services.platform.opendatahub.io + protected abstract HasMetadataOperationsImpl monitoringCustomResourcesClient( + CustomResourceDefinitionContext crdc); + + private static NonNamespaceOperation> ODH_DATA_SCIENCE_CLUSTER_CLIENT; + private static NonNamespaceOperation> ODH_DSC_INITIALIZATION_CLIENT; + private static NonNamespaceOperation> ODH_FEATURE_TRACKER_CLIENT; + private static NonNamespaceOperation> ODH_MONITORING_CLIENT; + + /** + * Get a client capable of working with {@link OpenDataHubOperatorProvisioner#ODH_DATA_SCIENCE_CLUSTER_CLIENT} custom resource. + * + * @return client for operations with {@link OpenDataHubOperatorProvisioner#ODH_DATA_SCIENCE_CLUSTER_CLIENT} custom resource + */ + public NonNamespaceOperation> dataScienceClusterClient() { + if (ODH_DATA_SCIENCE_CLUSTER_CLIENT == null) { + CustomResourceDefinition crd = customResourceDefinitionsClient() + .withName(ODH_DATA_SCIENCE_CLUSTER_CRD_NAME).get(); + if (crd == null) { + throw new RuntimeException(String.format("[%s] custom resource is not provided by [%s] operator.", + ODH_DATA_SCIENCE_CLUSTER_CRD_NAME, OPERATOR_ID)); + } + ODH_DATA_SCIENCE_CLUSTER_CLIENT = dataScienceClusterCustomResourcesClient( + CustomResourceDefinitionContext.fromCrd(crd)); + } + return ODH_DATA_SCIENCE_CLUSTER_CLIENT; + } + + /** + * Get a client capable of working with {@link OpenDataHubOperatorProvisioner#ODH_DSC_INITIALIZATION_CLIENT} custom resource. + * + * @return client for operations with {@link OpenDataHubOperatorProvisioner#ODH_DSC_INITIALIZATION_CLIENT} custom resource + */ + public NonNamespaceOperation> dscInitializationClient() { + if (ODH_DSC_INITIALIZATION_CLIENT == null) { + CustomResourceDefinition crd = customResourceDefinitionsClient() + .withName(ODH_DSC_INITIALIZATION_CRD_NAME).get(); + if (crd == null) { + throw new RuntimeException(String.format("[%s] custom resource is not provided by [%s] operator.", + ODH_DSC_INITIALIZATION_CRD_NAME, OPERATOR_ID)); + } + ODH_DSC_INITIALIZATION_CLIENT = dscInitializationCustomResourcesClient( + CustomResourceDefinitionContext.fromCrd(crd)); + } + return ODH_DSC_INITIALIZATION_CLIENT; + } + + /** + * Get a client capable of working with {@link OpenDataHubOperatorProvisioner#ODH_FEATURE_TRACKER_CLIENT} custom resource. + * + * @return client for operations with {@link OpenDataHubOperatorProvisioner#ODH_FEATURE_TRACKER_CLIENT} custom resource + */ + public NonNamespaceOperation> featureTrackerClient() { + if (ODH_FEATURE_TRACKER_CLIENT == null) { + CustomResourceDefinition crd = customResourceDefinitionsClient() + .withName(ODH_FEATURE_TRACKER_CRD_NAME).get(); + if (crd == null) { + throw new RuntimeException(String.format("[%s] custom resource is not provided by [%s] operator.", + ODH_FEATURE_TRACKER_CRD_NAME, OPERATOR_ID)); + } + ODH_FEATURE_TRACKER_CLIENT = featureTrackerCustomResourcesClient(CustomResourceDefinitionContext.fromCrd(crd)); + } + return ODH_FEATURE_TRACKER_CLIENT; + } + + /** + * Get a client capable of working with {@link OpenDataHubOperatorProvisioner#ODH_MONITORING_CLIENT} custom resource. + * + * @return client for operations with {@link OpenDataHubOperatorProvisioner#ODH_MONITORING_CLIENT} custom resource + */ + public NonNamespaceOperation> monitoringClient() { + if (ODH_MONITORING_CLIENT == null) { + CustomResourceDefinition crd = customResourceDefinitionsClient() + .withName(ODH_MONITORING_CRD_NAME).get(); + if (crd == null) { + throw new RuntimeException(String.format("[%s] custom resource is not provided by [%s] operator.", + ODH_MONITORING_CRD_NAME, OPERATOR_ID)); + } + ODH_MONITORING_CLIENT = monitoringCustomResourcesClient(CustomResourceDefinitionContext.fromCrd(crd)); + } + return ODH_MONITORING_CLIENT; + } + + /** + * Get a reference to an {@link DataScienceCluster} instance. + * Use get() to obtain the actual object, or null in case it does not exist on tested cluster. + * + * @return A concrete {@link Resource} instance representing the {@link DataScienceCluster} resource definition + */ + public Resource dataScienceCluster() { + return dataScienceClusterClient().withName(getApplication().getName()); + } + +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisionerFactory.java b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisionerFactory.java new file mode 100644 index 00000000..1cea10da --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/OpenDataHubOperatorProvisionerFactory.java @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2023 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.operator; + +import org.jboss.intersmash.application.Application; +import org.jboss.intersmash.application.openshift.OpenShiftApplication; +import org.jboss.intersmash.application.operator.OpenDataHubOperatorApplication; +import org.jboss.intersmash.provision.ProvisionerFactory; +import org.jboss.intersmash.provision.openshift.OpenDataHubOpenShiftOperatorProvisioner; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class OpenDataHubOperatorProvisionerFactory + implements ProvisionerFactory { + + @Override + public OpenDataHubOperatorProvisioner getProvisioner(Application application) { + if (OpenDataHubOperatorApplication.class.isAssignableFrom(application.getClass())) { + if (OpenShiftApplication.class.isAssignableFrom(application.getClass())) { + return new OpenDataHubOpenShiftOperatorProvisioner((OpenDataHubOperatorApplication) application); + } + throw new UnsupportedOperationException( + "Open Data Hub operator based provisioner requires OpenShift"); + } + return null; + } +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DSCInitializationList.java b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DSCInitializationList.java new file mode 100644 index 00000000..7fe84e09 --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DSCInitializationList.java @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.operator.model.odh; + +import io.fabric8.kubernetes.client.CustomResourceList; +import io.opendatahub.dscinitialization.v1.DSCInitialization; + +public class DSCInitializationList extends CustomResourceList { +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DataScienceClusterList.java b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DataScienceClusterList.java new file mode 100644 index 00000000..c153e6af --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/DataScienceClusterList.java @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.operator.model.odh; + +import io.fabric8.kubernetes.client.CustomResourceList; +import io.opendatahub.datasciencecluster.v1.DataScienceCluster; + +public class DataScienceClusterList extends CustomResourceList { +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/FeatureTrackerList.java b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/FeatureTrackerList.java new file mode 100644 index 00000000..bc000a77 --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/FeatureTrackerList.java @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.operator.model.odh; + +import io.fabric8.kubernetes.client.CustomResourceList; +import io.opendatahub.features.v1.FeatureTracker; + +public class FeatureTrackerList extends CustomResourceList { +} diff --git a/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/MonitoringList.java b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/MonitoringList.java new file mode 100644 index 00000000..707b2c78 --- /dev/null +++ b/provisioners/src/main/java/org/jboss/intersmash/provision/operator/model/odh/MonitoringList.java @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2024 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.intersmash.provision.operator.model.odh; + +import io.fabric8.kubernetes.client.CustomResourceList; +import io.opendatahub.platform.services.v1alpha1.Monitoring; + +public class MonitoringList extends CustomResourceList { +} diff --git a/provisioners/src/main/resources/META-INF/services/org.jboss.intersmash.provision.ProvisionerFactory b/provisioners/src/main/resources/META-INF/services/org.jboss.intersmash.provision.ProvisionerFactory index fb869619..f5351e30 100644 --- a/provisioners/src/main/resources/META-INF/services/org.jboss.intersmash.provision.ProvisionerFactory +++ b/provisioners/src/main/resources/META-INF/services/org.jboss.intersmash.provision.ProvisionerFactory @@ -16,5 +16,4 @@ org.jboss.intersmash.provision.openshift.Eap7LegacyS2iBuildTemplateProvisionerFa org.jboss.intersmash.provision.openshift.Eap7TemplateOpenShiftProvisionerFactory org.jboss.intersmash.provision.operator.HyperfoilOperatorProvisionerFactory org.jboss.intersmash.provision.openshift.auto.OpenShiftAutoProvisionerFactory - - +org.jboss.intersmash.provision.operator.OpenDataHubOperatorProvisionerFactory diff --git a/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenDataHubOpenShiftOperatorProvisionerTest.java b/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenDataHubOpenShiftOperatorProvisionerTest.java new file mode 100644 index 00000000..8a99f98b --- /dev/null +++ b/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenDataHubOpenShiftOperatorProvisionerTest.java @@ -0,0 +1,140 @@ +package org.jboss.intersmash.testsuite.provision.openshift; + +import java.io.IOException; + +import org.jboss.intersmash.application.operator.OpenDataHubOperatorApplication; +import org.jboss.intersmash.junit5.IntersmashExtension; +import org.jboss.intersmash.provision.olm.OperatorGroup; +import org.jboss.intersmash.provision.openshift.OpenDataHubOpenShiftOperatorProvisioner; +import org.jboss.intersmash.testsuite.openshift.ProjectCreationCapable; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.event.Level; + +import cz.xtf.core.config.OpenShiftConfig; +import cz.xtf.core.openshift.OpenShifts; +import cz.xtf.core.waiting.SimpleWaiter; +import cz.xtf.junit5.annotations.CleanBeforeAll; +import io.fabric8.kubernetes.api.model.DeletionPropagation; +import io.opendatahub.datasciencecluster.v1.DataScienceCluster; +import io.opendatahub.datasciencecluster.v1.DataScienceClusterBuilder; +import io.opendatahub.dscinitialization.v1.DSCInitialization; +import io.opendatahub.dscinitialization.v1.DSCInitializationBuilder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@CleanBeforeAll +public class OpenDataHubOpenShiftOperatorProvisionerTest implements ProjectCreationCapable { + private static final OpenDataHubOpenShiftOperatorProvisioner operatorProvisioner = initializeOperatorProvisioner(); + + private static OpenDataHubOpenShiftOperatorProvisioner initializeOperatorProvisioner() { + OpenDataHubOpenShiftOperatorProvisioner operatorProvisioner = new OpenDataHubOpenShiftOperatorProvisioner( + new OpenDataHubOperatorApplication() { + + private static final String APP_NAME = "example-odh"; + + @Override + public DataScienceCluster getDataScienceCluster() { + return new DataScienceClusterBuilder() + .withNewMetadata() + .withName(APP_NAME) + .endMetadata() + .build(); + } + + @Override + public DSCInitialization getDSCInitialization() { + return new DSCInitializationBuilder() + .withNewMetadata() + .withName(APP_NAME) + .endMetadata() + .withNewSpec() + .withApplicationsNamespace(OpenShifts.master().getNamespace()) + .endSpec() + .build(); + } + + @Override + public String getName() { + return APP_NAME; + } + }); + return operatorProvisioner; + } + + @BeforeAll + public static void createOperatorGroup() throws IOException { + operatorProvisioner.configure(); + IntersmashExtension.operatorCleanup(false, true); + // create operator group - this should be done by InteropExtension + OpenShifts.adminBinary().execute("apply", "-f", + new OperatorGroup(OpenShiftConfig.namespace()).save().getAbsolutePath()); + // clean any leftovers + operatorProvisioner.unsubscribe(); + // Let's skip subscribe operation here since we use regular deploy/undeploy where subscribe is called anyway. + } + + @AfterAll + public static void removeOperatorGroup() { + // clean any leftovers + operatorProvisioner.unsubscribe(); + OpenShifts.adminBinary().execute("delete", "operatorgroup", "--all"); + operatorProvisioner.dismiss(); + } + + @AfterEach + public void customResourcesCleanup() { + final String appName = operatorProvisioner.getApplication().getName(); + operatorProvisioner.dataScienceClusterClient().withName(appName).withPropagationPolicy(DeletionPropagation.FOREGROUND) + .delete(); + operatorProvisioner.dscInitializationClient().withName(appName).withPropagationPolicy(DeletionPropagation.FOREGROUND) + .delete(); + } + + /** + * This test case creates and validates minimal {@link DataScienceCluster} and {@link DSCInitialization} CRs + * + * This is not an integration test, the goal here is to assess that the created CRs are configured as per the + * model specification. + */ + @Test + public void testMinimalDataScienceCluster() { + operatorProvisioner.subscribe(); + try { + verifyDataScienceCluster(operatorProvisioner.getApplication().getDataScienceCluster()); + verifyDSCInitialization(operatorProvisioner.getApplication().getDSCInitialization()); + } finally { + operatorProvisioner.unsubscribe(); + } + } + + private void verifyDataScienceCluster(final DataScienceCluster dataScienceCluster) { + // create and verify that object exists + operatorProvisioner.dataScienceClusterClient().resource(dataScienceCluster).create(); + new SimpleWaiter(() -> operatorProvisioner.dataScienceClusterClient().list().getItems().size() == 1) + .level(Level.DEBUG) + .waitFor(); + Assertions.assertNotNull(operatorProvisioner.dataScienceCluster().get()); + // the DataScienceCluster spec gets populated on creation + Assertions.assertNotNull(operatorProvisioner.dataScienceCluster().get().getSpec()); + } + + private void verifyDSCInitialization(DSCInitialization dscInitialization) { + // create and verify that object exists + operatorProvisioner.dscInitializationClient().resource(dscInitialization).create(); + new SimpleWaiter(() -> operatorProvisioner.dscInitializationClient().list().getItems().size() == 1) + .level(Level.DEBUG) + .waitFor(); + Assertions.assertNotNull(operatorProvisioner.dataScienceCluster().get().getSpec()); + // the Monitoring spec gets populated on creation, so we verify 1st level resources individually + final DSCInitialization created = operatorProvisioner.dscInitializationClient() + .withName(dscInitialization.getMetadata().getName()).get(); + Assertions.assertEquals(dscInitialization.getSpec().getDevFlags(), created.getSpec().getDevFlags()); + Assertions.assertNotNull(created.getSpec().getMonitoring()); + Assertions.assertEquals(dscInitialization.getSpec().getServiceMesh(), created.getSpec().getServiceMesh()); + Assertions.assertEquals(dscInitialization.getSpec().getTrustedCABundle(), created.getSpec().getTrustedCABundle()); + } +} diff --git a/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenShiftProvisionerTestBase.java b/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenShiftProvisionerTestBase.java index f473f142..2d0a1dd0 100644 --- a/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenShiftProvisionerTestBase.java +++ b/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/OpenShiftProvisionerTestBase.java @@ -49,11 +49,11 @@ import org.jboss.intersmash.application.operator.InfinispanOperatorApplication; import org.jboss.intersmash.application.operator.KafkaOperatorApplication; import org.jboss.intersmash.application.operator.KeycloakOperatorApplication; +import org.jboss.intersmash.application.operator.OpenDataHubOperatorApplication; import org.jboss.intersmash.provision.operator.model.infinispan.infinispan.InfinispanBuilder; import org.jboss.intersmash.test.deployments.DeploymentsProvider; import org.jboss.intersmash.test.deployments.TestDeploymentProperties; import org.jboss.intersmash.test.deployments.WildflyDeploymentApplicationConfiguration; -import org.jboss.intersmash.testsuite.IntersmashTestsuiteProperties; import org.jboss.intersmash.testsuite.junit5.categories.OpenShiftTest; import org.jboss.intersmash.util.CommandLineBasedKeystoreGenerator; import org.jboss.intersmash.util.openshift.WildflyOpenShiftUtils; @@ -70,6 +70,10 @@ import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.EnvVarBuilder; import io.fabric8.kubernetes.api.model.Secret; +import io.opendatahub.datasciencecluster.v1.DataScienceCluster; +import io.opendatahub.datasciencecluster.v1.DataScienceClusterBuilder; +import io.opendatahub.dscinitialization.v1.DSCInitialization; +import io.opendatahub.dscinitialization.v1.DSCInitializationBuilder; import io.strimzi.api.kafka.model.AclOperation; import io.strimzi.api.kafka.model.AclResourcePatternType; import io.strimzi.api.kafka.model.AclRule; @@ -780,4 +784,37 @@ public String getName() { } }; } + + static OpenDataHubOperatorApplication getOpenDataHubOperatorApplication() { + return new OpenDataHubOperatorApplication() { + + private static final String APP_NAME = "example-odh"; + + @Override + public DataScienceCluster getDataScienceCluster() { + return new DataScienceClusterBuilder() + .withNewMetadata() + .withName(APP_NAME) + .endMetadata() + .build(); + } + + @Override + public DSCInitialization getDSCInitialization() { + return new DSCInitializationBuilder() + .withNewMetadata() + .withName(APP_NAME) + .endMetadata() + .withNewSpec() + .withApplicationsNamespace(OpenShifts.master().getNamespace()) + .endSpec() + .build(); + } + + @Override + public String getName() { + return APP_NAME; + } + }; + } } diff --git a/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/ProvisionerCleanupTestCase.java b/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/ProvisionerCleanupTestCase.java index 3ca1f853..50f994c8 100644 --- a/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/ProvisionerCleanupTestCase.java +++ b/testsuite/integration-tests/src/test/java/org/jboss/intersmash/testsuite/provision/openshift/ProvisionerCleanupTestCase.java @@ -62,7 +62,10 @@ private static Stream provisionerProvider() { , new PostgreSQLImageOpenShiftProvisioner( OpenShiftProvisionerTestBase.getPostgreSQLImageOpenShiftApplication()), new PostgreSQLTemplateOpenShiftProvisioner( - OpenShiftProvisionerTestBase.getPostgreSQLTemplateOpenShiftApplication())); + OpenShiftProvisionerTestBase.getPostgreSQLTemplateOpenShiftApplication()) + // ODH + , new OpenDataHubOpenShiftOperatorProvisioner( + OpenShiftProvisionerTestBase.getOpenDataHubOperatorApplication())); } else if (IntersmashTestsuiteProperties.isProductizedTestExecutionProfileEnabled()) { return Stream.of( // RHDG