diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml index 9e74f3a459378..8d52c23792857 100644 --- a/.github/workflows/docker-ingestion-smoke.yml +++ b/.github/workflows/docker-ingestion-smoke.yml @@ -3,8 +3,6 @@ on: release: types: [published] push: - branches: - - master paths: - "docker/datahub-ingestion-base/**" - "smoke-test/**" diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 31fead8a7ade6..de3e0ca93e6b7 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -58,7 +58,7 @@ jobs: echo "full_tag=$(get_tag)-full" >> $GITHUB_OUTPUT echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT echo "unique_slim_tag=$(get_unique_tag)-slim" >> $GITHUB_OUTPUT - echo "unique_full_tag=$(get_unique_tag)-full" >> $GITHUB_OUTPUT + echo "unique_full_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT - name: Check whether publishing enabled id: publish @@ -501,7 +501,7 @@ jobs: platforms: linux/amd64,linux/arm64/v8 - name: Compute DataHub Ingestion (Base-Slim) Tag id: tag - run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT datahub_ingestion_base_full_build: name: Build and Push DataHub Ingestion (Base-Full) Docker Image runs-on: ubuntu-latest @@ -567,13 +567,13 @@ jobs: datahub-ingestion: - 'docker/datahub-ingestion/**' - name: Build codegen - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }} + if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} run: ./gradlew :metadata-ingestion:codegen - name: Download Base Image uses: ishworkh/docker-image-artifact-download@v1 if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} with: - image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }} + image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} - name: Build and push Slim Image if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} uses: ./.github/actions/docker-custom-build-and-push @@ -583,7 +583,7 @@ jobs: ${{ env.DATAHUB_INGESTION_IMAGE }} build-args: | BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }} - DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }} + DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} APP_ENV=slim tags: ${{ needs.setup.outputs.slim_tag }} @@ -595,7 +595,7 @@ jobs: platforms: linux/amd64,linux/arm64/v8 - name: Compute Tag id: tag - run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT + run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT datahub_ingestion_slim_scan: permissions: contents: read # for actions/checkout to fetch code @@ -650,7 +650,7 @@ jobs: datahub-ingestion: - 'docker/datahub-ingestion/**' - name: Build codegen - if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }} + if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} run: ./gradlew :metadata-ingestion:codegen - name: Download Base Image uses: ishworkh/docker-image-artifact-download@v1 @@ -809,7 +809,7 @@ jobs: DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }} ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }} - ACTIONS_EXTRA_PACKAGES: 'acryl-datahub-actions[executor] acryl-datahub-actions' + ACTIONS_EXTRA_PACKAGES: 'acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5' ACTIONS_CONFIG: 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml' run: | ./smoke-test/run-quickstart.sh diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 1cbc65f2b6370..68432a4feb13d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -27,6 +27,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.10" + cache: pip - name: Install Python dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Build Docs diff --git a/README.md b/README.md index 951dcebad6498..79f85433fbc18 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ export const Logo = (props) => {
DataHub Logo
@@ -18,7 +18,7 @@ export const Logo = (props) => {

-DataHub +DataHub

@@ -156,7 +156,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to - [DataHub Blog](https://blog.datahubproject.io/) - [DataHub YouTube Channel](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) -- [Optum: Data Mesh via DataHub](https://optum.github.io/blog/2022/03/23/data-mesh-via-datahub/) +- [Optum: Data Mesh via DataHub](https://opensource.optum.com/blog/2022/03/23/data-mesh-via-datahub) - [Saxo Bank: Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) - [Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) - [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) diff --git a/build.gradle b/build.gradle index 1b6b82d51c2d4..0a94991b131aa 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ buildscript { ext.openTelemetryVersion = '1.18.0' ext.neo4jVersion = '4.4.9' ext.testContainersVersion = '1.17.4' - ext.elasticsearchVersion = '7.10.2' + ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x ext.jacksonVersion = '2.15.2' ext.jettyVersion = '9.4.46.v20220331' ext.playVersion = '2.8.18' @@ -90,15 +90,15 @@ project.ext.externalDependency = [ 'ebean': 'io.ebean:ebean:' + ebeanVersion, 'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion, 'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion, - 'elasticSearchRest': 'org.elasticsearch.client:elasticsearch-rest-high-level-client:' + elasticsearchVersion, - 'elasticSearchTransport': 'org.elasticsearch.client:transport:' + elasticsearchVersion, + 'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion, + 'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0', 'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1', 'graphqlJava': 'com.graphql-java:graphql-java:19.5', 'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1', 'gson': 'com.google.code.gson:gson:2.8.9', 'guice': 'com.google.inject:guice:4.2.3', 'guava': 'com.google.guava:guava:32.1.2-jre', - 'h2': 'com.h2database:h2:2.1.214', + 'h2': 'com.h2database:h2:2.2.224', 'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2', 'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2', "hadoopClient": "org.apache.hadoop:hadoop-client:$hadoop3Version", @@ -202,13 +202,15 @@ project.ext.externalDependency = [ 'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion", 'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.1.12', 'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41', - 'testng': 'org.testng:testng:7.3.0', + 'testngJava8': 'org.testng:testng:7.5.1', + 'testng': 'org.testng:testng:7.8.0', 'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion, 'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion, 'testContainersPostgresql':'org.testcontainers:postgresql:' + testContainersVersion, 'testContainersElasticsearch': 'org.testcontainers:elasticsearch:' + testContainersVersion, 'testContainersCassandra': 'org.testcontainers:cassandra:' + testContainersVersion, 'testContainersKafka': 'org.testcontainers:kafka:' + testContainersVersion, + 'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0', 'typesafeConfig':'com.typesafe:config:1.4.1', 'wiremock':'com.github.tomakehurst:wiremock:2.10.0', 'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14', @@ -257,7 +259,6 @@ subprojects { plugins.withType(JavaPlugin) { dependencies { - testImplementation externalDependency.testng constraints { implementation('io.netty:netty-all:4.1.86.Final') implementation('org.apache.commons:commons-compress:1.21') @@ -268,12 +269,6 @@ subprojects { } } - tasks.withType(Test) { - if (!name.startsWith('integ')) { - useTestNG() - } - } - checkstyle { configDirectory = file("${project.rootDir}/gradle/checkstyle") sourceSets = [ getProject().sourceSets.main, getProject().sourceSets.test ] @@ -292,6 +287,13 @@ subprojects { javaLauncher = javaToolchains.launcherFor { languageVersion = JavaLanguageVersion.of(11) } + // https://docs.gradle.org/current/userguide/performance.html + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + + if (project.configurations.getByName("testImplementation").getDependencies() + .any{ it.getName() == "testng" }) { + useTestNG() + } } afterEvaluate { diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index eb95078b1a640..98f3b82285eda 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -11,16 +11,19 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; -import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.metadata.restli.DefaultRestliClientFactory; import com.linkedin.parseq.retry.backoff.ExponentialBackoff; import com.linkedin.util.Configuration; +import config.ConfigurationProvider; import controllers.SsoCallbackController; + import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; + import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; @@ -34,6 +37,7 @@ import org.pac4j.play.store.PlayCookieSessionStore; import org.pac4j.play.store.PlaySessionStore; import org.pac4j.play.store.ShiroAesDataEncrypter; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; import play.Environment; import play.cache.SyncCacheApi; import utils.ConfigUtil; @@ -104,7 +108,7 @@ protected void configure() { bind(SsoCallbackController.class).toConstructor(SsoCallbackController.class.getConstructor( SsoManager.class, Authentication.class, - EntityClient.class, + SystemEntityClient.class, AuthServiceClient.class, com.typesafe.config.Config.class)); } catch (NoSuchMethodException | SecurityException e) { @@ -161,10 +165,19 @@ protected Authentication provideSystemAuthentication() { @Provides @Singleton - protected EntityClient provideEntityClient() { - return new RestliEntityClient(buildRestliClient(), + protected ConfigurationProvider provideConfigurationProvider() { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(ConfigurationProvider.class); + return context.getBean(ConfigurationProvider.class); + } + + @Provides + @Singleton + protected SystemEntityClient provideEntityClient(final Authentication systemAuthentication, + final ConfigurationProvider configurationProvider) { + return new SystemRestliEntityClient(buildRestliClient(), new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)), - _configs.getInt(ENTITY_CLIENT_NUM_RETRIES)); + _configs.getInt(ENTITY_CLIENT_NUM_RETRIES), systemAuthentication, + configurationProvider.getCache().getClient().getEntityClient()); } @Provides diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index 85139d1db0868..4bde0872fc082 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -13,7 +13,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.SetMode; import com.linkedin.entity.Entity; -import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.identity.CorpGroupInfo; import com.linkedin.identity.CorpUserEditableInfo; @@ -78,13 +78,14 @@ public class OidcCallbackLogic extends DefaultCallbackLogic { private final SsoManager _ssoManager; - private final EntityClient _entityClient; + private final SystemEntityClient _entityClient; private final Authentication _systemAuthentication; private final AuthServiceClient _authClient; private final CookieConfigs _cookieConfigs; public OidcCallbackLogic(final SsoManager ssoManager, final Authentication systemAuthentication, - final EntityClient entityClient, final AuthServiceClient authClient, final CookieConfigs cookieConfigs) { + final SystemEntityClient entityClient, final AuthServiceClient authClient, + final CookieConfigs cookieConfigs) { _ssoManager = ssoManager; _systemAuthentication = systemAuthentication; _entityClient = entityClient; diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java new file mode 100644 index 0000000000000..00a5472ec3476 --- /dev/null +++ b/datahub-frontend/app/config/ConfigurationProvider.java @@ -0,0 +1,27 @@ +package config; + +import com.linkedin.metadata.config.cache.CacheConfiguration; +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import lombok.Data; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.PropertySource; + + +/** + * Minimal sharing between metadata-service and frontend + * Initially for use of client caching configuration. + * Does not use the factories module to avoid transitive dependencies. + */ +@EnableConfigurationProperties +@PropertySource(value = "application.yml", factory = YamlPropertySourceFactory.class) +@ConfigurationProperties +@Data +public class ConfigurationProvider { + + /** + * Configuration for caching + */ + private CacheConfiguration cache; +} diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index 5a36d833deceb..7a4b5585cc21a 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -3,7 +3,7 @@ import auth.CookieConfigs; import client.AuthServiceClient; import com.datahub.authentication.Authentication; -import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.SystemEntityClient; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; @@ -40,7 +40,7 @@ public class SsoCallbackController extends CallbackController { public SsoCallbackController( @Nonnull SsoManager ssoManager, @Nonnull Authentication systemAuthentication, - @Nonnull EntityClient entityClient, + @Nonnull SystemEntityClient entityClient, @Nonnull AuthServiceClient authClient, @Nonnull com.typesafe.config.Config configs) { _ssoManager = ssoManager; @@ -79,7 +79,7 @@ public class SsoCallbackLogic implements CallbackLogic { private final OidcCallbackLogic _oidcCallbackLogic; SsoCallbackLogic(final SsoManager ssoManager, final Authentication systemAuthentication, - final EntityClient entityClient, final AuthServiceClient authClient, final CookieConfigs cookieConfigs) { + final SystemEntityClient entityClient, final AuthServiceClient authClient, final CookieConfigs cookieConfigs) { _oidcCallbackLogic = new OidcCallbackLogic(ssoManager, systemAuthentication, entityClient, authClient, cookieConfigs); } diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index e40f8e3eeb96d..daecba16cbf72 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -16,9 +16,6 @@ dependencies { implementation project(':datahub-web-react') constraints { - play(externalDependency.springCore) - play(externalDependency.springBeans) - play(externalDependency.springContext) play(externalDependency.jacksonDataBind) play('com.nimbusds:oauth2-oidc-sdk:8.36.2') play('com.nimbusds:nimbus-jose-jwt:8.18') @@ -35,7 +32,12 @@ dependencies { implementation project(":metadata-service:restli-client") implementation project(":metadata-service:auth-config") + implementation project(":metadata-service:configuration") + implementation externalDependency.springCore + implementation externalDependency.springBeans + implementation externalDependency.springContext + implementation externalDependency.springBootAutoconfigure implementation externalDependency.jettyJaas implementation externalDependency.graphqlJava implementation externalDependency.antlr4Runtime diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 89ba8f17b6aeb..fba0031351b58 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -24,6 +24,7 @@ dependencies { annotationProcessor externalDependency.lombok testImplementation externalDependency.mockito + testImplementation externalDependency.testng } graphqlCodegen { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 682710ad5d539..3ba0cc1f747e3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -81,6 +81,7 @@ import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.Owner; import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; +import com.linkedin.datahub.graphql.generated.ParentDomainsResult; import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue; import com.linkedin.datahub.graphql.generated.QueryEntity; import com.linkedin.datahub.graphql.generated.QuerySubject; @@ -124,6 +125,7 @@ import com.linkedin.datahub.graphql.resolvers.domain.DeleteDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.DomainEntitiesResolver; import com.linkedin.datahub.graphql.resolvers.domain.ListDomainsResolver; +import com.linkedin.datahub.graphql.resolvers.domain.ParentDomainsResolver; import com.linkedin.datahub.graphql.resolvers.domain.SetDomainResolver; import com.linkedin.datahub.graphql.resolvers.domain.UnsetDomainResolver; import com.linkedin.datahub.graphql.resolvers.embed.UpdateEmbedResolver; @@ -186,6 +188,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.BatchSetDomainResolver; import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateDeprecationResolver; import com.linkedin.datahub.graphql.resolvers.mutate.BatchUpdateSoftDeletedResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MoveDomainResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeBatchResolver; import com.linkedin.datahub.graphql.resolvers.mutate.MutableTypeResolver; import com.linkedin.datahub.graphql.resolvers.mutate.RemoveLinkResolver; @@ -299,6 +302,7 @@ import com.linkedin.datahub.graphql.types.test.TestType; import com.linkedin.datahub.graphql.types.view.DataHubViewType; import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.TestsConfiguration; @@ -361,6 +365,7 @@ public class GmsGraphQLEngine { private final EntityClient entityClient; + private final SystemEntityClient systemEntityClient; private final GraphClient graphClient; private final UsageClient usageClient; private final SiblingGraphService siblingGraphService; @@ -473,6 +478,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.graphQLPlugins.forEach(plugin -> plugin.init(args)); this.entityClient = args.entityClient; + this.systemEntityClient = args.systemEntityClient; this.graphClient = args.graphClient; this.usageClient = args.usageClient; this.siblingGraphService = args.siblingGraphService; @@ -944,6 +950,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("removeGroup", new RemoveGroupResolver(this.entityClient)) .dataFetcher("updateUserStatus", new UpdateUserStatusResolver(this.entityClient)) .dataFetcher("createDomain", new CreateDomainResolver(this.entityClient, this.entityService)) + .dataFetcher("moveDomain", new MoveDomainResolver(this.entityService, this.entityClient)) .dataFetcher("deleteDomain", new DeleteDomainResolver(entityClient)) .dataFetcher("setDomain", new SetDomainResolver(this.entityClient, this.entityService)) .dataFetcher("batchSetDomain", new BatchSetDomainResolver(this.entityService)) @@ -1029,6 +1036,13 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder .dataFetcher("entities", new EntityTypeBatchResolver(entityTypes, (env) -> ((BrowseResults) env.getSource()).getEntities())) ) + .type("ParentDomainsResult", typeWiring -> typeWiring + .dataFetcher("domains", new EntityTypeBatchResolver(entityTypes, + (env) -> { + final ParentDomainsResult result = env.getSource(); + return result != null ? result.getDomains() : null; + })) + ) .type("EntityRelationshipLegacy", typeWiring -> typeWiring .dataFetcher("entity", new EntityTypeResolver(entityTypes, (env) -> ((EntityRelationshipLegacy) env.getSource()).getEntity())) @@ -1675,8 +1689,8 @@ private void configureGlossaryRelationshipResolvers(final RuntimeWiring.Builder private void configureDomainResolvers(final RuntimeWiring.Builder builder) { builder.type("Domain", typeWiring -> typeWiring .dataFetcher("entities", new DomainEntitiesResolver(this.entityClient)) - .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient) - ) + .dataFetcher("parentDomains", new ParentDomainsResolver(this.entityClient)) + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) ); builder.type("DomainAssociation", typeWiring -> typeWiring .dataFetcher("domain", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java index cbcf42c4f93d9..157fb10ce7078 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java @@ -11,6 +11,7 @@ import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; import com.linkedin.datahub.graphql.featureflags.FeatureFlags; import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.TestsConfiguration; @@ -38,6 +39,7 @@ @Data public class GmsGraphQLEngineArgs { EntityClient entityClient; + SystemEntityClient systemEntityClient; GraphClient graphClient; UsageClient usageClient; AnalyticsService analyticsService; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java index 44b1779f8b006..4135a7b0da148 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java @@ -20,25 +20,25 @@ import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.Cardinality; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.opensearch.search.aggregations.bucket.filter.Filter; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.search.aggregations.bucket.terms.Terms; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.Cardinality; +import org.opensearch.search.builder.SearchSourceBuilder; @Slf4j diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java index db3e1dd03e419..44695c334855f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/exception/DataHubGraphQLErrorCode.java @@ -4,6 +4,7 @@ public enum DataHubGraphQLErrorCode { BAD_REQUEST(400), UNAUTHORIZED(403), NOT_FOUND(404), + CONFLICT(409), SERVER_ERROR(500); private final int _code; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index de3c217db01ec..4d6133f18df05 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -16,4 +16,5 @@ public class FeatureFlags { private PreProcessHooks preProcessHooks; private boolean showAcrylInfo = false; private boolean showAccessManagement = false; + private boolean nestedDomainsEnabled = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 09df985b19cf5..f6bc68caa0821 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -172,6 +172,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setShowBrowseV2(_featureFlags.isShowBrowseV2()) .setShowAcrylInfo(_featureFlags.isShowAcrylInfo()) .setShowAccessManagement(_featureFlags.isShowAccessManagement()) + .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java index f27fd604a746f..23be49c7e7140 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolver.java @@ -1,13 +1,16 @@ package com.linkedin.datahub.graphql.resolvers.dataset; +import com.datahub.authorization.ResourceSpec; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.DatasetStatsSummary; import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.usage.UsageClient; import com.linkedin.usage.UsageTimeRange; import com.linkedin.usage.UserUsageCounts; @@ -15,6 +18,7 @@ import graphql.schema.DataFetchingEnvironment; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -55,8 +59,15 @@ public CompletableFuture get(DataFetchingEnvironment enviro try { + if (!isAuthorized(resourceUrn, context)) { + log.debug("User {} is not authorized to view profile information for dataset {}", + context.getActorUrn(), + resourceUrn.toString()); + return null; + } + com.linkedin.usage.UsageQueryResult - usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), UsageTimeRange.MONTH, context.getAuthentication()); + usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), UsageTimeRange.MONTH); final DatasetStatsSummary result = new DatasetStatsSummary(); result.setQueryCountLast30Days(usageQueryResult.getAggregations().getTotalSqlQueries()); @@ -90,4 +101,10 @@ private CorpUser createPartialUser(final Urn userUrn) { result.setUrn(userUrn.toString()); return result; } + + private boolean isAuthorized(final Urn resourceUrn, final QueryContext context) { + return AuthorizationUtils.isAuthorized(context, + Optional.of(new ResourceSpec(resourceUrn.getEntityType(), resourceUrn.toString())), + PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java index 0476963b92e9a..20361830ad5a5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetUsageStatsResolver.java @@ -9,12 +9,10 @@ import com.linkedin.datahub.graphql.generated.UsageQueryResult; import com.linkedin.datahub.graphql.types.usage.UsageQueryResultMapper; import com.linkedin.metadata.authorization.PoliciesConfig; -import com.linkedin.r2.RemoteInvocationException; import com.linkedin.usage.UsageClient; import com.linkedin.usage.UsageTimeRange; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; -import java.net.URISyntaxException; import java.util.Optional; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; @@ -44,10 +42,10 @@ public CompletableFuture get(DataFetchingEnvironment environme } try { com.linkedin.usage.UsageQueryResult - usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), range, context.getAuthentication()); + usageQueryResult = usageClient.getUsageStats(resourceUrn.toString(), range); return UsageQueryResultMapper.map(usageQueryResult); - } catch (RemoteInvocationException | URISyntaxException e) { - throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", resourceUrn.toString()), e); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", resourceUrn), e); } }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java index 39aa1ea28da20..1930cdc1f8667 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java @@ -1,14 +1,18 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.CreateDomainInput; import com.linkedin.datahub.graphql.generated.OwnerEntityType; import com.linkedin.datahub.graphql.generated.OwnershipType; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.OwnerUtils; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; @@ -19,8 +23,11 @@ import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; + +import java.net.URISyntaxException; import java.util.UUID; import java.util.concurrent.CompletableFuture; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -45,9 +52,9 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws final QueryContext context = environment.getContext(); final CreateDomainInput input = bindArgument(environment.getArgument("input"), CreateDomainInput.class); + final Urn parentDomain = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; return CompletableFuture.supplyAsync(() -> { - if (!AuthorizationUtils.canCreateDomains(context)) { throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); } @@ -64,6 +71,17 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws throw new IllegalArgumentException("This Domain already exists!"); } + if (parentDomain != null && !_entityClient.exists(parentDomain, context.getAuthentication())) { + throw new IllegalArgumentException("Parent Domain does not exist!"); + } + + if (DomainUtils.hasNameConflict(input.getName(), parentDomain, context, _entityClient)) { + throw new DataHubGraphQLException( + String.format("\"%s\" already exists in this domain. Please pick a unique name.", input.getName()), + DataHubGraphQLErrorCode.CONFLICT + ); + } + // Create the MCP final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey(key, DOMAIN_ENTITY_NAME, DOMAIN_PROPERTIES_ASPECT_NAME, mapDomainProperties(input, context)); @@ -77,6 +95,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) throws } OwnerUtils.addCreatorAsOwner(context, domainUrn, OwnerEntityType.CORP_USER, ownershipType, _entityService); return domainUrn; + } catch (DataHubGraphQLException e) { + throw e; } catch (Exception e) { log.error("Failed to create Domain with id: {}, name: {}: {}", input.getId(), input.getName(), e.getMessage()); throw new RuntimeException(String.format("Failed to create Domain with id: %s, name: %s", input.getId(), input.getName()), e); @@ -89,6 +109,13 @@ private DomainProperties mapDomainProperties(final CreateDomainInput input, fina result.setName(input.getName()); result.setDescription(input.getDescription(), SetMode.IGNORE_NULL); result.setCreated(new AuditStamp().setActor(UrnUtils.getUrn(context.getActorUrn())).setTime(System.currentTimeMillis())); + if (input.getParentDomain() != null) { + try { + result.setParentDomain(Urn.createFromString(input.getParentDomain())); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to create Domain Urn from string: %s", input.getParentDomain()), e); + } + } return result; } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java index 60a03fcddcc4d..9ab90e8b4ff72 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolver.java @@ -4,6 +4,7 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.entity.client.EntityClient; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -32,6 +33,11 @@ public CompletableFuture get(final DataFetchingEnvironment environment) if (AuthorizationUtils.canManageDomains(context) || AuthorizationUtils.canDeleteEntity(urn, context)) { try { + // Make sure there are no child domains + if (DomainUtils.hasChildDomains(urn, context, _entityClient)) { + throw new RuntimeException(String.format("Cannot delete domain %s which has child domains", domainUrn)); + } + _entityClient.deleteEntity(urn, context.getAuthentication()); log.info(String.format("I've successfully deleted the entity %s with urn", domainUrn)); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 06bfa36fc3c14..0bf551c4683e6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -1,6 +1,5 @@ package com.linkedin.datahub.graphql.resolvers.domain; -import com.google.common.collect.ImmutableList; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.DomainEntitiesInput; @@ -67,17 +66,22 @@ public CompletableFuture get(final DataFetchingEnvironment enviro try { + final CriterionArray criteria = new CriterionArray(); final Criterion filterCriterion = new Criterion() .setField(DOMAINS_FIELD_NAME + ".keyword") .setCondition(Condition.EQUAL) .setValue(urn); + criteria.add(filterCriterion); + if (input.getFilters() != null) { + input.getFilters().forEach(filter -> { + criteria.add(new Criterion().setField(filter.getField()).setValue(filter.getValue())); + }); + } return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities( SEARCHABLE_ENTITY_TYPES.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()), query, - new Filter().setOr(new ConjunctiveCriterionArray( - new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of(filterCriterion))) - )), + new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(criteria))), start, count, null, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java index 6ed8639592d6e..3a751e502eb10 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java @@ -1,22 +1,24 @@ package com.linkedin.datahub.graphql.resolvers.domain; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; -import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; -import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.ListDomainsInput; import com.linkedin.datahub.graphql.generated.ListDomainsResult; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchResult; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -30,7 +32,6 @@ * Resolver used for listing all Domains defined within DataHub. Requires the MANAGE_DOMAINS platform privilege. */ public class ListDomainsResolver implements DataFetcher> { - private static final Integer DEFAULT_START = 0; private static final Integer DEFAULT_COUNT = 20; private static final String DEFAULT_QUERY = ""; @@ -48,18 +49,19 @@ public CompletableFuture get(final DataFetchingEnvironment en return CompletableFuture.supplyAsync(() -> { - if (AuthorizationUtils.canCreateDomains(context)) { final ListDomainsInput input = bindArgument(environment.getArgument("input"), ListDomainsInput.class); final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + final Urn parentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; + final Filter filter = DomainUtils.buildParentDomainFilter(parentDomainUrn); try { - // First, get all group Urns. + // First, get all domain Urns. final SearchResult gmsResult = _entityClient.search( Constants.DOMAIN_ENTITY_NAME, query, - null, + filter, new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING), start, count, @@ -78,8 +80,6 @@ public CompletableFuture get(final DataFetchingEnvironment en } catch (Exception e) { throw new RuntimeException("Failed to list domains", e); } - } - throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java new file mode 100644 index 0000000000000..dcaa7d61ed90c --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolver.java @@ -0,0 +1,59 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.ParentDomainsResult; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.metadata.Constants.DOMAIN_ENTITY_NAME; + +public class ParentDomainsResolver implements DataFetcher> { + + private final EntityClient _entityClient; + + public ParentDomainsResolver(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + final QueryContext context = environment.getContext(); + final Urn urn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn()); + final List parentDomains = new ArrayList<>(); + final Set visitedParentUrns = new HashSet<>(); + + if (!DOMAIN_ENTITY_NAME.equals(urn.getEntityType())) { + throw new IllegalArgumentException(String.format("Failed to resolve parents for entity type %s", urn)); + } + + return CompletableFuture.supplyAsync(() -> { + try { + Entity parentDomain = DomainUtils.getParentDomain(urn, context, _entityClient); + + while (parentDomain != null && !visitedParentUrns.contains(parentDomain.getUrn())) { + parentDomains.add(parentDomain); + visitedParentUrns.add(parentDomain.getUrn()); + parentDomain = DomainUtils.getParentDomain(Urn.createFromString(parentDomain.getUrn()), context, _entityClient); + } + + final ParentDomainsResult result = new ParentDomainsResult(); + result.setCount(parentDomains.size()); + result.setDomains(parentDomains); + return result; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to load parent domains for entity %s", urn), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java new file mode 100644 index 0000000000000..e5e3a5a0ee42e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/MoveDomainResolver.java @@ -0,0 +1,89 @@ +package com.linkedin.datahub.graphql.resolvers.mutate; + +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; +import com.linkedin.datahub.graphql.generated.MoveDomainInput; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.EntityUtils; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.CompletableFuture; + +@Slf4j +@RequiredArgsConstructor +public class MoveDomainResolver implements DataFetcher> { + + private final EntityService _entityService; + private final EntityClient _entityClient; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final MoveDomainInput input = ResolverUtils.bindArgument(environment.getArgument("input"), MoveDomainInput.class); + final QueryContext context = environment.getContext(); + final Urn resourceUrn = UrnUtils.getUrn(input.getResourceUrn()); + final Urn newParentDomainUrn = input.getParentDomain() != null ? UrnUtils.getUrn(input.getParentDomain()) : null; + + return CompletableFuture.supplyAsync(() -> { + if (!AuthorizationUtils.canManageDomains(context)) { + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + try { + if (!resourceUrn.getEntityType().equals(Constants.DOMAIN_ENTITY_NAME)) { + throw new IllegalArgumentException("Resource is not a domain."); + } + + DomainProperties properties = (DomainProperties) EntityUtils.getAspectFromEntity( + resourceUrn.toString(), + Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, + null + ); + + if (properties == null) { + throw new IllegalArgumentException("Domain properties do not exist."); + } + + if (newParentDomainUrn != null) { + if (!newParentDomainUrn.getEntityType().equals(Constants.DOMAIN_ENTITY_NAME)) { + throw new IllegalArgumentException("Parent entity is not a domain."); + } + if (!_entityService.exists(newParentDomainUrn)) { + throw new IllegalArgumentException("Parent entity does not exist."); + } + } + + if (DomainUtils.hasNameConflict(properties.getName(), newParentDomainUrn, context, _entityClient)) { + throw new DataHubGraphQLException( + String.format("\"%s\" already exists in the destination domain. Please pick a unique name.", properties.getName()), + DataHubGraphQLErrorCode.CONFLICT + ); + } + + properties.setParentDomain(newParentDomainUrn, SetMode.REMOVE_IF_NULL); + Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); + MutationUtils.persistAspect(resourceUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, properties, actor, _entityService); + return true; + } catch (DataHubGraphQLException e) { + throw e; + } catch (Exception e) { + log.error("Failed to move domain {} to parent {} : {}", input.getResourceUrn(), input.getParentDomain(), e.getMessage()); + throw new RuntimeException(String.format("Failed to move domain %s to %s", input.getResourceUrn(), input.getParentDomain()), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java index 225bee54142c4..0e316ac1296ee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateNameResolver.java @@ -6,8 +6,11 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLErrorCode; +import com.linkedin.datahub.graphql.exception.DataHubGraphQLException; import com.linkedin.datahub.graphql.generated.UpdateNameInput; import com.linkedin.datahub.graphql.resolvers.dataproduct.DataProductAuthorizationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.GlossaryUtils; import com.linkedin.dataproduct.DataProductProperties; import com.linkedin.domain.DomainProperties; @@ -124,14 +127,25 @@ private Boolean updateDomainName( try { DomainProperties domainProperties = (DomainProperties) EntityUtils.getAspectFromEntity( targetUrn.toString(), Constants.DOMAIN_PROPERTIES_ASPECT_NAME, _entityService, null); + if (domainProperties == null) { throw new IllegalArgumentException("Domain does not exist"); } + + if (DomainUtils.hasNameConflict(input.getName(), DomainUtils.getParentDomainSafely(domainProperties), context, _entityClient)) { + throw new DataHubGraphQLException( + String.format("\"%s\" already exists in this domain. Please pick a unique name.", input.getName()), + DataHubGraphQLErrorCode.CONFLICT + ); + } + domainProperties.setName(input.getName()); Urn actor = CorpuserUrn.createFromString(context.getActorUrn()); persistAspect(targetUrn, Constants.DOMAIN_PROPERTIES_ASPECT_NAME, domainProperties, actor, _entityService); return true; + } catch (DataHubGraphQLException e) { + throw e; } catch (Exception e) { throw new RuntimeException(String.format("Failed to perform update against input %s", input), e); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java index b57160be09d32..585fbdf53a2ba 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/DomainUtils.java @@ -5,29 +5,55 @@ import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.datahub.authorization.ConjunctivePrivilegeGroup; import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.ResourceRefInput; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.domain.DomainProperties; import com.linkedin.domain.Domains; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; + +import com.linkedin.r2.RemoteInvocationException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; + import lombok.extern.slf4j.Slf4j; import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.*; +import static com.linkedin.metadata.Constants.*; // TODO: Move to consuming from DomainService. @Slf4j public class DomainUtils { + private static final String PARENT_DOMAIN_INDEX_FIELD_NAME = "parentDomain.keyword"; + private static final String HAS_PARENT_DOMAIN_INDEX_FIELD_NAME = "hasParentDomain"; + private static final String NAME_INDEX_FIELD_NAME = "name"; + private static final ConjunctivePrivilegeGroup ALL_PRIVILEGES_GROUP = new ConjunctivePrivilegeGroup(ImmutableList.of( PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType() )); @@ -85,4 +111,200 @@ public static void validateDomain(Urn domainUrn, EntityService entityService) { throw new IllegalArgumentException(String.format("Failed to validate Domain with urn %s. Urn does not exist.", domainUrn)); } } + + private static List buildRootDomainCriteria() { + final List criteria = new ArrayList<>(); + + criteria.add( + new Criterion() + .setField(HAS_PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue("false") + .setCondition(Condition.EQUAL) + ); + criteria.add( + new Criterion() + .setField(HAS_PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue("") + .setCondition(Condition.IS_NULL) + ); + + return criteria; + } + + private static List buildParentDomainCriteria(@Nonnull final Urn parentDomainUrn) { + final List criteria = new ArrayList<>(); + + criteria.add( + new Criterion() + .setField(HAS_PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue("true") + .setCondition(Condition.EQUAL) + ); + criteria.add( + new Criterion() + .setField(PARENT_DOMAIN_INDEX_FIELD_NAME) + .setValue(parentDomainUrn.toString()) + .setCondition(Condition.EQUAL) + ); + + return criteria; + } + + private static Criterion buildNameCriterion(@Nonnull final String name) { + return new Criterion() + .setField(NAME_INDEX_FIELD_NAME) + .setValue(name) + .setCondition(Condition.EQUAL); + } + + /** + * Builds a filter that ORs together the root parent criterion / ANDs together the parent domain criterion. + * The reason for the OR on root is elastic can have a null|false value to represent an root domain in the index. + * @param name an optional name to AND in to each condition of the filter + * @param parentDomainUrn the parent domain (null means root). + * @return the Filter + */ + public static Filter buildNameAndParentDomainFilter(@Nullable final String name, @Nullable final Urn parentDomainUrn) { + if (parentDomainUrn == null) { + return new Filter().setOr( + new ConjunctiveCriterionArray( + buildRootDomainCriteria().stream().map(parentCriterion -> { + final CriterionArray array = new CriterionArray(parentCriterion); + if (name != null) { + array.add(buildNameCriterion(name)); + } + return new ConjunctiveCriterion().setAnd(array); + }).collect(Collectors.toList()) + ) + ); + } + + final CriterionArray andArray = new CriterionArray(buildParentDomainCriteria(parentDomainUrn)); + if (name != null) { + andArray.add(buildNameCriterion(name)); + } + return new Filter().setOr( + new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd(andArray) + ) + ); + } + + public static Filter buildParentDomainFilter(@Nullable final Urn parentDomainUrn) { + return buildNameAndParentDomainFilter(null, parentDomainUrn); + } + + /** + * Check if a domain has any child domains + * @param domainUrn the URN of the domain to check + * @param context query context (includes authorization context to authorize the request) + * @param entityClient client used to perform the check + * @return true if the domain has any child domains, false if it does not + */ + public static boolean hasChildDomains( + @Nonnull final Urn domainUrn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) throws RemoteInvocationException { + Filter parentDomainFilter = buildParentDomainFilter(domainUrn); + // Search for entities matching parent domain + // Limit count to 1 for existence check + final SearchResult searchResult = entityClient.filter( + DOMAIN_ENTITY_NAME, + parentDomainFilter, + null, + 0, + 1, + context.getAuthentication()); + return (searchResult.getNumEntities() > 0); + } + + private static Map getDomainsByNameAndParent( + @Nonnull final String name, + @Nullable final Urn parentDomainUrn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) { + try { + final Filter filter = buildNameAndParentDomainFilter(name, parentDomainUrn); + + final SearchResult searchResult = entityClient.filter( + DOMAIN_ENTITY_NAME, + filter, + null, + 0, + 1000, + context.getAuthentication()); + + final Set domainUrns = searchResult.getEntities() + .stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toSet()); + + return entityClient.batchGetV2( + DOMAIN_ENTITY_NAME, + domainUrns, + Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), + context.getAuthentication()); + } catch (Exception e) { + throw new RuntimeException("Failed fetching Domains by name and parent", e); + } + } + + public static boolean hasNameConflict( + @Nonnull final String name, + @Nullable final Urn parentDomainUrn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) { + final Map entities = getDomainsByNameAndParent(name, parentDomainUrn, context, entityClient); + + // Even though we searched by name, do one more pass to check the name is unique + return entities.values().stream().anyMatch(entityResponse -> { + if (entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { + DataMap dataMap = entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data(); + DomainProperties domainProperties = new DomainProperties(dataMap); + return (domainProperties.hasName() && domainProperties.getName().equals(name)); + } + return false; + }); + } + + @Nullable + public static Entity getParentDomain( + @Nonnull final Urn urn, + @Nonnull final QueryContext context, + @Nonnull final EntityClient entityClient + ) { + try { + final EntityResponse entityResponse = entityClient.getV2( + DOMAIN_ENTITY_NAME, + urn, + Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME), + context.getAuthentication() + ); + + if (entityResponse != null && entityResponse.getAspects().containsKey(DOMAIN_PROPERTIES_ASPECT_NAME)) { + final DomainProperties properties = new DomainProperties(entityResponse.getAspects().get(DOMAIN_PROPERTIES_ASPECT_NAME).getValue().data()); + final Urn parentDomainUrn = getParentDomainSafely(properties); + return parentDomainUrn != null ? UrnToEntityMapper.map(parentDomainUrn) : null; + } + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to retrieve parent domain for entity %s", urn), e); + } + + return null; + } + + /** + * Get a parent domain only if hasParentDomain was set. There is strange elastic behavior where moving a domain + * to the root leaves the parentDomain field set but makes hasParentDomain false. This helper makes sure that queries + * to elastic where hasParentDomain=false and parentDomain=value only gives us the parentDomain if hasParentDomain=true. + * @param properties the domain properties aspect + * @return the parentDomain or null + */ + @Nullable + public static Urn getParentDomainSafely(@Nonnull final DomainProperties properties) { + return properties.hasParentDomain() ? properties.getParentDomain() : null; + } } \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java index 9cb6840067e7b..254b43ecb96cc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java @@ -50,7 +50,8 @@ public DataProduct apply(@Nonnull final EntityResponse entityResponse) { EnvelopedAspectMap aspectMap = entityResponse.getAspects(); MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); - mappingHelper.mapToResult(DATA_PRODUCT_PROPERTIES_ASPECT_NAME, this::mapDataProductProperties); + mappingHelper.mapToResult(DATA_PRODUCT_PROPERTIES_ASPECT_NAME, (dataProduct, dataMap) -> + mapDataProductProperties(dataProduct, dataMap, entityUrn)); mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (dataProduct, dataMap) -> dataProduct.setTags(GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn))); mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (dataProduct, dataMap) -> @@ -65,11 +66,12 @@ public DataProduct apply(@Nonnull final EntityResponse entityResponse) { return result; } - private void mapDataProductProperties(@Nonnull DataProduct dataProduct, @Nonnull DataMap dataMap) { + private void mapDataProductProperties(@Nonnull DataProduct dataProduct, @Nonnull DataMap dataMap, @Nonnull Urn urn) { DataProductProperties dataProductProperties = new DataProductProperties(dataMap); com.linkedin.datahub.graphql.generated.DataProductProperties properties = new com.linkedin.datahub.graphql.generated.DataProductProperties(); - properties.setName(dataProductProperties.getName()); + final String name = dataProductProperties.hasName() ? dataProductProperties.getName() : urn.getId(); + properties.setName(name); properties.setDescription(dataProductProperties.getDescription()); if (dataProductProperties.hasExternalUrl()) { properties.setExternalUrl(dataProductProperties.getExternalUrl().toString()); diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index a5057bcf644da..075a3b0fac43b 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -441,10 +441,17 @@ type FeatureFlagsConfig { Whether we should show CTAs in the UI related to moving to Managed DataHub by Acryl. """ showAcrylInfo: Boolean! + """ Whether we should show AccessManagement tab in the datahub UI. """ showAccessManagement: Boolean! + + """ + Enables the nested Domains feature that allows users to have sub-Domains. + If this is off, Domains appear "flat" again. + """ + nestedDomainsEnabled: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 044c405942a3c..39f86948c77c4 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -434,6 +434,11 @@ type Mutation { """ createDomain(input: CreateDomainInput!): String + """ + Moves a domain to be parented under another domain. + """ + moveDomain(input: MoveDomainInput!): Boolean + """ Delete a Domain """ @@ -7735,6 +7740,21 @@ input UpdateParentNodeInput { resourceUrn: String! } +""" +Input for updating the parent domain of a domain. +""" +input MoveDomainInput { + """ + The new parent domain urn. If parentDomain is null, this will remove the parent from this entity + """ + parentDomain: String + + """ + The primary key of the resource to update the parent domain for + """ + resourceUrn: String! +} + """ Input for updating the name of an entity """ @@ -9584,15 +9604,31 @@ type Domain implements Entity { """ entities(input: DomainEntitiesInput): SearchResults + """ + Recursively get the lineage of parent domains for this entity + """ + parentDomains: ParentDomainsResult + """ Edges extending from this entity """ relationships(input: RelationshipsInput!): EntityRelationshipsResult } +""" +All of the parent domains starting from a single Domain through all of its ancestors +""" +type ParentDomainsResult { + """ + The number of parent domains bubbling up for this entity + """ + count: Int! - - + """ + A list of parent domains in order from direct parent, to parent's parent etc. If there are no parents, return an empty list + """ + domains: [Entity!]! +} """ Properties about a domain @@ -9652,6 +9688,11 @@ input CreateDomainInput { Optional description for the Domain """ description: String + + """ + Optional parent domain urn for the domain + """ + parentDomain: String } """ @@ -9672,6 +9713,11 @@ input ListDomainsInput { Optional search query """ query: String + + """ + Optional parent domain + """ + parentDomain: String } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java index c391615db9268..7cd548a4790ba 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ResolverUtilsTest.java @@ -15,7 +15,6 @@ import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import graphql.schema.DataFetchingEnvironment; -import junit.framework.TestCase; import org.testng.annotations.Test; import org.mockito.Mockito; @@ -24,9 +23,10 @@ import java.util.stream.Collectors; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; +import static org.testng.AssertJUnit.assertEquals; -public class ResolverUtilsTest extends TestCase { +public class ResolverUtilsTest { @Test public void testCriterionFromFilter() throws Exception { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java index 8c23335b7e9d3..52d06f73dcfab 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql.resolvers.auth; +import com.datahub.authentication.Authentication; import com.google.common.collect.ImmutableList; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.TestUtils; @@ -8,17 +9,21 @@ import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; import graphql.schema.DataFetchingEnvironment; import java.util.Collections; -import junit.framework.TestCase; import org.mockito.Mockito; +import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; -public class ListAccessTokensResolverTest extends TestCase { +public class ListAccessTokensResolverTest { -// @Test + @Test public void testGetSuccess() throws Exception { final DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); final QueryContext mockAllowContext = TestUtils.getMockAllowContext(); @@ -36,14 +41,17 @@ public void testGetSuccess() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input); final EntityClient mockClient = Mockito.mock(EntityClient.class); - Mockito.when(mockClient.filter( - Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME), - Mockito.eq(buildFilter(filters, Collections.emptyList())), - Mockito.notNull(), - Mockito.eq(input.getStart()), - Mockito.eq(input.getCount()), - Mockito.eq(getAuthentication(mockEnv)))) - .thenReturn(null); + final Authentication testAuth = getAuthentication(mockEnv); + Mockito.when(mockClient.search( + Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME), + Mockito.eq(""), + Mockito.eq(buildFilter(filters, Collections.emptyList())), + Mockito.any(SortCriterion.class), + Mockito.eq(input.getStart()), + Mockito.eq(input.getCount()), + Mockito.eq(testAuth), + Mockito.any(SearchFlags.class))) + .thenReturn(new SearchResult().setFrom(0).setNumEntities(0).setPageSize(0).setEntities(new SearchEntityArray())); final ListAccessTokensResolver resolver = new ListAccessTokensResolver(mockClient); final ListAccessTokenResult listAccessTokenResult = resolver.get(mockEnv).get(); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java index 163628c1bc590..6a9617ea41b44 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryTest.java @@ -117,8 +117,7 @@ public void testGetException() throws Exception { UsageClient mockClient = Mockito.mock(UsageClient.class); Mockito.when(mockClient.getUsageStats( Mockito.eq(TEST_DASHBOARD_URN), - Mockito.eq(UsageTimeRange.MONTH), - Mockito.any(Authentication.class) + Mockito.eq(UsageTimeRange.MONTH) )).thenThrow(RuntimeException.class); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java index bd3edf65bf7ad..013e23b779c51 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/dataset/DatasetStatsSummaryResolverTest.java @@ -1,6 +1,8 @@ package com.linkedin.datahub.graphql.resolvers.dataset; import com.datahub.authentication.Authentication; +import com.datahub.authorization.AuthorizationResult; +import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; @@ -53,13 +55,18 @@ public void testGetSuccess() throws Exception { UsageClient mockClient = Mockito.mock(UsageClient.class); Mockito.when(mockClient.getUsageStats( Mockito.eq(TEST_DATASET_URN), - Mockito.eq(UsageTimeRange.MONTH), - Mockito.any(Authentication.class) + Mockito.eq(UsageTimeRange.MONTH) )).thenReturn(testResult); // Execute resolver DatasetStatsSummaryResolver resolver = new DatasetStatsSummaryResolver(mockClient); QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); + Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); + AuthorizationResult mockAuthorizerResult = Mockito.mock(AuthorizationResult.class); + Mockito.when(mockAuthorizerResult.getType()).thenReturn(AuthorizationResult.Type.ALLOW); + Mockito.when(mockAuthorizer.authorize(Mockito.any())).thenReturn(mockAuthorizerResult); + Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getSource()).thenReturn(TEST_SOURCE); @@ -79,8 +86,7 @@ public void testGetSuccess() throws Exception { newResult.setAggregations(new UsageQueryResultAggregations()); Mockito.when(mockClient.getUsageStats( Mockito.eq(TEST_DATASET_URN), - Mockito.eq(UsageTimeRange.MONTH), - Mockito.any(Authentication.class) + Mockito.eq(UsageTimeRange.MONTH) )).thenReturn(newResult); // Then verify that the new result is _not_ returned (cache hit) @@ -116,8 +122,7 @@ public void testGetException() throws Exception { UsageClient mockClient = Mockito.mock(UsageClient.class); Mockito.when(mockClient.getUsageStats( Mockito.eq(TEST_DATASET_URN), - Mockito.eq(UsageTimeRange.MONTH), - Mockito.any(Authentication.class) + Mockito.eq(UsageTimeRange.MONTH) )).thenThrow(RuntimeException.class); // Execute resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java index 8c19f1dc3eb34..560a3865ce9e1 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java @@ -6,35 +6,57 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateDomainInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.DomainKey; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; + +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CompletionException; import org.mockito.Mockito; import org.testng.annotations.Test; import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.DOMAIN_PROPERTIES_ASPECT_NAME; import static org.testng.Assert.*; public class CreateDomainResolverTest { + private static final Urn TEST_DOMAIN_URN = Urn.createFromTuple("domain", "test-id"); + private static final Urn TEST_PARENT_DOMAIN_URN = Urn.createFromTuple("domain", "test-parent-id"); + private static final CreateDomainInput TEST_INPUT = new CreateDomainInput( "test-id", "test-name", - "test-description" + "test-description", + TEST_PARENT_DOMAIN_URN.toString() + ); + + private static final CreateDomainInput TEST_INPUT_NO_PARENT_DOMAIN = new CreateDomainInput( + "test-id", + "test-name", + "test-description", + null ); + private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test"); - private static final String TEST_ENTITY_URN = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)"; - private static final String TEST_TAG_1_URN = "urn:li:tag:test-id-1"; - private static final String TEST_TAG_2_URN = "urn:li:tag:test-id-2"; + @Test public void testGetSuccess() throws Exception { @@ -43,12 +65,31 @@ public void testGetSuccess() throws Exception { EntityService mockService = getMockEntityService(); CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_PARENT_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(true); + // Execute resolver QueryContext mockContext = getMockAllowContext(); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), TEST_PARENT_DOMAIN_URN)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + resolver.get(mockEnv).get(); final DomainKey key = new DomainKey(); @@ -60,6 +101,7 @@ public void testGetSuccess() throws Exception { props.setDescription("test-description"); props.setName("test-name"); props.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)); + props.setParentDomain(TEST_PARENT_DOMAIN_URN); proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); proposal.setAspect(GenericRecordUtils.serializeAspect(props)); proposal.setChangeType(ChangeType.UPSERT); @@ -72,6 +114,133 @@ public void testGetSuccess() throws Exception { ); } + @Test + public void testGetSuccessNoParentDomain() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NO_PARENT_DOMAIN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), null)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + + resolver.get(mockEnv).get(); + + final DomainKey key = new DomainKey(); + key.setId("test-id"); + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key)); + proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME); + DomainProperties props = new DomainProperties(); + props.setDescription("test-description"); + props.setName("test-name"); + props.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)); + proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(props)); + proposal.setChangeType(ChangeType.UPSERT); + + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.argThat(new CreateDomainProposalMatcher(proposal)), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + } + + @Test + public void testGetInvalidParent() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_PARENT_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetNameConflict() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + EntityService mockService = Mockito.mock(EntityService.class); + CreateDomainResolver resolver = new CreateDomainResolver(mockClient, mockService); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(false); + + Mockito.when(mockClient.exists( + Mockito.eq(TEST_PARENT_DOMAIN_URN), + Mockito.any(Authentication.class) + )).thenReturn(true); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(TEST_INPUT.getName(), TEST_PARENT_DOMAIN_URN)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities( + new SearchEntityArray(new SearchEntity().setEntity(TEST_DOMAIN_URN)) + )); + + DomainProperties domainProperties = new DomainProperties(); + domainProperties.setDescription(TEST_INPUT.getDescription()); + domainProperties.setName(TEST_INPUT.getName()); + domainProperties.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L)); + domainProperties.setParentDomain(TEST_PARENT_DOMAIN_URN); + + EntityResponse entityResponse = new EntityResponse(); + EnvelopedAspectMap envelopedAspectMap = new EnvelopedAspectMap(); + envelopedAspectMap.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(domainProperties.data()))); + entityResponse.setAspects(envelopedAspectMap); + + Map entityResponseMap = new HashMap<>(); + entityResponseMap.put(TEST_DOMAIN_URN, entityResponse); + + Mockito.when(mockClient.batchGetV2( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.any(), + Mockito.any(), + Mockito.any(Authentication.class) + )).thenReturn(entityResponseMap); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + @Test public void testGetUnauthorized() throws Exception { // Create resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java index 1c450b0e85424..9bcdbe6d2a0e0 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DeleteDomainResolverTest.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.search.SearchResult; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; import org.mockito.Mockito; @@ -28,6 +29,10 @@ public void testGetSuccess() throws Exception { Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + // Domain has 0 child domains + Mockito.when(mockClient.filter(Mockito.eq("domain"), Mockito.any(), Mockito.any(), Mockito.eq(0), Mockito.eq(1), Mockito.any())) + .thenReturn(new SearchResult().setNumEntities(0)); + assertTrue(resolver.get(mockEnv).get()); Mockito.verify(mockClient, Mockito.times(1)).deleteEntity( @@ -36,6 +41,28 @@ public void testGetSuccess() throws Exception { ); } + @Test + public void testDeleteWithChildDomains() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + DeleteDomainResolver resolver = new DeleteDomainResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Domain has child domains + Mockito.when(mockClient.filter(Mockito.eq("domain"), Mockito.any(), Mockito.any(), Mockito.eq(0), Mockito.eq(1), Mockito.any())) + .thenReturn(new SearchResult().setNumEntities(1)); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + Mockito.verify(mockClient, Mockito.times(0)).deleteEntity( + Mockito.any(), + Mockito.any(Authentication.class)); + } + @Test public void testGetUnauthorized() throws Exception { // Create resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java index c143f3480fcff..bd8a8f98de497 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.ListDomainsInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.SearchFlags; @@ -28,9 +29,14 @@ public class ListDomainsResolverTest { private static final Urn TEST_DOMAIN_URN = Urn.createFromTuple("domain", "test-id"); + private static final Urn TEST_PARENT_DOMAIN_URN = Urn.createFromTuple("domain", "test-parent-id"); private static final ListDomainsInput TEST_INPUT = new ListDomainsInput( - 0, 20, null + 0, 20, null, TEST_PARENT_DOMAIN_URN.toString() + ); + + private static final ListDomainsInput TEST_INPUT_NO_PARENT_DOMAIN = new ListDomainsInput( + 0, 20, null, null ); @Test @@ -41,7 +47,7 @@ public void testGetSuccess() throws Exception { Mockito.when(mockClient.search( Mockito.eq(Constants.DOMAIN_ENTITY_NAME), Mockito.eq(""), - Mockito.eq(null), + Mockito.eq(DomainUtils.buildParentDomainFilter(TEST_PARENT_DOMAIN_URN)), Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), Mockito.eq(0), Mockito.eq(20), @@ -71,6 +77,44 @@ public void testGetSuccess() throws Exception { assertEquals(resolver.get(mockEnv).get().getDomains().get(0).getUrn(), TEST_DOMAIN_URN.toString()); } + @Test + public void testGetSuccessNoParentDomain() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.search( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(""), + Mockito.eq(DomainUtils.buildParentDomainFilter(null)), + Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)), + Mockito.eq(0), + Mockito.eq(20), + Mockito.any(Authentication.class), + Mockito.eq(new SearchFlags().setFulltext(true)) + )).thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities(new SearchEntityArray(ImmutableSet.of(new SearchEntity().setEntity(TEST_DOMAIN_URN)))) + ); + + ListDomainsResolver resolver = new ListDomainsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_NO_PARENT_DOMAIN); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Data Assertions + assertEquals((int) resolver.get(mockEnv).get().getStart(), 0); + assertEquals((int) resolver.get(mockEnv).get().getCount(), 1); + assertEquals((int) resolver.get(mockEnv).get().getTotal(), 1); + assertEquals(resolver.get(mockEnv).get().getDomains().size(), 1); + assertEquals(resolver.get(mockEnv).get().getDomains().get(0).getUrn(), TEST_DOMAIN_URN.toString()); + } + @Test public void testGetUnauthorized() throws Exception { // Create resolver diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java new file mode 100644 index 0000000000000..4059c180b0eb0 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/MoveDomainResolverTest.java @@ -0,0 +1,140 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.MoveDomainInput; +import com.linkedin.datahub.graphql.resolvers.mutate.MoveDomainResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; +import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.mxe.MetadataChangeProposal; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.util.concurrent.CompletionException; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +public class MoveDomainResolverTest { + + private static final String CONTAINER_URN = "urn:li:container:00005397daf94708a8822b8106cfd451"; + private static final String PARENT_DOMAIN_URN = "urn:li:domain:00005397daf94708a8822b8106cfd451"; + private static final String DOMAIN_URN = "urn:li:domain:11115397daf94708a8822b8106cfd451"; + private static final MoveDomainInput INPUT = new MoveDomainInput(PARENT_DOMAIN_URN, DOMAIN_URN); + private static final MoveDomainInput INVALID_INPUT = new MoveDomainInput(CONTAINER_URN, DOMAIN_URN); + private static final CorpuserUrn TEST_ACTOR_URN = new CorpuserUrn("test"); + + private MetadataChangeProposal setupTests(DataFetchingEnvironment mockEnv, EntityService mockService, EntityClient mockClient) throws Exception { + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + final String name = "test name"; + Mockito.when(mockService.getAspect( + Urn.createFromString(DOMAIN_URN), + Constants.DOMAIN_PROPERTIES_ASPECT_NAME, + 0)) + .thenReturn(new DomainProperties().setName(name)); + + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(name, Urn.createFromString(PARENT_DOMAIN_URN))), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + + DomainProperties properties = new DomainProperties(); + properties.setName(name); + properties.setParentDomain(Urn.createFromString(PARENT_DOMAIN_URN)); + return MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(DOMAIN_URN), + DOMAIN_PROPERTIES_ASPECT_NAME, properties); + } + + @Test + public void testGetSuccess() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + setupTests(mockEnv, mockService, mockClient); + + assertTrue(resolver.get(mockEnv).get()); + Mockito.verify(mockService, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + ); + } + + @Test + public void testGetFailureEntityDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT); + + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getActorUrn()).thenReturn(TEST_ACTOR_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockService.getAspect( + Urn.createFromString(DOMAIN_URN), + DOMAIN_PROPERTIES_ASPECT_NAME, + 0)) + .thenReturn(null); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + verifyNoIngestProposal(mockService); + } + + @Test + public void testGetFailureParentDoesNotExist() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(false); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INPUT); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + setupTests(mockEnv, mockService, mockClient); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + verifyNoIngestProposal(mockService); + } + + @Test + public void testGetFailureParentIsNotDomain() throws Exception { + EntityService mockService = Mockito.mock(EntityService.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockService.exists(Urn.createFromString(PARENT_DOMAIN_URN))).thenReturn(true); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument("input")).thenReturn(INVALID_INPUT); + + MoveDomainResolver resolver = new MoveDomainResolver(mockService, mockClient); + setupTests(mockEnv, mockService, mockClient); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + verifyNoIngestProposal(mockService); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java new file mode 100644 index 0000000000000..7bd7c3afac001 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ParentDomainsResolverTest.java @@ -0,0 +1,95 @@ +package com.linkedin.datahub.graphql.resolvers.domain; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Domain; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.ParentDomainsResult; +import com.linkedin.domain.DomainProperties; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import graphql.schema.DataFetchingEnvironment; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.assertEquals; + +public class ParentDomainsResolverTest { + @Test + public void testGetSuccessForDomain() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Urn domainUrn = Urn.createFromString("urn:li:domain:00005397daf94708a8822b8106cfd451"); + Domain domainEntity = new Domain(); + domainEntity.setUrn(domainUrn.toString()); + domainEntity.setType(EntityType.DOMAIN); + Mockito.when(mockEnv.getSource()).thenReturn(domainEntity); + + final DomainProperties parentDomain1 = new DomainProperties().setParentDomain(Urn.createFromString( + "urn:li:domain:11115397daf94708a8822b8106cfd451") + ).setName("test def"); + final DomainProperties parentDomain2 = new DomainProperties().setParentDomain(Urn.createFromString( + "urn:li:domain:22225397daf94708a8822b8106cfd451") + ).setName("test def 2"); + + Map domainAspects = new HashMap<>(); + domainAspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(parentDomain1.data()))); + + Map parentDomain1Aspects = new HashMap<>(); + parentDomain1Aspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( + new DomainProperties().setName("domain parent 1").setParentDomain(parentDomain2.getParentDomain()).data() + ))); + + Map parentDomain2Aspects = new HashMap<>(); + parentDomain2Aspects.put(DOMAIN_PROPERTIES_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect( + new DomainProperties().setName("domain parent 2").data() + ))); + + Mockito.when(mockClient.getV2( + Mockito.eq(domainUrn.getEntityType()), + Mockito.eq(domainUrn), + Mockito.eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(domainAspects))); + + Mockito.when(mockClient.getV2( + Mockito.eq(parentDomain1.getParentDomain().getEntityType()), + Mockito.eq(parentDomain1.getParentDomain()), + Mockito.eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentDomain1Aspects))); + + Mockito.when(mockClient.getV2( + Mockito.eq(parentDomain2.getParentDomain().getEntityType()), + Mockito.eq(parentDomain2.getParentDomain()), + Mockito.eq(Collections.singleton(DOMAIN_PROPERTIES_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn(new EntityResponse().setAspects(new EnvelopedAspectMap(parentDomain2Aspects))); + + ParentDomainsResolver resolver = new ParentDomainsResolver(mockClient); + ParentDomainsResult result = resolver.get(mockEnv).get(); + + Mockito.verify(mockClient, Mockito.times(3)).getV2( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any() + ); + assertEquals(result.getCount(), 2); + assertEquals(result.getDomains().get(0).getUrn(), parentDomain1.getParentDomain().toString()); + assertEquals(result.getDomains().get(1).getUrn(), parentDomain2.getParentDomain().toString()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java index 064e2dd3bd59b..eee9cfbae8fcb 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/glossary/UpdateNameResolverTest.java @@ -8,12 +8,15 @@ import com.linkedin.datahub.graphql.generated.UpdateNameInput; import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; +import com.linkedin.datahub.graphql.resolvers.mutate.util.DomainUtils; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.client.EntityClient; import com.linkedin.glossary.GlossaryNodeInfo; import com.linkedin.glossary.GlossaryTermInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; import org.mockito.Mockito; @@ -121,6 +124,15 @@ public void testGetSuccessForDomain() throws Exception { 0)) .thenReturn(new DomainProperties().setName(name)); + Mockito.when(mockClient.filter( + Mockito.eq(Constants.DOMAIN_ENTITY_NAME), + Mockito.eq(DomainUtils.buildNameAndParentDomainFilter(INPUT_FOR_DOMAIN.getName(), null)), + Mockito.eq(null), + Mockito.any(Integer.class), + Mockito.any(Integer.class), + Mockito.any(Authentication.class) + )).thenReturn(new SearchResult().setEntities(new SearchEntityArray())); + DomainProperties properties = new DomainProperties(); properties.setName(NEW_NAME); final MetadataChangeProposal proposal = MutationUtils.buildMetadataChangeProposalWithUrn(Urn.createFromString(DOMAIN_URN), diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java index 2d9f43029c479..ebafd1782e000 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/source/GetIngestionSourceResolverTest.java @@ -14,11 +14,12 @@ import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import java.util.HashSet; + import org.mockito.Mockito; import org.testng.annotations.Test; -import static org.testng.Assert.*; import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; +import static org.testng.Assert.assertThrows; public class GetIngestionSourceResolverTest { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java index d8325e9a74740..1adf7b1200574 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/SiblingsUtilsTest.java @@ -6,7 +6,6 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.resolvers.mutate.util.SiblingsUtils; import com.linkedin.metadata.entity.EntityService; -import junit.framework.TestCase; import org.mockito.Mockito; import org.testng.annotations.Test; @@ -14,8 +13,9 @@ import java.util.Optional; import static com.linkedin.metadata.Constants.SIBLINGS_ASPECT_NAME; +import static org.testng.AssertJUnit.assertEquals; -public class SiblingsUtilsTest extends TestCase { +public class SiblingsUtilsTest { private static final String TEST_DATASET_URN1 = "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created,PROD)"; private static final String TEST_DATASET_URN2 = "urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created2,PROD)"; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java index 989ebc18e9f6c..0a58ff88586c6 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java @@ -1,12 +1,13 @@ package com.linkedin.datahub.graphql.utils; import com.linkedin.datahub.graphql.util.DateUtil; -import junit.framework.TestCase; import org.joda.time.DateTime; import org.mockito.Mockito; import org.testng.annotations.Test; -public class DateUtilTest extends TestCase { +import static org.testng.AssertJUnit.assertEquals; + +public class DateUtilTest { private DateTime setTimeParts(int dayOfMonth, boolean zeroTime) { DateTime result = new DateTime() diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java index e205fd2f5c20e..270aa11c7b070 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java @@ -1,11 +1,10 @@ package com.linkedin.datahub.upgrade.common.steps; -import com.datahub.authentication.Authentication; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import java.util.function.Function; import lombok.RequiredArgsConstructor; @@ -13,8 +12,7 @@ @RequiredArgsConstructor public class GMSDisableWriteModeStep implements UpgradeStep { - private final Authentication _systemAuthentication; - private final RestliEntityClient _entityClient; + private final SystemRestliEntityClient _entityClient; @Override public String id() { @@ -30,7 +28,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - _entityClient.setWritable(false, _systemAuthentication); + _entityClient.setWritable(false); } catch (Exception e) { e.printStackTrace(); context.report().addLine("Failed to turn write mode off in GMS"); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java index 270eff8df227c..8df02123983e8 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java @@ -1,20 +1,17 @@ package com.linkedin.datahub.upgrade.common.steps; -import com.datahub.authentication.Authentication; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import java.util.function.Function; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class GMSEnableWriteModeStep implements UpgradeStep { - - private final Authentication _systemAuthentication; - private final RestliEntityClient _entityClient; + private final SystemRestliEntityClient _entityClient; @Override public String id() { @@ -30,7 +27,7 @@ public int retryCount() { public Function executable() { return (context) -> { try { - _entityClient.setWritable(true, _systemAuthentication); + _entityClient.setWritable(true); } catch (Exception e) { e.printStackTrace(); context.report().addLine("Failed to turn write mode back on in GMS"); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java index 0fb8b0eb6e20f..23ea81009fa1d 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java @@ -5,7 +5,7 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import io.ebean.Database; import javax.annotation.Nonnull; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java index 30175c6fa78c8..cd264e529e9a5 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java @@ -1,8 +1,7 @@ package com.linkedin.datahub.upgrade.config; -import com.datahub.authentication.Authentication; import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; import io.ebean.Database; @@ -21,15 +20,14 @@ public class NoCodeUpgradeConfig { ApplicationContext applicationContext; @Bean(name = "noCodeUpgrade") - @DependsOn({"ebeanServer", "entityService", "systemAuthentication", "restliEntityClient", "entityRegistry"}) + @DependsOn({"ebeanServer", "entityService", "systemRestliEntityClient", "entityRegistry"}) @Nonnull public NoCodeUpgrade createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); final EntityService entityService = applicationContext.getBean(EntityService.class); - final Authentication systemAuthentication = applicationContext.getBean(Authentication.class); - final RestliEntityClient entityClient = applicationContext.getBean(RestliEntityClient.class); + final SystemRestliEntityClient entityClient = applicationContext.getBean(SystemRestliEntityClient.class); final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); - return new NoCodeUpgrade(ebeanServer, entityService, entityRegistry, systemAuthentication, entityClient); + return new NoCodeUpgrade(ebeanServer, entityService, entityRegistry, entityClient); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java index 9b0fcf279abf5..97a08800534de 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java @@ -1,8 +1,7 @@ package com.linkedin.datahub.upgrade.config; -import com.datahub.authentication.Authentication; import com.linkedin.datahub.upgrade.restorebackup.RestoreBackup; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -22,19 +21,18 @@ public class RestoreBackupConfig { ApplicationContext applicationContext; @Bean(name = "restoreBackup") - @DependsOn({"ebeanServer", "entityService", "systemAuthentication", "restliEntityClient", "graphService", + @DependsOn({"ebeanServer", "entityService", "systemRestliEntityClient", "graphService", "searchService", "entityRegistry"}) @Nonnull public RestoreBackup createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); final EntityService entityService = applicationContext.getBean(EntityService.class); - final Authentication systemAuthentication = applicationContext.getBean(Authentication.class); - final RestliEntityClient entityClient = applicationContext.getBean(RestliEntityClient.class); + final SystemRestliEntityClient entityClient = applicationContext.getBean(SystemRestliEntityClient.class); final GraphService graphClient = applicationContext.getBean(GraphService.class); final EntitySearchService searchClient = applicationContext.getBean(EntitySearchService.class); final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); - return new RestoreBackup(ebeanServer, entityService, entityRegistry, systemAuthentication, entityClient, + return new RestoreBackup(ebeanServer, entityService, entityRegistry, entityClient, graphClient, searchClient); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java index ee4a3bc504e77..a299deb874721 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java @@ -1,13 +1,12 @@ package com.linkedin.datahub.upgrade.nocode; -import com.datahub.authentication.Authentication; import com.google.common.collect.ImmutableMap; import com.linkedin.datahub.upgrade.Upgrade; import com.linkedin.datahub.upgrade.UpgradeCleanupStep; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep; import com.linkedin.datahub.upgrade.common.steps.GMSQualificationStep; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; import io.ebean.Database; @@ -30,12 +29,10 @@ public NoCodeUpgrade( final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final Authentication systemAuthentication, - final RestliEntityClient entityClient) { + final SystemRestliEntityClient entityClient) { _steps = buildUpgradeSteps( server, entityService, entityRegistry, - systemAuthentication, entityClient); _cleanupSteps = buildCleanupSteps(); } @@ -63,15 +60,14 @@ private List buildUpgradeSteps( final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final Authentication systemAuthentication, - final RestliEntityClient entityClient) { + final SystemRestliEntityClient entityClient) { final List steps = new ArrayList<>(); steps.add(new RemoveAspectV2TableStep(server)); steps.add(new GMSQualificationStep(ImmutableMap.of("noCode", "true"))); steps.add(new UpgradeQualificationStep(server)); steps.add(new CreateAspectTableStep(server)); steps.add(new DataMigrationStep(server, entityService, entityRegistry)); - steps.add(new GMSEnableWriteModeStep(systemAuthentication, entityClient)); + steps.add(new GMSEnableWriteModeStep(entityClient)); return steps; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java index 15bbe40d1e566..9a64d5fe1810c 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/DeleteLegacySearchIndicesStep.java @@ -7,9 +7,9 @@ import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import java.util.function.Function; import lombok.RequiredArgsConstructor; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; // Do we need SQL-tech specific migration paths? diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java index 2b5e23c5f8269..a5d8d6ce9b666 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; public class NoCodeCleanupUpgrade implements Upgrade { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java index 67718a6739beb..9175ad606e3c8 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java @@ -1,6 +1,5 @@ package com.linkedin.datahub.upgrade.restorebackup; -import com.datahub.authentication.Authentication; import com.google.common.collect.ImmutableList; import com.linkedin.datahub.upgrade.Upgrade; import com.linkedin.datahub.upgrade.UpgradeCleanupStep; @@ -9,7 +8,7 @@ import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; import com.linkedin.datahub.upgrade.common.steps.GMSDisableWriteModeStep; import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -27,11 +26,10 @@ public RestoreBackup( final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final Authentication systemAuthentication, - final RestliEntityClient entityClient, + final SystemRestliEntityClient entityClient, final GraphService graphClient, final EntitySearchService searchClient) { - _steps = buildSteps(server, entityService, entityRegistry, systemAuthentication, entityClient, graphClient, searchClient); + _steps = buildSteps(server, entityService, entityRegistry, entityClient, graphClient, searchClient); } @Override @@ -48,17 +46,16 @@ private List buildSteps( final Database server, final EntityService entityService, final EntityRegistry entityRegistry, - final Authentication systemAuthentication, - final RestliEntityClient entityClient, + final SystemRestliEntityClient entityClient, final GraphService graphClient, final EntitySearchService searchClient) { final List steps = new ArrayList<>(); - steps.add(new GMSDisableWriteModeStep(systemAuthentication, entityClient)); + steps.add(new GMSDisableWriteModeStep(entityClient)); steps.add(new ClearSearchServiceStep(searchClient, true)); steps.add(new ClearGraphServiceStep(graphClient, true)); steps.add(new ClearAspectV2TableStep(server)); steps.add(new RestoreStorageStep(entityService, entityRegistry)); - steps.add(new GMSEnableWriteModeStep(systemAuthentication, entityClient)); + steps.add(new GMSEnableWriteModeStep(entityClient)); return steps; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java index ee6a5ed6f1536..3c0a9762a28c9 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java @@ -25,6 +25,8 @@ public class RestoreIndices implements Upgrade { public static final String URN_ARG_NAME = "urn"; public static final String URN_LIKE_ARG_NAME = "urnLike"; + public static final String STARTING_OFFSET_ARG_NAME = "startingOffset"; + private final List _steps; public RestoreIndices(final Database server, final EntityService entityService, diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java index ce39b3fb562af..2ac4fea2e653a 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java @@ -30,6 +30,8 @@ public class SendMAEStep implements UpgradeStep { private static final int DEFAULT_BATCH_SIZE = 1000; private static final long DEFAULT_BATCH_DELAY_MS = 250; + + private static final int DEFAULT_STARTING_OFFSET = 0; private static final int DEFAULT_THREADS = 1; private final Database _server; @@ -83,6 +85,7 @@ private RestoreIndicesArgs getArgs(UpgradeContext context) { result.batchSize = getBatchSize(context.parsedArgs()); result.numThreads = getThreadCount(context.parsedArgs()); result.batchDelayMs = getBatchDelayMs(context.parsedArgs()); + result.start = getStartingOffset(context.parsedArgs()); if (containsKey(context.parsedArgs(), RestoreIndices.ASPECT_NAME_ARG_NAME)) { result.aspectName = context.parsedArgs().get(RestoreIndices.ASPECT_NAME_ARG_NAME).get(); } @@ -124,7 +127,7 @@ public Function executable() { final int rowCount = getRowCount(args); context.report().addLine(String.format("Found %s latest aspects in aspects table in %.2f minutes.", rowCount, (float) (System.currentTimeMillis() - startTime) / 1000 / 60)); - int start = 0; + int start = args.start; List> futures = new ArrayList<>(); startTime = System.currentTimeMillis(); @@ -186,6 +189,10 @@ private int getBatchSize(final Map> parsedArgs) { return getInt(parsedArgs, DEFAULT_BATCH_SIZE, RestoreIndices.BATCH_SIZE_ARG_NAME); } + private int getStartingOffset(final Map> parsedArgs) { + return getInt(parsedArgs, DEFAULT_STARTING_OFFSET, RestoreIndices.STARTING_OFFSET_ARG_NAME); + } + private long getBatchDelayMs(final Map> parsedArgs) { long resolvedBatchDelayMs = DEFAULT_BATCH_DELAY_MS; if (containsKey(parsedArgs, RestoreIndices.BATCH_DELAY_MS_ARG_NAME)) { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java index 465a5fe342667..2feca1f27e625 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPostStep.java @@ -16,8 +16,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.client.RequestOptions; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.client.RequestOptions; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java index 6f2f3a8bd727c..82b9428c89fb8 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java @@ -19,10 +19,10 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.indices.ResizeRequest; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.indices.ResizeRequest; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; @@ -97,7 +97,7 @@ private boolean blockWrites(String indexName) throws InterruptedException, IOExc ack = _esComponents.getSearchClient().indices() .putSettings(request, RequestOptions.DEFAULT).isAcknowledged(); log.info("Updated index {} with new settings. Settings: {}, Acknowledged: {}", indexName, indexSettings, ack); - } catch (ElasticsearchStatusException | IOException ese) { + } catch (OpenSearchStatusException | IOException ese) { // Cover first run case, indices won't exist so settings updates won't work nor will the rest of the preConfigure steps. // Since no data are in there they are skippable. // Have to hack around HighLevelClient not sending the actual Java type nor having an easy way to extract it :( diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java index f60aa283c0140..bb042bac6df95 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/CleanIndicesStep.java @@ -9,7 +9,7 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.client.RestHighLevelClient; import java.util.List; import java.util.function.Function; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java index fa414798ccfea..4b04feac62cbf 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java @@ -4,12 +4,12 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; +import org.opensearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.client.GetAliasesResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestHighLevelClient; import java.io.IOException; import java.util.ArrayList; diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index 82606befd2663..d2ad4ab6f4db1 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -8,20 +8,27 @@ import { EntityPage } from './entity/EntityPage'; import { BrowseResultsPage } from './browse/BrowseResultsPage'; import { SearchPage } from './search/SearchPage'; import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage'; -import { ManageDomainsPage } from './domain/ManageDomainsPage'; import { ManageIngestionPage } from './ingest/ManageIngestionPage'; import GlossaryRoutes from './glossary/GlossaryRoutes'; import { SettingsPage } from './settings/SettingsPage'; +import DomainRoutes from './domain/DomainRoutes'; +import { useIsNestedDomainsEnabled } from './useAppConfig'; +import { ManageDomainsPage } from './domain/ManageDomainsPage'; /** * Container for all searchable page routes */ export const SearchRoutes = (): JSX.Element => { const entityRegistry = useEntityRegistry(); + const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); + const entities = isNestedDomainsEnabled + ? entityRegistry.getEntitiesForSearchRoutes() + : entityRegistry.getNonGlossaryEntities(); + return ( - {entityRegistry.getNonGlossaryEntities().map((entity) => ( + {entities.map((entity) => ( { /> } /> } /> - } /> + {isNestedDomainsEnabled && } />} + {!isNestedDomainsEnabled && } />} } /> } /> } /> diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index 84173b522fb07..2734026400933 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -35,6 +35,8 @@ export enum EventType { SearchBarExploreAllClickEvent, SearchResultsExploreAllClickEvent, SearchAcrossLineageEvent, + VisualLineageViewEvent, + VisualLineageExpandGraphEvent, SearchAcrossLineageResultsViewEvent, DownloadAsCsvEvent, SignUpEvent, @@ -55,6 +57,7 @@ export enum EventType { ShowStandardHomepageEvent, CreateGlossaryEntityEvent, CreateDomainEvent, + MoveDomainEvent, CreateIngestionSourceEvent, UpdateIngestionSourceEvent, DeleteIngestionSourceEvent, @@ -339,12 +342,23 @@ export interface HomePageRecommendationClickEvent extends BaseEvent { index?: number; } +export interface VisualLineageViewEvent extends BaseEvent { + type: EventType.VisualLineageViewEvent; + entityType?: EntityType; +} + +export interface VisualLineageExpandGraphEvent extends BaseEvent { + type: EventType.VisualLineageExpandGraphEvent; + targetEntityType?: EntityType; +} + export interface SearchAcrossLineageEvent extends BaseEvent { type: EventType.SearchAcrossLineageEvent; query: string; entityTypeFilter?: EntityType; pageNumber: number; originPath: string; + maxDegree?: string; } export interface SearchAcrossLineageResultsViewEvent extends BaseEvent { type: EventType.SearchAcrossLineageResultsViewEvent; @@ -352,6 +366,7 @@ export interface SearchAcrossLineageResultsViewEvent extends BaseEvent { entityTypeFilter?: EntityType; page?: number; total: number; + maxDegree?: string; } export interface DownloadAsCsvEvent extends BaseEvent { @@ -454,6 +469,13 @@ export interface CreateGlossaryEntityEvent extends BaseEvent { export interface CreateDomainEvent extends BaseEvent { type: EventType.CreateDomainEvent; + parentDomainUrn?: string; +} + +export interface MoveDomainEvent extends BaseEvent { + type: EventType.MoveDomainEvent; + oldParentDomainUrn?: string; + parentDomainUrn?: string; } // Managed Ingestion Events @@ -633,6 +655,8 @@ export type Event = | RecommendationImpressionEvent | SearchAcrossLineageEvent | SearchAcrossLineageResultsViewEvent + | VisualLineageViewEvent + | VisualLineageExpandGraphEvent | DownloadAsCsvEvent | RecommendationClickEvent | HomePageRecommendationClickEvent @@ -653,6 +677,7 @@ export type Event = | ShowStandardHomepageEvent | CreateGlossaryEntityEvent | CreateDomainEvent + | MoveDomainEvent | CreateIngestionSourceEvent | UpdateIngestionSourceEvent | DeleteIngestionSourceEvent diff --git a/datahub-web-react/src/app/domain/CreateDomainModal.tsx b/datahub-web-react/src/app/domain/CreateDomainModal.tsx index f33a951f52417..bd4f14f0bb57d 100644 --- a/datahub-web-react/src/app/domain/CreateDomainModal.tsx +++ b/datahub-web-react/src/app/domain/CreateDomainModal.tsx @@ -8,9 +8,12 @@ import { useEnterKeyListener } from '../shared/useEnterKeyListener'; import { validateCustomUrnId } from '../shared/textUtil'; import analytics, { EventType } from '../analytics'; import { useEntityRegistry } from '../useEntityRegistry'; +import DomainParentSelect from '../entity/shared/EntityDropdown/DomainParentSelect'; +import { useIsNestedDomainsEnabled } from '../useAppConfig'; +import { useDomainsContext } from './DomainsContext'; const SuggestedNamesGroup = styled.div` - margin-top: 12px; + margin-top: 8px; `; const ClickableTag = styled(Tag)` @@ -19,9 +22,38 @@ const ClickableTag = styled(Tag)` } `; +const FormItem = styled(Form.Item)` + .ant-form-item-label { + padding-bottom: 2px; + } +`; + +const FormItemWithMargin = styled(FormItem)` + margin-bottom: 16px; +`; + +const FormItemNoMargin = styled(FormItem)` + margin-bottom: 0; +`; + +const FormItemLabel = styled(Typography.Text)` + font-weight: 600; + color: #373d44; +`; + +const AdvancedLabel = styled(Typography.Text)` + color: #373d44; +`; + type Props = { onClose: () => void; - onCreate: (urn: string, id: string | undefined, name: string, description: string | undefined) => void; + onCreate: ( + urn: string, + id: string | undefined, + name: string, + description: string | undefined, + parentDomain?: string, + ) => void; }; const SUGGESTED_DOMAIN_NAMES = ['Engineering', 'Marketing', 'Sales', 'Product']; @@ -31,9 +63,14 @@ const NAME_FIELD_NAME = 'name'; const DESCRIPTION_FIELD_NAME = 'description'; export default function CreateDomainModal({ onClose, onCreate }: Props) { + const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); const entityRegistry = useEntityRegistry(); const { t } = useTranslation(); const [createDomainMutation] = useCreateDomainMutation(); + const { entityData } = useDomainsContext(); + const [selectedParentUrn, setSelectedParentUrn] = useState( + (isNestedDomainsEnabled && entityData?.urn) || '', + ); const [createButtonEnabled, setCreateButtonEnabled] = useState(false); const [form] = Form.useForm(); @@ -44,6 +81,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { id: form.getFieldValue(ID_FIELD_NAME), name: form.getFieldValue(NAME_FIELD_NAME), description: form.getFieldValue(DESCRIPTION_FIELD_NAME), + parentDomain: selectedParentUrn || undefined, }, }, }) @@ -51,6 +89,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { if (!errors) { analytics.event({ type: EventType.CreateDomainEvent, + parentDomainUrn: selectedParentUrn || undefined, }); message.success({ content: t('crud.success.createWithName', { @@ -63,6 +102,7 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { form.getFieldValue(ID_FIELD_NAME), form.getFieldValue(NAME_FIELD_NAME), form.getFieldValue(DESCRIPTION_FIELD_NAME), + selectedParentUrn || undefined, ); form.resetFields(); } @@ -113,9 +153,16 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { setCreateButtonEnabled(!form.getFieldsError().some((field) => field.errors.length > 0)); }} > - {t('common.name')}}> - {t('form.giveYourNewDomainAName')} - {t('common.parent')} ({t('common.optional')})}> + + + )} + {t('common.name')}}> + - + {SUGGESTED_DOMAIN_NAMES.map((name) => { return ( @@ -148,31 +195,28 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { ); })} - - {t('common.description')}}> - {t('domain.domainDescriptionDescription')} - + {t('common.description')}} + help={t('domain.domainDescriptionDescription')} + > + - - + + - {t('common.advanced')}} - key="1" - > - - {entityRegistry.getEntityNameTrans(EntityType.Domain, t)} {t('common.id')} - - } + {t('common.advancedOptions')}} key="1"> + + {entityRegistry.getEntityNameTrans(EntityType.Domain, t)} {t('common.id')} + } + help={t('domain.domainIdDescription')} > - {t('domain.domainIdDescription')} - ({ @@ -186,8 +230,8 @@ export default function CreateDomainModal({ onClose, onCreate }: Props) { ]} > - - + + diff --git a/datahub-web-react/src/app/domain/DomainIcon.tsx b/datahub-web-react/src/app/domain/DomainIcon.tsx new file mode 100644 index 0000000000000..0fe9892f0c281 --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainIcon.tsx @@ -0,0 +1,11 @@ +import Icon from '@ant-design/icons/lib/components/Icon'; +import React from 'react'; +import { ReactComponent as DomainsIcon } from '../../images/domain.svg'; + +type Props = { + style?: React.CSSProperties; +}; + +export default function DomainIcon({ style }: Props) { + return ; +} diff --git a/datahub-web-react/src/app/domain/DomainRoutes.tsx b/datahub-web-react/src/app/domain/DomainRoutes.tsx new file mode 100644 index 0000000000000..56811ddc48c0c --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainRoutes.tsx @@ -0,0 +1,39 @@ +import React, { useState } from 'react'; +import styled from 'styled-components/macro'; +import { Switch, Route } from 'react-router-dom'; +import { PageRoutes } from '../../conf/Global'; +import { EntityPage } from '../entity/EntityPage'; +import { useEntityRegistry } from '../useEntityRegistry'; +import ManageDomainsPageV2 from './nestedDomains/ManageDomainsPageV2'; +import { EntityType } from '../../types.generated'; +import ManageDomainsSidebar from './nestedDomains/ManageDomainsSidebar'; +import { DomainsContext } from './DomainsContext'; +import { GenericEntityProperties } from '../entity/shared/types'; + +const ContentWrapper = styled.div` + display: flex; + flex: 1; + overflow: hidden; +`; + +export default function DomainRoutes() { + const entityRegistry = useEntityRegistry(); + const [entityData, setEntityData] = useState(null); + const [parentDomainsToUpdate, setParentDomainsToUpdate] = useState([]); + + return ( + + + + + } + /> + } /> + + + + ); +} diff --git a/datahub-web-react/src/app/domain/DomainSearch.tsx b/datahub-web-react/src/app/domain/DomainSearch.tsx new file mode 100644 index 0000000000000..e82dae9c2c9e6 --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainSearch.tsx @@ -0,0 +1,143 @@ +import React, { CSSProperties, useRef, useState } from 'react'; +import { Link } from 'react-router-dom'; +import styled from 'styled-components/macro'; +import Highlight from 'react-highlighter'; +import { useGetSearchResultsForMultipleQuery } from '../../graphql/search.generated'; +import { EntityType } from '../../types.generated'; +import { IconStyleType } from '../entity/Entity'; +import { ANTD_GRAY } from '../entity/shared/constants'; +import { SearchBar } from '../search/SearchBar'; +import ClickOutside from '../shared/ClickOutside'; +import { useEntityRegistry } from '../useEntityRegistry'; +import DomainIcon from './DomainIcon'; +import ParentEntities from '../search/filters/ParentEntities'; +import { getParentDomains } from './utils'; + +const DomainSearchWrapper = styled.div` + position: relative; +`; + +const ResultsWrapper = styled.div` + background-color: white; + border-radius: 5px; + box-shadow: 0 3px 6px -4px rgb(0 0 0 / 12%), 0 6px 16px 0 rgb(0 0 0 / 8%), 0 9px 28px 8px rgb(0 0 0 / 5%); + max-height: 380px; + overflow: auto; + padding: 8px; + position: absolute; + max-height: 210px; + overflow: auto; + width: calc(100% - 24px); + left: 12px; + top: 45px; + z-index: 1; +`; + +const SearchResult = styled(Link)` + color: #262626; + display: flex; + align-items: center; + gap: 8px; + height: 100%; + padding: 6px 8px; + width: 100%; + &:hover { + background-color: ${ANTD_GRAY[3]}; + color: #262626; + } +`; + +const IconWrapper = styled.span``; + +const highlightMatchStyle: CSSProperties = { + fontWeight: 'bold', + background: 'none', + padding: 0, +}; + +function DomainSearch() { + const [query, setQuery] = useState(''); + const [isSearchBarFocused, setIsSearchBarFocused] = useState(false); + const entityRegistry = useEntityRegistry(); + + const { data } = useGetSearchResultsForMultipleQuery({ + variables: { + input: { + types: [EntityType.Domain], + query, + start: 0, + count: 50, + }, + }, + skip: !query, + }); + + const searchResults = data?.searchAcrossEntities?.searchResults; + const timerRef = useRef(-1); + const handleQueryChange = (q: string) => { + window.clearTimeout(timerRef.current); + timerRef.current = window.setTimeout(() => { + setQuery(q); + }, 250); + }; + + return ( + + setIsSearchBarFocused(false)}> + null} + onQueryChange={(q) => handleQueryChange(q)} + entityRegistry={entityRegistry} + onFocus={() => setIsSearchBarFocused(true)} + /> + {isSearchBarFocused && searchResults && !!searchResults.length && ( + + {searchResults.map((result) => { + return ( + setIsSearchBarFocused(false)} + > + + {result.entity.type === EntityType.Domain ? ( + + ) : ( + entityRegistry.getIcon(result.entity.type, 12, IconStyleType.ACCENT) + )} + +
+ + + {entityRegistry.getDisplayName(result.entity.type, result.entity)} + +
+ + ); + })} + + )} + + + ); +} + +export default DomainSearch; diff --git a/datahub-web-react/src/app/domain/DomainsContext.tsx b/datahub-web-react/src/app/domain/DomainsContext.tsx new file mode 100644 index 0000000000000..ecbdaebd03817 --- /dev/null +++ b/datahub-web-react/src/app/domain/DomainsContext.tsx @@ -0,0 +1,21 @@ +import React, { useContext } from 'react'; +import { GenericEntityProperties } from '../entity/shared/types'; + +export interface DomainsContextType { + entityData: GenericEntityProperties | null; + setEntityData: (entityData: GenericEntityProperties | null) => void; + parentDomainsToUpdate: string[]; + setParentDomainsToUpdate: (values: string[]) => void; +} + +export const DomainsContext = React.createContext({ + entityData: null, + setEntityData: () => {}, + parentDomainsToUpdate: [], // used to tell domains to refetch their children count after updates (create, move, delete) + setParentDomainsToUpdate: () => {}, +}); + +export const useDomainsContext = () => { + const { entityData, setEntityData, parentDomainsToUpdate, setParentDomainsToUpdate } = useContext(DomainsContext); + return { entityData, setEntityData, parentDomainsToUpdate, setParentDomainsToUpdate }; +}; diff --git a/datahub-web-react/src/app/domain/DomainsList.tsx b/datahub-web-react/src/app/domain/DomainsList.tsx index 664214b3a02fa..98f961225e768 100644 --- a/datahub-web-react/src/app/domain/DomainsList.tsx +++ b/datahub-web-react/src/app/domain/DomainsList.tsx @@ -19,8 +19,8 @@ import { OnboardingTour } from '../onboarding/OnboardingTour'; import { DOMAINS_INTRO_ID, DOMAINS_CREATE_DOMAIN_ID } from '../onboarding/config/DomainsOnboardingConfig'; import { getElasticCappedTotalValueText } from '../entity/shared/constants'; import { StyledTable } from '../entity/shared/components/styled/StyledTable'; -import { IconStyleType } from '../entity/Entity'; import { DomainOwnersColumn, DomainListMenuColumn, DomainNameColumn } from './DomainListColumns'; +import DomainIcon from './DomainIcon'; const DomainsContainer = styled.div``; @@ -83,7 +83,6 @@ export const DomainsList = () => { }, 2000); }; - const logoIcon = entityRegistry.getIcon(EntityType.Domain, 12, IconStyleType.ACCENT); const allColumns = [ { title: t('common.name'), @@ -92,7 +91,14 @@ export const DomainsList = () => { sorter: (sourceA, sourceB) => { return sourceA.name.localeCompare(sourceB.name); }, - render: DomainNameColumn(logoIcon, t), + render: DomainNameColumn( + , + ), }, { title: t('common.owners'), diff --git a/datahub-web-react/src/app/domain/ManageDomainsPage.tsx b/datahub-web-react/src/app/domain/ManageDomainsPage.tsx index 3f921af514cc8..fd4b853521410 100644 --- a/datahub-web-react/src/app/domain/ManageDomainsPage.tsx +++ b/datahub-web-react/src/app/domain/ManageDomainsPage.tsx @@ -1,8 +1,10 @@ import { Typography } from 'antd'; -import React from 'react'; +import React, { useState } from 'react'; import styled from 'styled-components'; import { useTranslation } from 'react-i18next'; import { DomainsList } from './DomainsList'; +import { DomainsContext } from './DomainsContext'; +import { GenericEntityProperties } from '../entity/shared/types'; const PageContainer = styled.div` padding-top: 20px; @@ -24,15 +26,20 @@ const ListContainer = styled.div``; export const ManageDomainsPage = () => { const { t } = useTranslation(); + const [entityData, setEntityData] = useState(null); + const [parentDomainsToUpdate, setParentDomainsToUpdate] = useState([]); + return ( - - + + + {t('common.domains')} {t('domain.domainManagementDescription')} - - - - - + + + + + + ); }; diff --git a/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx new file mode 100644 index 0000000000000..d9ff18514d8cf --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/DomainsSidebarHeader.tsx @@ -0,0 +1,58 @@ +import { useApolloClient } from '@apollo/client'; +import { PlusOutlined } from '@ant-design/icons'; +import { Button } from 'antd'; +import React, { useState } from 'react'; +import { Link } from 'react-router-dom'; +import styled from 'styled-components'; +import { ANTD_GRAY, ANTD_GRAY_V2 } from '../../entity/shared/constants'; +import DomainsTitle from './DomainsTitle'; +import { PageRoutes } from '../../../conf/Global'; +import CreateDomainModal from '../CreateDomainModal'; +import { updateListDomainsCache } from '../utils'; +import { useDomainsContext } from '../DomainsContext'; + +const HeaderWrapper = styled.div` + border-bottom: 1px solid ${ANTD_GRAY[4]}; + padding: 16px; + font-size: 20px; + display: flex; + align-items: center; + justify-content: space-between; +`; + +const StyledButton = styled(Button)` + box-shadow: none; + border-color: ${ANTD_GRAY_V2[6]}; +`; + +const StyledLink = styled(Link)` + color: inherit; + + &:hover { + color: inherit; + } +`; + +export default function DomainsSidebarHeader() { + const { setParentDomainsToUpdate } = useDomainsContext(); + const [isCreatingDomain, setIsCreatingDomain] = useState(false); + const client = useApolloClient(); + + return ( + + + + + } onClick={() => setIsCreatingDomain(true)} /> + {isCreatingDomain && ( + setIsCreatingDomain(false)} + onCreate={(urn, id, name, description, parentDomain) => { + updateListDomainsCache(client, urn, id, name, description, parentDomain); + if (parentDomain) setParentDomainsToUpdate([parentDomain]); + }} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx b/datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx new file mode 100644 index 0000000000000..3aa7c8330d079 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/DomainsTitle.tsx @@ -0,0 +1,18 @@ +import React from 'react'; +import styled from 'styled-components'; +import DomainIcon from '../DomainIcon'; + +const IconWrapper = styled.span` + margin-right: 10px; +`; + +export default function DomainsTitle() { + return ( + + + + + Domains + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx new file mode 100644 index 0000000000000..0e5c035df00c1 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsPageV2.tsx @@ -0,0 +1,61 @@ +import { useApolloClient } from '@apollo/client'; +import { Button } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; +import React, { useEffect, useState } from 'react'; +import styled from 'styled-components/macro'; +import DomainsTitle from './DomainsTitle'; +import RootDomains from './RootDomains'; +import { DOMAINS_CREATE_DOMAIN_ID, DOMAINS_INTRO_ID } from '../../onboarding/config/DomainsOnboardingConfig'; +import { OnboardingTour } from '../../onboarding/OnboardingTour'; +import { ANTD_GRAY_V2 } from '../../entity/shared/constants'; +import CreateDomainModal from '../CreateDomainModal'; +import { updateListDomainsCache } from '../utils'; +import { useDomainsContext } from '../DomainsContext'; + +const PageWrapper = styled.div` + background-color: ${ANTD_GRAY_V2[1]}; + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +`; + +const Header = styled.div` + display: flex; + justify-content: space-between; + padding: 32px 24px; + font-size: 30px; + align-items: center; +`; + +export default function ManageDomainsPageV2() { + const { setEntityData, setParentDomainsToUpdate } = useDomainsContext(); + const [isCreatingDomain, setIsCreatingDomain] = useState(false); + const client = useApolloClient(); + + useEffect(() => { + setEntityData(null); + }, [setEntityData]); + + return ( + + +
+ + +
+ + {isCreatingDomain && ( + setIsCreatingDomain(false)} + onCreate={(urn, id, name, description, parentDomain) => { + updateListDomainsCache(client, urn, id, name, description, parentDomain); + if (parentDomain) setParentDomainsToUpdate([parentDomain]); + }} + /> + )} + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx new file mode 100644 index 0000000000000..827031138dcdb --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/ManageDomainsSidebar.tsx @@ -0,0 +1,28 @@ +import React, { useState } from 'react'; +import { MAX_BROWSER_WIDTH, MIN_BROWSWER_WIDTH } from '../../glossary/BusinessGlossaryPage'; +import { ProfileSidebarResizer } from '../../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; +import DomainsSidebarHeader from './DomainsSidebarHeader'; +import { SidebarWrapper } from '../../shared/sidebar/components'; +import DomainNavigator from './domainNavigator/DomainNavigator'; +import DomainSearch from '../DomainSearch'; + +export default function ManageDomainsSidebar() { + const [browserWidth, setBrowserWith] = useState(window.innerWidth * 0.2); + + return ( + <> + + + + + + + setBrowserWith(Math.min(Math.max(width, MIN_BROWSWER_WIDTH), MAX_BROWSER_WIDTH)) + } + initialSize={browserWidth} + isSidebarOnLeft + /> + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx new file mode 100644 index 0000000000000..757119919e336 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/RootDomains.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Message } from '../../shared/Message'; +import { ResultWrapper } from '../../search/SearchResultList'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { EntityType } from '../../../types.generated'; +import useListDomains from '../useListDomains'; + +const DomainsWrapper = styled.div` + overflow: auto; + padding: 0 28px 16px 28px; +`; + +export default function RootDomains() { + const entityRegistry = useEntityRegistry(); + const { loading, error, data, sortedDomains } = useListDomains({}); + + return ( + <> + {!data && loading && } + {error && } + + {sortedDomains?.map((domain) => ( + + {entityRegistry.renderSearchResult(EntityType.Domain, { entity: domain, matchedFields: [] })} + + ))} + + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx new file mode 100644 index 0000000000000..0fbcffb9a260c --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNavigator.tsx @@ -0,0 +1,37 @@ +import { Alert } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; +import useListDomains from '../../useListDomains'; +import DomainNode from './DomainNode'; +import { Domain } from '../../../../types.generated'; + +const NavigatorWrapper = styled.div` + font-size: 14px; + max-height: calc(100% - 65px); + padding: 8px 8px 16px 16px; + overflow: auto; +`; + +interface Props { + domainUrnToHide?: string; + selectDomainOverride?: (domain: Domain) => void; +} + +export default function DomainNavigator({ domainUrnToHide, selectDomainOverride }: Props) { + const { sortedDomains, error } = useListDomains({}); + + return ( + + {error && } + {sortedDomains?.map((domain) => ( + + ))} + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx new file mode 100644 index 0000000000000..09c8e13853bb7 --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/DomainNode.tsx @@ -0,0 +1,137 @@ +import { Typography } from 'antd'; +import React, { useEffect, useMemo } from 'react'; +import { useHistory } from 'react-router'; +import styled from 'styled-components'; +import { Domain } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { RotatingTriangle } from '../../../shared/sidebar/components'; +import DomainIcon from '../../DomainIcon'; +import useListDomains from '../../useListDomains'; +import useToggle from '../../../shared/useToggle'; +import { BodyContainer, BodyGridExpander } from '../../../shared/components'; +import { ANTD_GRAY_V2 } from '../../../entity/shared/constants'; +import { useDomainsContext } from '../../DomainsContext'; +import { applyOpacity } from '../../../shared/styleUtils'; +import useHasDomainChildren from './useHasDomainChildren'; + +const RowWrapper = styled.div` + align-items: center; + display: flex; + padding: 2px 2px 4px 0; + overflow: hidden; +`; + +const NameWrapper = styled(Typography.Text)<{ isSelected: boolean; addLeftPadding: boolean }>` + flex: 1; + overflow: hidden; + padding: 2px; + ${(props) => + props.isSelected && `background-color: ${applyOpacity(props.theme.styles['primary-color'] || '', 10)};`} + ${(props) => props.addLeftPadding && 'padding-left: 22px;'} + + &:hover { + ${(props) => !props.isSelected && `background-color: ${ANTD_GRAY_V2[1]};`} + cursor: pointer; + } + + svg { + margin-right: 6px; + } +`; + +const ButtonWrapper = styled.span` + margin-right: 4px; + font-size: 16px; + height: 16px; + width: 16px; + + svg { + height: 10px; + width: 10px; + } + + .ant-btn { + height: 16px; + width: 16px; + } +`; + +const StyledExpander = styled(BodyGridExpander)` + padding-left: 24px; +`; + +interface Props { + domain: Domain; + numDomainChildren: number; + domainUrnToHide?: string; + selectDomainOverride?: (domain: Domain) => void; +} + +export default function DomainNode({ domain, numDomainChildren, domainUrnToHide, selectDomainOverride }: Props) { + const shouldHideDomain = domainUrnToHide === domain.urn; + const history = useHistory(); + const entityRegistry = useEntityRegistry(); + const { entityData } = useDomainsContext(); + const { isOpen, isClosing, toggle, toggleOpen } = useToggle({ + initialValue: false, + closeDelay: 250, + }); + const { sortedDomains } = useListDomains({ parentDomain: domain.urn, skip: !isOpen || shouldHideDomain }); + const isOnEntityPage = entityData && entityData.urn === domain.urn; + const displayName = entityRegistry.getDisplayName(domain.type, isOnEntityPage ? entityData : domain); + const isInSelectMode = !!selectDomainOverride; + const hasDomainChildren = useHasDomainChildren({ domainUrn: domain.urn, numDomainChildren }); + + const shouldAutoOpen = useMemo( + () => !isInSelectMode && entityData?.parentDomains?.domains.some((parent) => parent.urn === domain.urn), + [isInSelectMode, entityData, domain.urn], + ); + + useEffect(() => { + if (shouldAutoOpen) toggleOpen(); + }, [shouldAutoOpen, toggleOpen]); + + function handleSelectDomain() { + if (selectDomainOverride) { + selectDomainOverride(domain); + } else { + history.push(entityRegistry.getEntityUrl(domain.type, domain.urn)); + } + } + + if (shouldHideDomain) return null; + + return ( + <> + + {hasDomainChildren && ( + + + + )} + + {!isInSelectMode && } + {displayName} + + + + + {sortedDomains?.map((childDomain) => ( + + ))} + + + + ); +} diff --git a/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts new file mode 100644 index 0000000000000..d16d5de23fbaf --- /dev/null +++ b/datahub-web-react/src/app/domain/nestedDomains/domainNavigator/useHasDomainChildren.ts @@ -0,0 +1,29 @@ +import { useEffect } from 'react'; +import { useGetDomainChildrenCountLazyQuery } from '../../../../graphql/domain.generated'; +import { useDomainsContext } from '../../DomainsContext'; + +interface Props { + domainUrn: string; + numDomainChildren: number; // number that comes from parent query to render this domain +} + +export default function useHasDomainChildren({ domainUrn, numDomainChildren }: Props) { + const { parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); + const [getDomainChildrenCount, { data: childrenData }] = useGetDomainChildrenCountLazyQuery(); + + useEffect(() => { + let timer; + // fetch updated children count to determine if we show triangle toggle + if (parentDomainsToUpdate.includes(domainUrn)) { + timer = setTimeout(() => { + getDomainChildrenCount({ variables: { urn: domainUrn } }); + setParentDomainsToUpdate(parentDomainsToUpdate.filter((urn) => urn !== domainUrn)); + }, 2000); + } + return () => { + if (timer) window.clearTimeout(timer); + }; + }, [domainUrn, getDomainChildrenCount, parentDomainsToUpdate, setParentDomainsToUpdate]); + + return childrenData ? !!childrenData.domain?.children?.total : !!numDomainChildren; +} diff --git a/datahub-web-react/src/app/domain/useListDomains.tsx b/datahub-web-react/src/app/domain/useListDomains.tsx new file mode 100644 index 0000000000000..74f6b454f11d4 --- /dev/null +++ b/datahub-web-react/src/app/domain/useListDomains.tsx @@ -0,0 +1,27 @@ +import { useListDomainsQuery } from '../../graphql/domain.generated'; +import { useSortedDomains } from './utils'; + +interface Props { + parentDomain?: string; + skip?: boolean; + sortBy?: 'displayName'; +} + +export default function useListDomains({ parentDomain, skip, sortBy = 'displayName' }: Props) { + const { data, error, loading, refetch } = useListDomainsQuery({ + skip, + variables: { + input: { + start: 0, + count: 1000, // don't paginate the home page, get all root level domains + parentDomain, + }, + }, + fetchPolicy: 'network-only', // always use network request first to populate cache + nextFetchPolicy: 'cache-first', // then use cache after that so we can manipulate it + }); + + const sortedDomains = useSortedDomains(data?.listDomains?.domains, sortBy); + + return { data, sortedDomains, error, loading, refetch }; +} diff --git a/datahub-web-react/src/app/domain/utils.ts b/datahub-web-react/src/app/domain/utils.ts index 3af161bc44565..8273c33e2c41d 100644 --- a/datahub-web-react/src/app/domain/utils.ts +++ b/datahub-web-react/src/app/domain/utils.ts @@ -1,9 +1,18 @@ +import { ApolloClient } from '@apollo/client'; +import { useEffect } from 'react'; +import { isEqual } from 'lodash'; import { ListDomainsDocument, ListDomainsQuery } from '../../graphql/domain.generated'; +import { Entity, EntityType } from '../../types.generated'; +import { GenericEntityProperties } from '../entity/shared/types'; +import usePrevious from '../shared/usePrevious'; +import { useDomainsContext } from './DomainsContext'; +import { useEntityRegistry } from '../useEntityRegistry'; +import EntityRegistry from '../entity/EntityRegistry'; /** * Add an entry to the list domains cache. */ -export const addToListDomainsCache = (client, newDomain, pageSize) => { +export const addToListDomainsCache = (client, newDomain, pageSize, parentDomain?: string) => { // Read the data from our cache for this query. const currData: ListDomainsQuery | null = client.readQuery({ query: ListDomainsDocument, @@ -11,6 +20,7 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { input: { start: 0, count: pageSize, + parentDomain, }, }, }); @@ -25,6 +35,7 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { input: { start: 0, count: pageSize, + parentDomain, }, }, data: { @@ -38,10 +49,39 @@ export const addToListDomainsCache = (client, newDomain, pageSize) => { }); }; +export const updateListDomainsCache = ( + client: ApolloClient, + urn: string, + id: string | undefined, + name: string, + description: string | undefined, + parentDomain?: string, +) => { + addToListDomainsCache( + client, + { + urn, + id: id || null, + type: EntityType.Domain, + properties: { + name, + description: description || null, + }, + ownership: null, + entities: null, + children: null, + dataProducts: null, + parentDomains: null, + }, + 1000, + parentDomain, + ); +}; + /** * Remove an entry from the list domains cache. */ -export const removeFromListDomainsCache = (client, urn, page, pageSize) => { +export const removeFromListDomainsCache = (client, urn, page, pageSize, parentDomain?: string) => { // Read the data from our cache for this query. const currData: ListDomainsQuery | null = client.readQuery({ query: ListDomainsDocument, @@ -49,6 +89,7 @@ export const removeFromListDomainsCache = (client, urn, page, pageSize) => { input: { start: (page - 1) * pageSize, count: pageSize, + parentDomain, }, }, }); @@ -63,6 +104,7 @@ export const removeFromListDomainsCache = (client, urn, page, pageSize) => { input: { start: (page - 1) * pageSize, count: pageSize, + parentDomain, }, }, data: { @@ -75,3 +117,29 @@ export const removeFromListDomainsCache = (client, urn, page, pageSize) => { }, }); }; + +export function useUpdateDomainEntityDataOnChange(entityData: GenericEntityProperties | null, entityType: EntityType) { + const { setEntityData } = useDomainsContext(); + const previousEntityData = usePrevious(entityData); + + useEffect(() => { + if (EntityType.Domain === entityType && !isEqual(entityData, previousEntityData)) { + setEntityData(entityData); + } + }); +} + +export function useSortedDomains(domains?: Array, sortBy?: 'displayName') { + const entityRegistry = useEntityRegistry(); + if (!domains || !sortBy) return domains; + return [...domains].sort((a, b) => { + const nameA = entityRegistry.getDisplayName(EntityType.Domain, a) || ''; + const nameB = entityRegistry.getDisplayName(EntityType.Domain, b) || ''; + return nameA.localeCompare(nameB); + }); +} + +export function getParentDomains(domain: T, entityRegistry: EntityRegistry) { + const props = entityRegistry.getGenericEntityProperties(EntityType.Domain, domain); + return props?.parentDomains?.domains ?? []; +} diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index a28061d37e397..de9556b973940 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -46,6 +46,12 @@ export default class EntityRegistry { return this.entities; } + getEntitiesForSearchRoutes(): Array> { + return this.entities.filter( + (entity) => !GLOSSARY_ENTITY_TYPES.includes(entity.type) && entity.type !== EntityType.Domain, + ); + } + getNonGlossaryEntities(): Array> { return this.entities.filter((entity) => !GLOSSARY_ENTITY_TYPES.includes(entity.type)); } diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index 3b01bee3341ba..b814abc608cef 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -1,5 +1,4 @@ import * as React from 'react'; -import { FolderOutlined } from '@ant-design/icons'; import { Domain, EntityType, SearchResult } from '../../../types.generated'; import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { Preview } from './preview/Preview'; @@ -14,7 +13,7 @@ import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; import { EntityActionItem } from '../shared/entity/EntityActions'; import DataProductsTab from './DataProductsTab/DataProductsTab'; import { EntityProfileTab } from '../shared/constants'; -// import { EntityActionItem } from '../shared/entity/EntityActions'; +import DomainIcon from '../../domain/DomainIcon'; /** * Definition of the DataHub Domain entity. @@ -24,21 +23,26 @@ export class DomainEntity implements Entity { icon = (fontSize: number, styleType: IconStyleType, color?: string) => { if (styleType === IconStyleType.TAB_VIEW) { - return ; + return ; } if (styleType === IconStyleType.HIGHLIGHT) { - return ; + return ; } if (styleType === IconStyleType.SVG) { return ( - + ); } return ( - { useEntityQuery={useGetDomainQuery} useUpdateQuery={undefined} getOverrideProperties={this.getOverridePropertiesFromEntity} - headerDropdownItems={new Set([EntityMenuItems.DELETE])} + headerDropdownItems={new Set([EntityMenuItems.MOVE, EntityMenuItems.DELETE])} headerActionItems={new Set([EntityActionItem.BATCH_ADD_DOMAIN])} isNameEditable tabs={[ @@ -105,11 +109,11 @@ export class DomainEntity implements Entity { renderPreview = (_: PreviewType, data: Domain) => { return ( ); @@ -119,11 +123,11 @@ export class DomainEntity implements Entity { const data = result.entity as Domain; return ( ); diff --git a/datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx b/datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx new file mode 100644 index 0000000000000..6d36964004d64 --- /dev/null +++ b/datahub-web-react/src/app/entity/domain/preview/DomainEntitiesSnippet.tsx @@ -0,0 +1,45 @@ +import { DatabaseOutlined, FileDoneOutlined } from '@ant-design/icons'; +import { VerticalDivider } from '@remirror/react'; +import React from 'react'; +import styled from 'styled-components'; +import { SearchResultFields_Domain_Fragment } from '../../../../graphql/search.generated'; +import { ANTD_GRAY_V2 } from '../../shared/constants'; +import DomainIcon from '../../../domain/DomainIcon'; +import { pluralize } from '../../../shared/textUtil'; + +const Wrapper = styled.div` + color: ${ANTD_GRAY_V2[8]}; + font-size: 12px; + display: flex; + align-items: center; + + svg { + margin-right: 4px; + } +`; + +const StyledDivider = styled(VerticalDivider)` + &&& { + margin: 0 8px; + } +`; + +interface Props { + domain: SearchResultFields_Domain_Fragment; +} + +export default function DomainEntitiesSnippet({ domain }: Props) { + const entityCount = domain.entities?.total || 0; + const subDomainCount = domain.children?.total || 0; + const dataProductCount = domain.dataProducts?.total || 0; + + return ( + + {entityCount} {entityCount === 1 ? 'entity' : 'entities'} + + {subDomainCount} {pluralize(subDomainCount, 'sub-domain')} + + {dataProductCount} {pluralize(dataProductCount, 'data product')} + + ); +} diff --git a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx index 9192e9c1f361d..b0fbf96251a1d 100644 --- a/datahub-web-react/src/app/entity/domain/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/domain/preview/Preview.tsx @@ -1,23 +1,24 @@ import React from 'react'; -import { EntityType, Owner, SearchInsight } from '../../../../types.generated'; +import { Domain, EntityType, Owner, SearchInsight } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; -import { IconStyleType } from '../../Entity'; +import DomainEntitiesSnippet from './DomainEntitiesSnippet'; +import DomainIcon from '../../../domain/DomainIcon'; export const Preview = ({ + domain, urn, name, description, owners, - count, insights, logoComponent, }: { + domain: Domain; urn: string; name: string; description?: string | null; owners?: Array | null; - count?: number | null; insights?: Array | null; logoComponent?: JSX.Element; }): JSX.Element => { @@ -29,11 +30,19 @@ export const Preview = ({ urn={urn} description={description || ''} type={EntityType.Domain} - typeIcon={entityRegistry.getIcon(EntityType.Domain, 14, IconStyleType.ACCENT)} + typeIcon={ + + } owners={owners} insights={insights} logoComponent={logoComponent} - entityCount={count || undefined} + parentEntities={domain.parentDomains?.domains} + snippet={} /> ); }; diff --git a/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx b/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx index c30c4e583456a..051b5408e4d12 100644 --- a/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/glossaryNode/preview/Preview.tsx @@ -28,7 +28,7 @@ export const Preview = ({ owners={owners} logoComponent={} type={EntityType.GlossaryNode} - parentNodes={parentNodes} + parentEntities={parentNodes?.nodes} /> ); }; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx b/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx index 2f8567d144684..d28637bbbdcd6 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/preview/Preview.tsx @@ -39,7 +39,7 @@ export const Preview = ({ type={EntityType.GlossaryTerm} typeIcon={entityRegistry.getIcon(EntityType.GlossaryTerm, 14, IconStyleType.ACCENT)} deprecation={deprecation} - parentNodes={parentNodes} + parentEntities={parentNodes?.nodes} domain={domain} entityTitleSuffix={ View Related Entities diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx new file mode 100644 index 0000000000000..d43b04ec11a16 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/DomainParentSelect.tsx @@ -0,0 +1,108 @@ +import React, { MouseEvent } from 'react'; +import { Select } from 'antd'; +import { CloseCircleFilled } from '@ant-design/icons'; +import styled from 'styled-components'; +import { Domain, EntityType } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import ClickOutside from '../../../shared/ClickOutside'; +import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; +import useParentSelector from './useParentSelector'; +import DomainNavigator from '../../../domain/nestedDomains/domainNavigator/DomainNavigator'; +import { useDomainsContext } from '../../../domain/DomainsContext'; +import ParentEntities from '../../../search/filters/ParentEntities'; +import { getParentDomains } from '../../../domain/utils'; + +const SearchResultContainer = styled.div` + display: flex; + flex-direction: column; + justify-content: center; +`; + +// filter out entity itself and its children +export function filterResultsForMove(entity: Domain, entityUrn: string) { + return ( + entity.urn !== entityUrn && + entity.__typename === 'Domain' && + !entity.parentDomains?.domains.some((node) => node.urn === entityUrn) + ); +} + +interface Props { + selectedParentUrn: string; + setSelectedParentUrn: (parent: string) => void; + isMoving?: boolean; +} + +export default function DomainParentSelect({ selectedParentUrn, setSelectedParentUrn, isMoving }: Props) { + const entityRegistry = useEntityRegistry(); + const { entityData } = useDomainsContext(); + const domainUrn = entityData?.urn; + + const { + searchResults, + searchQuery, + isFocusedOnInput, + selectedParentName, + selectParentFromBrowser, + onSelectParent, + handleSearch, + clearSelectedParent, + setIsFocusedOnInput, + } = useParentSelector({ + entityType: EntityType.Domain, + entityData, + selectedParentUrn, + setSelectedParentUrn, + }); + const domainSearchResultsFiltered = + isMoving && domainUrn + ? searchResults.filter((r) => filterResultsForMove(r.entity as Domain, domainUrn)) + : searchResults; + + function selectDomain(domain: Domain) { + selectParentFromBrowser(domain.urn, entityRegistry.getDisplayName(EntityType.Domain, domain)); + } + + const isShowingDomainNavigator = !searchQuery && isFocusedOnInput; + + const handleFocus = () => setIsFocusedOnInput(true); + const handleClickOutside = () => setIsFocusedOnInput(false); + + const handleClear = (event: MouseEvent) => { + // Prevent, otherwise antd will close the select menu but leaves it focused + event.stopPropagation(); + clearSelectedParent(); + }; + + return ( + + + + + + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx index f0244b0822088..1908b9fd75982 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/EntityDropdown.tsx @@ -21,7 +21,10 @@ import { ANTD_GRAY } from '../constants'; import { useEntityRegistry } from '../../../useEntityRegistry'; import useDeleteEntity from './useDeleteEntity'; import { getEntityProfileDeleteRedirectPath } from '../../../shared/deleteUtils'; -import { isDeleteDisabled } from './utils'; +import { shouldDisplayChildDeletionWarning, isDeleteDisabled, isMoveDisabled } from './utils'; +import { useUserContext } from '../../../context/useUserContext'; +import MoveDomainModal from './MoveDomainModal'; +import { useIsNestedDomainsEnabled } from '../../../useAppConfig'; export enum EntityMenuItems { COPY_URL, @@ -91,8 +94,10 @@ function EntityDropdown(props: Props) { options, } = props; + const me = useUserContext(); const entityRegistry = useEntityRegistry(); const [updateDeprecation] = useUpdateDeprecationMutation(); + const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); const { onDeleteEntity, hasBeenDeleted } = useDeleteEntity( urn, entityType, @@ -141,9 +146,9 @@ function EntityDropdown(props: Props) { const pageUrl = window.location.href; const isGlossaryEntity = entityType === EntityType.GlossaryNode || entityType === EntityType.GlossaryTerm; - const entityHasChildren = !!entityData?.children?.total; - const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; + const isDomainEntity = entityType === EntityType.Domain; const canCreateGlossaryEntity = !!entityData?.privileges?.canManageChildren; + const isDomainMoveHidden = !isNestedDomainsEnabled && isDomainEntity; /** * A default path to redirect to if the entity is deleted. @@ -214,10 +219,10 @@ function EntityDropdown(props: Props) { )} - {menuItems.has(EntityMenuItems.MOVE) && ( + {!isDomainMoveHidden && menuItems.has(EntityMenuItems.MOVE) && ( setIsMoveModalVisible(true)} > @@ -228,17 +233,18 @@ function EntityDropdown(props: Props) { {menuItems.has(EntityMenuItems.DELETE) && ( @@ -274,7 +280,10 @@ function EntityDropdown(props: Props) { refetch={refetchForEntity} /> )} - {isMoveModalVisible && setIsMoveModalVisible(false)} />} + {isMoveModalVisible && isGlossaryEntity && ( + setIsMoveModalVisible(false)} /> + )} + {isMoveModalVisible && isDomainEntity && setIsMoveModalVisible(false)} />} {hasBeenDeleted && !onDelete && deleteRedirectPath && } ); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx new file mode 100644 index 0000000000000..cdbf6fdabf3c9 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveDomainModal.tsx @@ -0,0 +1,102 @@ +import React, { useState } from 'react'; +import styled from 'styled-components/macro'; +import { message, Button, Modal, Typography, Form } from 'antd'; +import { useRefetch } from '../EntityContext'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { useMoveDomainMutation } from '../../../../graphql/domain.generated'; +import DomainParentSelect from './DomainParentSelect'; +import { useHandleMoveDomainComplete } from './useHandleMoveDomainComplete'; +import { useDomainsContext } from '../../../domain/DomainsContext'; +import { EntityType } from '../../../../types.generated'; + +const StyledItem = styled(Form.Item)` + margin-bottom: 0; +`; + +const OptionalWrapper = styled.span` + font-weight: normal; +`; + +interface Props { + onClose: () => void; +} + +function MoveDomainModal(props: Props) { + const { onClose } = props; + const { entityData } = useDomainsContext(); + const domainUrn = entityData?.urn; + const [form] = Form.useForm(); + const entityRegistry = useEntityRegistry(); + const [selectedParentUrn, setSelectedParentUrn] = useState(''); + const refetch = useRefetch(); + + const [moveDomainMutation] = useMoveDomainMutation(); + + const { handleMoveDomainComplete } = useHandleMoveDomainComplete(); + + function moveDomain() { + if (!domainUrn) return; + + moveDomainMutation({ + variables: { + input: { + resourceUrn: domainUrn, + parentDomain: selectedParentUrn || undefined, + }, + }, + }) + .then(() => { + message.loading({ content: 'Updating...', duration: 2 }); + const newParentToUpdate = selectedParentUrn || undefined; + handleMoveDomainComplete(domainUrn, newParentToUpdate); + setTimeout(() => { + message.success({ + content: `Moved ${entityRegistry.getEntityName(EntityType.Domain)}!`, + duration: 2, + }); + refetch(); + }, 2000); + }) + .catch((e) => { + message.destroy(); + message.error({ content: `Failed to move: \n ${e.message || ''}`, duration: 3 }); + }); + onClose(); + } + + return ( + + + + + } + > +
+ + Move To (optional) + + } + > + + + + +
+ + ); +} + +export default MoveDomainModal; diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx index c1f1d135a0e30..46a8a645d26c0 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx @@ -1,5 +1,6 @@ -import React, { useState, useEffect } from 'react'; +import React from 'react'; import { Select } from 'antd'; +import { EntityType, GlossaryNode, SearchResult } from '../../../../types.generated'; import { useTranslation } from 'react-i18next'; import { useGetSearchResultsLazyQuery } from '../../../../graphql/search.generated'; import { EntityType, GlossaryNode } from '../../../../types.generated'; @@ -8,6 +9,7 @@ import { useEntityData } from '../EntityContext'; import ClickOutside from '../../../shared/ClickOutside'; import GlossaryBrowser from '../../../glossary/GlossaryBrowser/GlossaryBrowser'; import { BrowserWrapper } from '../../../shared/tags/AddTagsTermsModal'; +import useParentSelector from './useParentSelector'; // filter out entity itself and its children export function filterResultsForMove(entity: GlossaryNode, entityUrn: string) { @@ -26,61 +28,30 @@ interface Props { function NodeParentSelect(props: Props) { const { selectedParentUrn, setSelectedParentUrn, isMoving } = props; - const [selectedParentName, setSelectedParentName] = useState(''); - const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); - const [searchQuery, setSearchQuery] = useState(''); const entityRegistry = useEntityRegistry(); const { t } = useTranslation(); const { entityData, urn: entityDataUrn, entityType } = useEntityData(); - const [nodeSearch, { data: nodeData }] = useGetSearchResultsLazyQuery(); - let nodeSearchResults = nodeData?.search?.searchResults || []; - if (isMoving) { - nodeSearchResults = nodeSearchResults.filter((r) => - filterResultsForMove(r.entity as GlossaryNode, entityDataUrn), - ); - } - - useEffect(() => { - if (entityData && selectedParentUrn === entityDataUrn) { - const displayName = entityRegistry.getDisplayName(EntityType.GlossaryNode, entityData); - setSelectedParentName(displayName); - } - }, [entityData, entityRegistry, selectedParentUrn, entityDataUrn]); - - function handleSearch(text: string) { - setSearchQuery(text); - nodeSearch({ - variables: { - input: { - type: EntityType.GlossaryNode, - query: text, - start: 0, - count: 5, - }, - }, - }); - } + const { + searchResults, + searchQuery, + isFocusedOnInput, + selectedParentName, + selectParentFromBrowser, + onSelectParent, + handleSearch, + clearSelectedParent, + setIsFocusedOnInput, + } = useParentSelector({ + entityType: EntityType.GlossaryNode, + entityData, + selectedParentUrn, + setSelectedParentUrn, + }); - function onSelectParentNode(parentNodeUrn: string) { - const selectedNode = nodeSearchResults.find((result) => result.entity.urn === parentNodeUrn); - if (selectedNode) { - setSelectedParentUrn(parentNodeUrn); - const displayName = entityRegistry.getDisplayName(selectedNode.entity.type, selectedNode.entity); - setSelectedParentName(displayName); - } - } - - function clearSelectedParent() { - setSelectedParentUrn(''); - setSelectedParentName(''); - setSearchQuery(''); - } - - function selectNodeFromBrowser(urn: string, displayName: string) { - setIsFocusedOnInput(false); - setSelectedParentUrn(urn); - setSelectedParentName(displayName); + let nodeSearchResults: SearchResult[] = []; + if (isMoving) { + nodeSearchResults = searchResults.filter((r) => filterResultsForMove(r.entity as GlossaryNode, entityDataUrn)); } const isShowingGlossaryBrowser = !searchQuery && isFocusedOnInput; @@ -93,7 +64,7 @@ function NodeParentSelect(props: Props) { allowClear filterOption={false} value={selectedParentName} - onSelect={onSelectParentNode} + onSelect={onSelectParent} onSearch={handleSearch} onClear={clearSelectedParent} onFocus={() => setIsFocusedOnInput(true)} @@ -109,7 +80,7 @@ function NodeParentSelect(props: Props) { diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx index 037c96f24c29d..3e1e8349d5a2a 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useDeleteEntity.tsx @@ -7,6 +7,7 @@ import { getDeleteEntityMutation } from '../../../shared/deleteUtils'; import analytics, { EventType } from '../../../analytics'; import { useGlossaryEntityData } from '../GlossaryEntityContext'; import { getParentNodeToUpdate, updateGlossarySidebar } from '../../../glossary/utils'; +import { useHandleDeleteDomain } from './useHandleDeleteDomain'; /** * Performs the flow for deleting an entity of a given type. @@ -27,6 +28,7 @@ function useDeleteEntity( const [hasBeenDeleted, setHasBeenDeleted] = useState(false); const entityRegistry = useEntityRegistry(); const { isInGlossaryContext, urnsToUpdate, setUrnsToUpdate } = useGlossaryEntityData(); + const { handleDeleteDomain } = useHandleDeleteDomain({ entityData, urn }); const maybeDeleteEntity = getDeleteEntityMutation(type)(); const deleteEntity = (maybeDeleteEntity && maybeDeleteEntity[0]) || undefined; @@ -49,6 +51,11 @@ function useDeleteEntity( duration: 2, }); } + + if (entityData.type === EntityType.Domain) { + handleDeleteDomain(); + } + setTimeout( () => { setHasBeenDeleted(true); diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts new file mode 100644 index 0000000000000..ebbb8f9968a6a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleDeleteDomain.ts @@ -0,0 +1,27 @@ +import { useApolloClient } from '@apollo/client'; +import { GenericEntityProperties } from '../types'; +import { removeFromListDomainsCache } from '../../../domain/utils'; +import { useDomainsContext } from '../../../domain/DomainsContext'; + +interface DeleteDomainProps { + entityData: GenericEntityProperties; + urn: string; +} + +export function useHandleDeleteDomain({ entityData, urn }: DeleteDomainProps) { + const client = useApolloClient(); + const { parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); + + const handleDeleteDomain = () => { + if (entityData.parentDomains && entityData.parentDomains.domains.length > 0) { + const parentDomainUrn = entityData.parentDomains.domains[0].urn; + + removeFromListDomainsCache(client, urn, 1, 1000, parentDomainUrn); + setParentDomainsToUpdate([...parentDomainsToUpdate, parentDomainUrn]); + } else { + removeFromListDomainsCache(client, urn, 1, 1000); + } + }; + + return { handleDeleteDomain }; +} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts new file mode 100644 index 0000000000000..81f19331e18b7 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useHandleMoveDomainComplete.ts @@ -0,0 +1,40 @@ +import { useApolloClient } from '@apollo/client'; +import { removeFromListDomainsCache, updateListDomainsCache } from '../../../domain/utils'; +import { useDomainsContext } from '../../../domain/DomainsContext'; +import { Domain } from '../../../../types.generated'; +import analytics from '../../../analytics/analytics'; +import { EventType } from '../../../analytics'; + +export function useHandleMoveDomainComplete() { + const client = useApolloClient(); + const { entityData, parentDomainsToUpdate, setParentDomainsToUpdate } = useDomainsContext(); + + const handleMoveDomainComplete = (urn: string, newParentUrn?: string) => { + if (!entityData) return; + + const domain = entityData as Domain; + const oldParentUrn = domain.parentDomains?.domains.length ? domain.parentDomains.domains[0].urn : undefined; + + analytics.event({ + type: EventType.MoveDomainEvent, + oldParentDomainUrn: oldParentUrn, + parentDomainUrn: newParentUrn, + }); + + removeFromListDomainsCache(client, urn, 1, 1000, oldParentUrn); + updateListDomainsCache( + client, + domain.urn, + undefined, + domain.properties?.name ?? '', + domain.properties?.description ?? '', + newParentUrn, + ); + const newParentDomainsToUpdate = [...parentDomainsToUpdate]; + if (oldParentUrn) newParentDomainsToUpdate.push(oldParentUrn); + if (newParentUrn) newParentDomainsToUpdate.push(newParentUrn); + setParentDomainsToUpdate(newParentDomainsToUpdate); + }; + + return { handleMoveDomainComplete }; +} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts new file mode 100644 index 0000000000000..32b5d8ca790cc --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/useParentSelector.ts @@ -0,0 +1,76 @@ +import { useEffect, useState } from 'react'; +import { useGetSearchResultsLazyQuery } from '../../../../graphql/search.generated'; +import { EntityType } from '../../../../types.generated'; +import { useEntityRegistry } from '../../../useEntityRegistry'; +import { GenericEntityProperties } from '../types'; + +interface Props { + entityType: EntityType; + entityData: GenericEntityProperties | null; + selectedParentUrn: string; + setSelectedParentUrn: (parent: string) => void; +} + +export default function useParentSelector({ entityType, entityData, selectedParentUrn, setSelectedParentUrn }: Props) { + const [selectedParentName, setSelectedParentName] = useState(); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); + const [searchQuery, setSearchQuery] = useState(''); + const entityRegistry = useEntityRegistry(); + + const [search, { data }] = useGetSearchResultsLazyQuery(); + const searchResults = data?.search?.searchResults || []; + + useEffect(() => { + if (entityData && selectedParentUrn === entityData.urn) { + const displayName = entityRegistry.getDisplayName(entityType, entityData); + setSelectedParentName(displayName); + } + }, [entityData, entityRegistry, selectedParentUrn, entityData?.urn, entityType]); + + function handleSearch(text: string) { + setSearchQuery(text); + search({ + variables: { + input: { + type: entityType, + query: text, + start: 0, + count: 5, + }, + }, + }); + } + + function onSelectParent(parentUrn: string) { + const selectedParent = searchResults.find((result) => result.entity.urn === parentUrn); + if (selectedParent) { + setSelectedParentUrn(parentUrn); + const displayName = entityRegistry.getDisplayName(selectedParent.entity.type, selectedParent.entity); + setSelectedParentName(displayName); + } + } + + function clearSelectedParent() { + setSelectedParentUrn(''); + setSelectedParentName(undefined); + setSearchQuery(''); + } + + function selectParentFromBrowser(urn: string, displayName: string) { + setIsFocusedOnInput(false); + setSelectedParentUrn(urn); + setSelectedParentName(displayName); + } + + return { + searchQuery, + searchResults, + isFocusedOnInput, + selectedParentName, + onSelectParent, + handleSearch, + setIsFocusedOnInput, + selectParentFromBrowser, + clearSelectedParent, + }; +} diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts b/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts index 9e3d14cfd32e1..0a4c2c34441a4 100644 --- a/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts +++ b/datahub-web-react/src/app/entity/shared/EntityDropdown/utils.ts @@ -1,7 +1,11 @@ -import { EntityType } from '../../../../types.generated'; +import { EntityType, PlatformPrivileges } from '../../../../types.generated'; import { GenericEntityProperties } from '../types'; -export function isDeleteDisabled(entityType: EntityType, entityData: GenericEntityProperties | null) { +export function isDeleteDisabled( + entityType: EntityType, + entityData: GenericEntityProperties | null, + platformPrivileges: PlatformPrivileges | null | undefined, +) { if (entityType === EntityType.GlossaryTerm || entityType === EntityType.GlossaryNode) { const entityHasChildren = !!entityData?.children?.total; const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; @@ -11,5 +15,47 @@ export function isDeleteDisabled(entityType: EntityType, entityData: GenericEnti if (entityType === EntityType.DataProduct) { return false; // TODO: update with permissions } + if (entityType === EntityType.Domain) { + const entityHasChildren = !!entityData?.children?.total; + const canManageDomains = !!platformPrivileges?.manageDomains; + const canDeleteDomainEntity = !entityHasChildren && canManageDomains; + return !canDeleteDomainEntity; + } + return false; +} + +export function isMoveDisabled( + entityType: EntityType, + entityData: GenericEntityProperties | null, + platformPrivileges: PlatformPrivileges | null | undefined, +) { + if (entityType === EntityType.GlossaryTerm || entityType === EntityType.GlossaryNode) { + const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; + return !canManageGlossaryEntity; + } + if (entityType === EntityType.Domain) { + const canManageDomains = !!platformPrivileges?.manageDomains; + return !canManageDomains; + } + return false; +} + +export function shouldDisplayChildDeletionWarning( + entityType: EntityType, + entityData: GenericEntityProperties | null, + platformPrivileges: PlatformPrivileges | null | undefined, +) { + if (entityType === EntityType.GlossaryTerm || entityType === EntityType.GlossaryNode) { + const entityHasChildren = !!entityData?.children?.total; + const canManageGlossaryEntity = !!entityData?.privileges?.canManageEntity; + const hasTooltip = entityHasChildren && canManageGlossaryEntity; + return hasTooltip; + } + if (entityType === EntityType.Domain) { + const entityHasChildren = !!entityData?.children?.total; + const canManageDomains = !!platformPrivileges?.manageDomains; + const hasTooltip = entityHasChildren && canManageDomains; + return hasTooltip; + } return false; } diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx index 8259c419eef53..52b057ec05b09 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx @@ -8,7 +8,7 @@ import { FacetMetadata, SearchAcrossEntitiesInput, } from '../../../../../../types.generated'; -import { UnionType } from '../../../../../search/utils/constants'; +import { DEGREE_FILTER_NAME, UnionType } from '../../../../../search/utils/constants'; import { SearchCfg } from '../../../../../../conf'; import { EmbeddedListSearchResults } from './EmbeddedListSearchResults'; import EmbeddedListSearchHeader from './EmbeddedListSearchHeader'; @@ -28,6 +28,7 @@ import { import { useEntityContext } from '../../../EntityContext'; import { EntityActionProps } from './EntitySearchResults'; import { useUserContext } from '../../../../../context/useUserContext'; +import analytics, { EventType } from '../../../../../analytics'; const Container = styled.div` display: flex; @@ -268,6 +269,20 @@ export const EmbeddedListSearch = ({ const finalFacets = (fixedFilters && removeFixedFiltersFromFacets(fixedFilters, data?.facets || [])) || data?.facets; + // used for logging impact anlaysis events + const degreeFilter = filters.find((filter) => filter.field === DEGREE_FILTER_NAME); + + // we already have some lineage logging through Tab events, but this adds additional context, particularly degree + if (!loading && (degreeFilter?.values?.length || 0) > 0) { + analytics.event({ + type: EventType.SearchAcrossLineageResultsViewEvent, + query, + page, + total: data?.total || 0, + maxDegree: degreeFilter?.values?.sort()?.reverse()[0] || '1', + }); + } + return ( {error && } diff --git a/datahub-web-react/src/app/entity/shared/constants.ts b/datahub-web-react/src/app/entity/shared/constants.ts index ede49772217c4..ed626c616c520 100644 --- a/datahub-web-react/src/app/entity/shared/constants.ts +++ b/datahub-web-react/src/app/entity/shared/constants.ts @@ -21,6 +21,7 @@ export const ANTD_GRAY = { }; export const ANTD_GRAY_V2 = { + 1: '#F8F9Fa', 2: '#F3F5F6', 5: '#DDE0E4', 6: '#B2B8BD', diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx index 1b5a6ef6177b0..660cce681e3b6 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx @@ -46,6 +46,7 @@ import { LINEAGE_GRAPH_TIME_FILTER_ID, } from '../../../../onboarding/config/LineageGraphOnboardingConfig'; import { useAppConfig } from '../../../../useAppConfig'; +import { useUpdateDomainEntityDataOnChange } from '../../../../domain/utils'; type Props = { urn: string; @@ -214,6 +215,7 @@ export const EntityProfile = ({ useGetDataForProfile({ urn, entityType, useEntityQuery, getOverrideProperties }); useUpdateGlossaryEntityDataOnChange(entityData, entityType); + useUpdateDomainEntityDataOnChange(entityData, entityType); const maybeUpdateEntity = useUpdateQuery?.({ onCompleted: () => refetch(), diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx index ec06ad2a11267..e128068a885fd 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx @@ -35,17 +35,27 @@ function EntityName(props: Props) { const { urn, entityType, entityData } = useEntityData(); const entityName = entityData ? entityRegistry.getDisplayName(entityType, entityData) : ''; const [updatedName, setUpdatedName] = useState(entityName); + const [isEditing, setIsEditing] = useState(false); useEffect(() => { setUpdatedName(entityName); }, [entityName]); - const [updateName] = useUpdateNameMutation(); + const [updateName, { loading: isMutatingName }] = useUpdateNameMutation(); - const handleSaveName = (name: string) => { + const handleStartEditing = () => { + setIsEditing(true); + }; + + const handleChangeName = (name: string) => { + if (name === entityName) { + setIsEditing(false); + return; + } setUpdatedName(name); updateName({ variables: { input: { name, urn } } }) .then(() => { + setIsEditing(false); message.success({ content: t('crud.success.updateWithName', { name: t('common.name') }), duration: 2 }); refetch(); if (isInGlossaryContext) { @@ -67,13 +77,19 @@ function EntityName(props: Props) { return ( <> {isNameEditable ? ( - + {updatedName} ) : ( - - {entityData && entityRegistry.getDisplayName(entityType, entityData)} - + {entityName} )} ); diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx index b7d554d74cc26..16a5de35ac4a7 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentContainer.tsx @@ -54,6 +54,7 @@ function PlatformContentContainer() { parentContainers={entityData?.parentContainers?.containers} parentContainersRef={contentRef} areContainersTruncated={isContentTruncated} + parentEntities={entityData?.parentDomains?.domains} /> ); } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx index 51a422ba93418..1090dac501d0b 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx @@ -2,15 +2,16 @@ import React from 'react'; import styled from 'styled-components'; import { Typography, Image } from 'antd'; import { Maybe } from 'graphql/jsutils/Maybe'; -import { Container, GlossaryNode } from '../../../../../../../types.generated'; +import { Container, Entity } from '../../../../../../../types.generated'; import { ANTD_GRAY } from '../../../../constants'; import ContainerLink from './ContainerLink'; -import ParentNodesView, { +import { StyledRightOutlined, ParentNodesWrapper as ParentContainersWrapper, Ellipsis, StyledTooltip, } from './ParentNodesView'; +import ParentEntities from '../../../../../../search/filters/ParentEntities'; const LogoIcon = styled.span` display: flex; @@ -75,14 +76,14 @@ interface Props { typeIcon?: JSX.Element; entityType?: string; parentContainers?: Maybe[] | null; - parentNodes?: GlossaryNode[] | null; + parentEntities?: Entity[] | null; parentContainersRef: React.RefObject; areContainersTruncated: boolean; } function PlatformContentView(props: Props) { const { - parentNodes, + parentEntities, platformName, platformLogoUrl, platformNames, @@ -103,7 +104,7 @@ function PlatformContentView(props: Props) { {typeIcon && {typeIcon}} {entityType} - {(!!platformName || !!instanceId || !!parentContainers?.length || !!parentNodes?.length) && ( + {(!!platformName || !!instanceId || !!parentContainers?.length || !!parentEntities?.length) && ( )} {platformName && ( @@ -146,7 +147,7 @@ function PlatformContentView(props: Props) { {directParentContainer && } - + ); } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx index 201c8a8ea67f4..f4b5e10993f04 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx @@ -3,14 +3,16 @@ import { Button, Form, message, Modal, Select } from 'antd'; import { useTranslation } from 'react-i18next'; import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/search.generated'; -import { Entity, EntityType } from '../../../../../../../types.generated'; +import { Domain, Entity, EntityType } from '../../../../../../../types.generated'; import { useBatchSetDomainMutation } from '../../../../../../../graphql/mutations.generated'; import { useEntityRegistry } from '../../../../../../useEntityRegistry'; import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener'; -import { useGetRecommendations } from '../../../../../../shared/recommendation'; import { DomainLabel } from '../../../../../../shared/DomainLabel'; import { handleBatchError } from '../../../../utils'; import { tagRender } from '../tagRenderer'; +import { BrowserWrapper } from '../../../../../../shared/tags/AddTagsTermsModal'; +import DomainNavigator from '../../../../../../domain/nestedDomains/domainNavigator/DomainNavigator'; +import ClickOutside from '../../../../../../shared/ClickOutside'; type Props = { urns: string[]; @@ -30,6 +32,7 @@ type SelectedDomain = { export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOkOverride, titleOverride }: Props) => { const { t } = useTranslation(); const entityRegistry = useEntityRegistry(); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); const [inputValue, setInputValue] = useState(''); const [selectedDomain, setSelectedDomain] = useState( defaultValue @@ -44,8 +47,8 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk const domainSearchResults = domainSearchData?.search?.searchResults?.map((searchResult) => searchResult.entity) || []; const [batchSetDomainMutation] = useBatchSetDomainMutation(); - const [recommendedData] = useGetRecommendations([EntityType.Domain]); const inputEl = useRef(null); + const isShowingDomainNavigator = !inputValue && isFocusedOnInput; const onModalClose = () => { setInputValue(''); @@ -76,7 +79,7 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk ); }; - const domainResult = !inputValue || inputValue.length === 0 ? recommendedData : domainSearchResults; + const domainResult = !inputValue || inputValue.length === 0 ? [] : domainSearchResults; const domainSearchOptions = domainResult?.map((result) => { return renderSearchResult(result); @@ -97,6 +100,15 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk } }; + function selectDomainFromBrowser(domain: Domain) { + setIsFocusedOnInput(false); + setSelectedDomain({ + displayName: entityRegistry.getDisplayName(EntityType.Domain, domain), + type: EntityType.Domain, + urn: domain.urn, + }); + } + const onDeselectDomain = () => { setInputValue(''); setSelectedDomain(undefined); @@ -160,6 +172,11 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk setInputValue(''); } + function handleCLickOutside() { + // delay closing the domain navigator so we don't get a UI "flash" between showing search results and navigator + setTimeout(() => setIsFocusedOnInput(false), 0); + } + return (
- onSelectDomain(domainUrn)} - onDeselect={onDeselectDomain} - onSearch={(value: string) => { - // eslint-disable-next-line react/prop-types - handleSearch(value.trim()); - // eslint-disable-next-line react/prop-types - setInputValue(value.trim()); - }} - ref={inputEl} - value={selectValue} - tagRender={tagRender} - onBlur={handleBlur} - > - {domainSearchOptions} - + onSelect={(domainUrn: any) => onSelectDomain(domainUrn)} + onDeselect={onDeselectDomain} + onSearch={(value: string) => { + // eslint-disable-next-line react/prop-types + handleSearch(value.trim()); + // eslint-disable-next-line react/prop-types + setInputValue(value.trim()); + }} + ref={inputEl} + value={selectValue} + tagRender={tagRender} + onBlur={handleBlur} + onFocus={() => setIsFocusedOnInput(true)} + dropdownStyle={isShowingDomainNavigator ? { display: 'none' } : {}} + > + {domainSearchOptions} + + + + +
diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index 6bf8ae0554398..9adc112f5652c 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -37,6 +37,7 @@ import { FabricType, BrowsePathV2, DataJobInputOutput, + ParentDomainsResult, } from '../../../types.generated'; import { FetchedEntity } from '../../lineage/types'; @@ -66,6 +67,7 @@ export type EntitySubHeaderSection = { export type GenericEntityProperties = { urn?: string; + type?: EntityType; name?: Maybe; properties?: Maybe<{ description?: Maybe; @@ -99,6 +101,7 @@ export type GenericEntityProperties = { status?: Maybe; deprecation?: Maybe; parentContainers?: Maybe; + parentDomains?: Maybe; children?: Maybe; parentNodes?: Maybe; isAChildren?: Maybe; diff --git a/datahub-web-react/src/app/entity/user/UserEditProfileModal.tsx b/datahub-web-react/src/app/entity/user/UserEditProfileModal.tsx index 931b2751587ca..377162d8606d2 100644 --- a/datahub-web-react/src/app/entity/user/UserEditProfileModal.tsx +++ b/datahub-web-react/src/app/entity/user/UserEditProfileModal.tsx @@ -140,6 +140,7 @@ export default function UserEditProfileModal({ visible, onClose, onSave, editMod placeholder="John Smith" value={data.name} onChange={(event) => setData({ ...data, name: event.target.value })} + disabled={readOnlyModeEnabled} /> setData({ ...data, title: event.target.value })} + disabled={readOnlyModeEnabled} /> setData({ ...data, team: event.target.value })} + disabled={readOnlyModeEnabled} /> setData({ ...data, email: event.target.value })} + disabled={readOnlyModeEnabled} /> setData({ ...data, slack: event.target.value })} + disabled={readOnlyModeEnabled} /> setData({ ...data, phone: event.target.value })} + disabled={readOnlyModeEnabled} /> diff --git a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx index 23c925dc40547..a1df2b9124f28 100644 --- a/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx +++ b/datahub-web-react/src/app/glossary/BusinessGlossaryPage.tsx @@ -39,12 +39,6 @@ const MainContentWrapper = styled.div` flex-direction: column; `; -export const BrowserWrapper = styled.div<{ width: number }>` - max-height: 100%; - width: ${(props) => props.width}px; - min-width: ${(props) => props.width}px; -`; - export const MAX_BROWSER_WIDTH = 500; export const MIN_BROWSWER_WIDTH = 200; diff --git a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx index 0bdcbf707ce09..2d620fb06df38 100644 --- a/datahub-web-react/src/app/glossary/GlossarySidebar.tsx +++ b/datahub-web-react/src/app/glossary/GlossarySidebar.tsx @@ -1,14 +1,8 @@ import React, { useState } from 'react'; -import styled from 'styled-components/macro'; import GlossarySearch from './GlossarySearch'; import GlossaryBrowser from './GlossaryBrowser/GlossaryBrowser'; import { ProfileSidebarResizer } from '../entity/shared/containers/profile/sidebar/ProfileSidebarResizer'; - -const BrowserWrapper = styled.div<{ width: number }>` - max-height: 100%; - width: ${(props) => props.width}px; - min-width: ${(props) => props.width}px; -`; +import { SidebarWrapper } from '../shared/sidebar/components'; export const MAX_BROWSER_WIDTH = 500; export const MIN_BROWSWER_WIDTH = 200; @@ -18,10 +12,10 @@ export default function GlossarySidebar() { return ( <> - + - + setBrowserWith(Math.min(Math.max(width, MIN_BROWSWER_WIDTH), MAX_BROWSER_WIDTH)) diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 8d41c3533575a..dba8e8bb1dce6 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -27,6 +27,8 @@ import powerbiLogo from '../../../../images/powerbilogo.png'; import modeLogo from '../../../../images/modelogo.png'; import databricksLogo from '../../../../images/databrickslogo.png'; import verticaLogo from '../../../../images/verticalogo.png'; +import mlflowLogo from '../../../../images/mlflowlogo.png'; +import dynamodbLogo from '../../../../images/dynamodblogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -43,6 +45,8 @@ export const DBT = 'dbt'; export const DBT_URN = `urn:li:dataPlatform:${DBT}`; export const DRUID = 'druid'; export const DRUID_URN = `urn:li:dataPlatform:${DRUID}`; +export const DYNAMODB = 'dynamodb'; +export const DYNAMODB_URN = `urn:li:dataPlatform:${DYNAMODB}`; export const ELASTICSEARCH = 'elasticsearch'; export const ELASTICSEARCH_URN = `urn:li:dataPlatform:${ELASTICSEARCH}`; export const FEAST = 'feast'; @@ -61,6 +65,8 @@ export const MARIA_DB = 'mariadb'; export const MARIA_DB_URN = `urn:li:dataPlatform:${MARIA_DB}`; export const METABASE = 'metabase'; export const METABASE_URN = `urn:li:dataPlatform:${METABASE}`; +export const MLFLOW = 'mlflow'; +export const MLFLOW_URN = `urn:li:dataPlatform:${MLFLOW}`; export const MODE = 'mode'; export const MODE_URN = `urn:li:dataPlatform:${MODE}`; export const MONGO_DB = 'mongodb'; @@ -107,6 +113,7 @@ export const PLATFORM_URN_TO_LOGO = { [CLICKHOUSE_URN]: clickhouseLogo, [DBT_URN]: dbtLogo, [DRUID_URN]: druidLogo, + [DYNAMODB_URN]: dynamodbLogo, [ELASTICSEARCH_URN]: elasticsearchLogo, [FEAST_URN]: feastLogo, [GLUE_URN]: glueLogo, @@ -115,6 +122,7 @@ export const PLATFORM_URN_TO_LOGO = { [LOOKER_URN]: lookerLogo, [MARIA_DB_URN]: mariadbLogo, [METABASE_URN]: metabaseLogo, + [MLFLOW_URN]: mlflowLogo, [MODE_URN]: modeLogo, [MONGO_DB_URN]: mongodbLogo, [MSSQL_URN]: mssqlLogo, diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 13643c58f72e1..1bd5b6f1f768b 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -125,6 +125,13 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mongodb/", "recipe": "source:\n type: mongodb\n config:\n # Coordinates\n connect_uri: # Your MongoDB connect URI, e.g. \"mongodb://localhost\"\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${MONGO_USERNAME}\" # Your MongoDB username, e.g. admin\n password: \"${MONGO_PASSWORD}\" # Your MongoDB password, e.g. password_01\n\n # Options (recommended)\n enableSchemaInference: True\n useRandomSampling: True\n maxSchemaSize: 300" }, + { + "urn": "urn:li:dataPlatform:dynamodb", + "name": "dynamodb", + "displayName": "DynamoDB", + "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", + "recipe": "source:\n type: dynamodb\n config:\n platform_instance: \"AWS_ACCOUNT_ID\"\n aws_access_key_id : '${AWS_ACCESS_KEY_ID}'\n aws_secret_access_key : '${AWS_SECRET_ACCESS_KEY}'\n # User could use the below option to provide a list of primary keys of a table in dynamodb format,\n # those items from given primary keys will be included when we scan the table.\n # For each table we can retrieve up to 16 MB of data, which can contain as many as 100 items.\n # We'll enforce the the primary keys list size not to exceed 100\n # The total items we'll try to retrieve in these two scenarios:\n # 1. If user don't specify include_table_item: we'll retrieve up to 100 items\n # 2. If user specifies include_table_item: we'll retrieve up to 100 items plus user specified items in\n # the table, with a total not more than 200 items\n # include_table_item:\n # table_name:\n # [\n # {\n # 'partition_key_name': { 'attribute_type': 'attribute_value' },\n # 'sort_key_name': { 'attribute_type': 'attribute_value' },\n # },\n # ]" + }, { "urn": "urn:li:dataPlatform:glue", "name": "glue", @@ -174,6 +181,13 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/", "recipe": "source:\n type: metabase\n config:\n # Coordinates\n connect_uri:\n\n # Credentials\n username: root\n password: example" }, + { + "urn": "urn:li:dataPlatform:mlflow", + "name": "mlflow", + "displayName": "MLflow", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mlflow/", + "recipe": "source:\n type: mlflow\n config:\n tracking_uri: tracking_uri" + }, { "urn": "urn:li:dataPlatform:mode", "name": "mode", diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx index 5476a68d6b928..3eddb872defdc 100644 --- a/datahub-web-react/src/app/lineage/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineage/LineageExplorer.tsx @@ -18,6 +18,7 @@ import { SHOW_COLUMNS_URL_PARAMS, useIsShowColumnsMode } from './utils/useIsShow import { ErrorSection } from '../shared/error/ErrorSection'; import usePrevious from '../shared/usePrevious'; import { useGetLineageTimeParams } from './utils/useGetLineageTimeParams'; +import analytics, { EventType } from '../analytics'; const DEFAULT_DISTANCE_FROM_TOP = 106; @@ -87,7 +88,13 @@ export default function LineageExplorer({ urn, type }: Props) { // they should be added to the dependency array below. useEffect(() => { setAsyncEntities({}); - }, [isHideSiblingMode, startTimeMillis, endTimeMillis]); + // this can also be our hook for emitting the tracking event + + analytics.event({ + type: EventType.VisualLineageViewEvent, + entityType: entityData?.type, + }); + }, [isHideSiblingMode, startTimeMillis, endTimeMillis, entityData?.type]); useEffect(() => { if (showColumns) { @@ -185,6 +192,10 @@ export default function LineageExplorer({ urn, type }: Props) { onLineageExpand={(asyncData: EntityAndType) => { resetAsyncEntity(asyncData.entity.urn); maybeAddAsyncLoadedEntity(asyncData); + analytics.event({ + type: EventType.VisualLineageExpandGraphEvent, + targetEntityType: asyncData?.type, + }); }} refetchCenterNode={() => { refetch().then(() => { diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index dfcf22ec8b517..67474c4ef3103 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import { Link } from 'react-router-dom'; import { Form, Select, Tag, Tooltip, Typography } from 'antd'; import styled from 'styled-components/macro'; @@ -10,7 +10,7 @@ import { useGetSearchResultsForMultipleLazyQuery, useGetSearchResultsLazyQuery, } from '../../../graphql/search.generated'; -import { ResourceFilter, PolicyType, EntityType } from '../../../types.generated'; +import { ResourceFilter, PolicyType, EntityType, Domain } from '../../../types.generated'; import { convertLegacyResourceFilter, createCriterionValue, @@ -22,6 +22,9 @@ import { mapResourceTypeToPrivileges, setFieldValues, } from './policyUtils'; +import DomainNavigator from '../../domain/nestedDomains/domainNavigator/DomainNavigator'; +import { BrowserWrapper } from '../../shared/tags/AddTagsTermsModal'; +import ClickOutside from '../../shared/ClickOutside'; type Props = { policyType: PolicyType; @@ -57,6 +60,8 @@ export default function PolicyPrivilegeForm({ }: Props) { const { t } = useTranslation(); const entityRegistry = useEntityRegistry(); + const [domainInputValue, setDomainInputValue] = useState(''); + const [isFocusedOnInput, setIsFocusedOnInput] = useState(false); // Configuration used for displaying options const { @@ -100,6 +105,7 @@ export default function PolicyPrivilegeForm({ const resourceSelectValue = resourceEntities.map((criterionValue) => criterionValue.value); const domainSelectValue = getFieldValues(resources.filter, 'DOMAIN').map((criterionValue) => criterionValue.value); const privilegesSelectValue = privileges; + const isShowingDomainNavigator = !domainInputValue && isFocusedOnInput; // Construct privilege options for dropdown const platformPrivileges = policiesConfig?.platformPrivileges || []; @@ -195,13 +201,14 @@ export default function PolicyPrivilegeForm({ }; // When a domain is selected, add its urn to the list of domains - const onSelectDomain = (domain) => { + const onSelectDomain = (domainUrn, domainObj?: Domain) => { const filter = resources.filter || { criteria: [], }; + const domainEntity = domainObj || getEntityFromSearchResults(domainSearchResults, domainUrn); const updatedFilter = setFieldValues(filter, 'DOMAIN', [ ...domains, - createCriterionValueWithEntity(domain, getEntityFromSearchResults(domainSearchResults, domain) || null), + createCriterionValueWithEntity(domainUrn, domainEntity || null), ]); setResources({ ...resources, @@ -209,6 +216,11 @@ export default function PolicyPrivilegeForm({ }); }; + function selectDomainFromBrowser(domain: Domain) { + onSelectDomain(domain.urn, domain); + setIsFocusedOnInput(false); + } + // When a domain is deselected, remove its urn from the list of domains const onDeselectDomain = (domain) => { const filter = resources.filter || { @@ -245,6 +257,7 @@ export default function PolicyPrivilegeForm({ // Handle domain search, if the domain type has an associated EntityType mapping. const handleDomainSearch = (text: string) => { const trimmedText: string = text.trim(); + setDomainInputValue(trimmedText); searchDomains({ variables: { input: { @@ -278,6 +291,15 @@ export default function PolicyPrivilegeForm({ : displayStr; }; + function handleCLickOutside() { + // delay closing the domain navigator so we don't get a UI "flash" between showing search results and navigator + setTimeout(() => setIsFocusedOnInput(false), 0); + } + + function handleBlur() { + setDomainInputValue(''); + } + return ( {showResourceFilterInput && ( @@ -355,37 +377,45 @@ export default function PolicyPrivilegeForm({ )} {showResourceFilterInput && ( - {t('common.domain')}}> + {t('crud.selectWithName', { name: t('common.domain') })}}> }, - }} + {...{ + i18nKey: 'permissions.domainDescription', + components: { bold: }, + }} /> - + + + + + + )} {t('common.privileges')}}> diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index a7572420cf14c..bdbf5e4b622f9 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -15,11 +15,11 @@ import { CorpUser, Deprecation, Domain, - ParentNodesResult, EntityPath, DataProduct, Health, EntityType, + Entity, } from '../../types.generated'; import TagTermGroup from '../shared/tags/TagTermGroup'; import { ANTD_GRAY } from '../entity/shared/constants'; @@ -194,7 +194,7 @@ interface Props { // how the listed node is connected to the source node degree?: number; parentContainers?: ParentContainersResult | null; - parentNodes?: ParentNodesResult | null; + parentEntities?: Entity[] | null; previewType?: Maybe; paths?: EntityPath[]; health?: Health[]; @@ -234,7 +234,7 @@ export default function DefaultPreviewCard({ onClick, degree, parentContainers, - parentNodes, + parentEntities, platforms, logoUrls, previewType, @@ -287,7 +287,7 @@ export default function DefaultPreviewCard({ typeIcon={typeIcon} entityType={entityTypeName} parentContainers={parentContainers?.containers} - parentNodes={parentNodes?.nodes} + parentEntities={parentEntities} parentContainersRef={contentRef} areContainersTruncated={isContentTruncated} /> diff --git a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx index d3cc35ef6a932..c82521dab1bc9 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/DomainSearchList.tsx @@ -1,10 +1,14 @@ +import { ArrowRightOutlined } from '@ant-design/icons'; import React from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { Domain, EntityType, RecommendationContent } from '../../../../types.generated'; -import { IconStyleType } from '../../../entity/Entity'; import { LogoCountCard } from '../../../shared/LogoCountCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; +import DomainIcon from '../../../domain/DomainIcon'; +import { PageRoutes } from '../../../../conf/Global'; +import { HomePageButton } from '../../../shared/components'; +import { HoverEntityTooltip } from './HoverEntityTooltip'; const DomainListContainer = styled.div` display: flex; @@ -13,6 +17,17 @@ const DomainListContainer = styled.div` flex-wrap: wrap; `; +const AllDomainsWrapper = styled.div` + color: ${(props) => props.theme.styles['primary-color']}; + font-size: 14px; +`; + +const AllDomainsText = styled.div` + margin-bottom: 8px; +`; + +const NUM_DOMAIN_CARDS = 9; + type Props = { content: Array; onClick?: (index: number) => void; @@ -23,7 +38,8 @@ export const DomainSearchList = ({ content, onClick }: Props) => { const domainsWithCounts: Array<{ domain: Domain; count?: number }> = content .map((cnt) => ({ domain: cnt.entity, count: cnt.params?.contentParams?.count })) - .filter((domainWithCount) => domainWithCount.domain !== null && domainWithCount !== undefined) as Array<{ + .filter((domainWithCount) => domainWithCount?.domain !== null) + .slice(0, NUM_DOMAIN_CARDS) as Array<{ domain: Domain; count?: number; }>; @@ -31,18 +47,34 @@ export const DomainSearchList = ({ content, onClick }: Props) => { return ( {domainsWithCounts.map((domain, index) => ( - onClick?.(index)} - > - - + + onClick?.(index)} + > + + } + count={domain.count} + /> + + ))} + + + + View All Domains + + + + ); }; diff --git a/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx b/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx index a39a39cd52db9..9ff0a1a2f940b 100644 --- a/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx +++ b/datahub-web-react/src/app/recommendations/renderer/component/HoverEntityTooltip.tsx @@ -1,3 +1,4 @@ +import { TooltipPlacement } from 'antd/es/tooltip'; import { Tooltip } from 'antd'; import React from 'react'; import { Entity } from '../../../../types.generated'; @@ -9,9 +10,10 @@ type Props = { // whether the tooltip can be opened or if it should always stay closed canOpen?: boolean; children: React.ReactNode; + placement?: TooltipPlacement; }; -export const HoverEntityTooltip = ({ entity, canOpen = true, children }: Props) => { +export const HoverEntityTooltip = ({ entity, canOpen = true, children, placement }: Props) => { const entityRegistry = useEntityRegistry(); if (!entity || !entity.type || !entity.urn) { @@ -23,7 +25,7 @@ export const HoverEntityTooltip = ({ entity, canOpen = true, children }: Props) {entityRegistry.renderPreview(entity.type, PreviewType.HOVER_CARD, entity)}} diff --git a/datahub-web-react/src/app/search/SearchResultList.tsx b/datahub-web-react/src/app/search/SearchResultList.tsx index 59940eb6ac0b2..375ec50a44313 100644 --- a/datahub-web-react/src/app/search/SearchResultList.tsx +++ b/datahub-web-react/src/app/search/SearchResultList.tsx @@ -31,7 +31,7 @@ const ThinDivider = styled(Divider)` margin-bottom: 16px; `; -const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` +export const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` ${(props) => props.showUpdatedStyles && ` @@ -39,7 +39,6 @@ const ResultWrapper = styled.div<{ showUpdatedStyles: boolean }>` border-radius: 5px; margin: 0 auto 8px auto; padding: 8px 16px; - max-width: 1200px; border-bottom: 1px solid ${ANTD_GRAY[5]}; `} `; diff --git a/datahub-web-react/src/app/search/SearchResults.tsx b/datahub-web-react/src/app/search/SearchResults.tsx index f7ee6380d3b75..f690c63294075 100644 --- a/datahub-web-react/src/app/search/SearchResults.tsx +++ b/datahub-web-react/src/app/search/SearchResults.tsx @@ -28,6 +28,7 @@ import useToggleSidebar from './useToggleSidebar'; import SearchSortSelect from './sorting/SearchSortSelect'; import { combineSiblingsInSearchResults } from './utils/combineSiblingsInSearchResults'; import SearchQuerySuggester from './suggestions/SearchQuerySugggester'; +import { ANTD_GRAY_V2 } from '../entity/shared/constants'; const SearchResultsWrapper = styled.div<{ v2Styles: boolean }>` display: flex; @@ -56,7 +57,7 @@ const ResultContainer = styled.div<{ v2Styles: boolean }>` ? ` display: flex; flex-direction: column; - background-color: #F8F9FA; + background-color: ${ANTD_GRAY_V2[1]}; ` : ` max-width: calc(100% - 260px); diff --git a/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx b/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx index d241a3895f19f..2154837fa5e26 100644 --- a/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx +++ b/datahub-web-react/src/app/search/autoComplete/AutoCompleteEntity.tsx @@ -10,6 +10,8 @@ import AutoCompleteEntityIcon from './AutoCompleteEntityIcon'; import { SuggestionText } from './styledComponents'; import AutoCompletePlatformNames from './AutoCompletePlatformNames'; import { getPlatformName } from '../../entity/shared/utils'; +import { getParentEntities } from '../filters/utils'; +import ParentEntities from '../filters/ParentEntities'; const AutoCompleteEntityWrapper = styled.div` display: flex; @@ -76,11 +78,12 @@ export default function AutoCompleteEntity({ query, entity, siblings, hasParentT // Need to reverse parentContainers since it returns direct parent first. const orderedParentContainers = [...parentContainers].reverse(); const subtype = genericEntityProps?.subTypes?.typeNames?.[0]; + const parentEntities = getParentEntities(entity) || []; const showPlatforms = !!platforms.length; const showPlatformDivider = !!platforms.length && !!parentContainers.length; const showParentContainers = !!parentContainers.length; - const showHeader = showPlatforms || showParentContainers; + const showHeader = showPlatforms || showParentContainers || parentEntities.length > 0; return ( @@ -96,6 +99,7 @@ export default function AutoCompleteEntity({ query, entity, siblings, hasParentT {showPlatforms && } {showPlatformDivider && } {showParentContainers && } + )} ` @@ -103,6 +103,10 @@ const ArrowButton = styled(Button)<{ isOpen: boolean }>` `} `; +const ParentWrapper = styled.div` + max-width: 220px; +`; + interface Props { filterOption: FilterOptionType; selectedFilterOptions: FilterOptionType[]; @@ -126,8 +130,7 @@ export default function FilterOption({ const shouldShowIcon = field === PLATFORM_FILTER_NAME && icon !== null; const shouldShowTagColor = field === TAGS_FILTER_NAME && entity?.type === EntityType.Tag; const isSubTypeFilter = field === TYPE_NAMES_FILTER_NAME; - const isGlossaryTerm = entity?.type === EntityType.GlossaryTerm; - const parentNodes: GlossaryNode[] = isGlossaryTerm ? (entity as GlossaryTerm).parentNodes?.nodes || [] : []; + const parentEntities: Entity[] = getParentEntities(entity as Entity) || []; // only entity type filters return 10,000 max aggs const countText = count === MAX_COUNT_VAL && field === ENTITY_SUB_TYPE_FILTER_NAME ? '10k+' : formatNumber(count); @@ -145,7 +148,7 @@ export default function FilterOption({ return ( <> - 0} addPadding={addPadding}> + 0} addPadding={addPadding}> - {isGlossaryTerm && } + {parentEntities.length > 0 && ( + + + + )} {shouldShowIcon && <>{icon}} {shouldShowTagColor && ( diff --git a/datahub-web-react/src/app/search/filters/ParentNodes.tsx b/datahub-web-react/src/app/search/filters/ParentEntities.tsx similarity index 54% rename from datahub-web-react/src/app/search/filters/ParentNodes.tsx rename to datahub-web-react/src/app/search/filters/ParentEntities.tsx index 7012f07c16e64..2504d5f0ff25a 100644 --- a/datahub-web-react/src/app/search/filters/ParentNodes.tsx +++ b/datahub-web-react/src/app/search/filters/ParentEntities.tsx @@ -2,19 +2,16 @@ import { FolderOpenOutlined } from '@ant-design/icons'; import { Tooltip, Typography } from 'antd'; import React from 'react'; import styled from 'styled-components'; -import { EntityType, GlossaryNode, GlossaryTerm } from '../../../types.generated'; +import { Entity } from '../../../types.generated'; import { ANTD_GRAY } from '../../entity/shared/constants'; import { useEntityRegistry } from '../../useEntityRegistry'; -const NUM_VISIBLE_NODES = 2; - const ParentNodesWrapper = styled.div` font-size: 12px; color: ${ANTD_GRAY[7]}; display: flex; align-items: center; margin-bottom: 3px; - max-width: 220px; overflow: hidden; `; @@ -27,54 +24,62 @@ export const ArrowWrapper = styled.span` margin: 0 3px; `; +const StyledTooltip = styled(Tooltip)` + display: flex; + white-space: nowrap; + overflow: hidden; +`; + +const DEFAULT_NUM_VISIBLE = 2; + interface Props { - glossaryTerm: GlossaryTerm; + parentEntities: Entity[]; + numVisible?: number; } -export default function ParentNodes({ glossaryTerm }: Props) { +export default function ParentEntities({ parentEntities, numVisible = DEFAULT_NUM_VISIBLE }: Props) { const entityRegistry = useEntityRegistry(); - const parentNodes: GlossaryNode[] = glossaryTerm.parentNodes?.nodes || []; - // parent nodes are returned with direct parent first - const orderedParentNodes = [...parentNodes].reverse(); - const visibleNodes = orderedParentNodes.slice(orderedParentNodes.length - NUM_VISIBLE_NODES); - const numHiddenNodes = orderedParentNodes.length - NUM_VISIBLE_NODES; - const includeNodePathTooltip = parentNodes.length > NUM_VISIBLE_NODES; + // parent nodes/domains are returned with direct parent first + const orderedParentEntities = [...parentEntities].reverse(); + const numHiddenEntities = orderedParentEntities.length - numVisible; + const hasHiddenEntities = numHiddenEntities > 0; + const visibleNodes = hasHiddenEntities ? orderedParentEntities.slice(numHiddenEntities) : orderedParentEntities; - if (!parentNodes.length) return null; + if (!parentEntities.length) return null; return ( - - {orderedParentNodes.map((glossaryNode, index) => ( + {orderedParentEntities.map((parentEntity, index) => ( <> - {entityRegistry.getDisplayName(EntityType.GlossaryNode, glossaryNode)} + {entityRegistry.getDisplayName(parentEntity.type, parentEntity)} - {index !== orderedParentNodes.length - 1 && {'>'}} + {index !== orderedParentEntities.length - 1 && {'>'}} ))} } > - {numHiddenNodes > 0 && - [...Array(numHiddenNodes)].map(() => ( + {hasHiddenEntities && + [...Array(numHiddenEntities)].map(() => ( <> {'>'} ))} - {visibleNodes.map((glossaryNode, index) => { - const displayName = entityRegistry.getDisplayName(EntityType.GlossaryNode, glossaryNode); + {visibleNodes.map((parentEntity, index) => { + const displayName = entityRegistry.getDisplayName(parentEntity.type, parentEntity); return ( <> - + {displayName} {index !== visibleNodes.length - 1 && {'>'}} @@ -82,6 +87,6 @@ export default function ParentNodes({ glossaryTerm }: Props) { ); })} - + ); } diff --git a/datahub-web-react/src/app/search/filters/utils.tsx b/datahub-web-react/src/app/search/filters/utils.tsx index b4ce80a795edf..0643816210074 100644 --- a/datahub-web-react/src/app/search/filters/utils.tsx +++ b/datahub-web-react/src/app/search/filters/utils.tsx @@ -15,10 +15,12 @@ import { AggregationMetadata, DataPlatform, DataPlatformInstance, + Domain, Entity, EntityType, FacetFilterInput, FacetMetadata, + GlossaryTerm, } from '../../../types.generated'; import { IconStyleType } from '../../entity/Entity'; import { @@ -339,3 +341,16 @@ export function canCreateViewFromFilters(activeFilters: FacetFilterInput[]) { } return true; } + +export function getParentEntities(entity: Entity): Entity[] | null { + if (!entity) { + return null; + } + if (entity.type === EntityType.GlossaryTerm) { + return (entity as GlossaryTerm).parentNodes?.nodes || []; + } + if (entity.type === EntityType.Domain) { + return (entity as Domain).parentDomains?.domains || []; + } + return null; +} diff --git a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx index ca666c8e346e2..d7e0778bc913a 100644 --- a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx +++ b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx @@ -7,13 +7,14 @@ import { BrowseProvider } from './BrowseContext'; import SidebarLoadingError from './SidebarLoadingError'; import { SEARCH_RESULTS_BROWSE_SIDEBAR_ID } from '../../onboarding/config/SearchOnboardingConfig'; import useSidebarEntities from './useSidebarEntities'; +import { ANTD_GRAY_V2 } from '../../entity/shared/constants'; const Sidebar = styled.div<{ visible: boolean; width: number }>` height: 100%; width: ${(props) => (props.visible ? `${props.width}px` : '0')}; transition: width 250ms ease-in-out; border-right: 1px solid ${(props) => props.theme.styles['border-color-base']}; - background-color: #f8f9fa; + background-color: ${ANTD_GRAY_V2[1]}; background: white; `; diff --git a/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx b/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx index 32d2c4af948ef..ba93cf94fba2b 100644 --- a/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx +++ b/datahub-web-react/src/app/search/sidebar/ExpandableNode.tsx @@ -1,9 +1,10 @@ import React, { MouseEventHandler, ReactNode } from 'react'; import styled from 'styled-components'; import { VscTriangleRight } from 'react-icons/vsc'; -import { Button, Typography } from 'antd'; +import { Typography } from 'antd'; import { UpCircleOutlined } from '@ant-design/icons'; import { ANTD_GRAY } from '../../entity/shared/constants'; +import { BaseButton, BodyContainer, BodyGridExpander, RotatingButton } from '../../shared/components'; const Layout = styled.div` margin-left: 8px; @@ -11,17 +12,6 @@ const Layout = styled.div` const HeaderContainer = styled.div``; -const BodyGridExpander = styled.div<{ isOpen: boolean }>` - display: grid; - grid-template-rows: ${(props) => (props.isOpen ? '1fr' : '0fr')}; - transition: grid-template-rows 250ms; - overflow: hidden; -`; - -const BodyContainer = styled.div` - min-height: 0; -`; - type ExpandableNodeProps = { isOpen: boolean; header: ReactNode; @@ -68,22 +58,6 @@ ExpandableNode.HeaderLeft = styled.div` align-items: center; `; -const BaseButton = styled(Button)` - &&& { - display: flex; - align-items: center; - justify-content: center; - border: none; - box-shadow: none; - border-radius: 50%; - } -`; - -const RotatingButton = styled(BaseButton)<{ deg: number }>` - transform: rotate(${(props) => props.deg}deg); - transition: transform 250ms; -`; - ExpandableNode.StaticButton = ({ icon, onClick }: { icon: JSX.Element; onClick?: () => void }) => { const onClickButton: MouseEventHandler = (e) => { e.stopPropagation(); diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx index 4fa342b5c8cfc..53055be0fe2fb 100644 --- a/datahub-web-react/src/app/settings/SettingsPage.tsx +++ b/datahub-web-react/src/app/settings/SettingsPage.tsx @@ -91,12 +91,13 @@ export const SettingsPage = () => { const isPoliciesEnabled = config?.policiesConfig.enabled; const isIdentityManagementEnabled = config?.identityManagementConfig.enabled; const isViewsEnabled = config?.viewsConfig.enabled; + const { readOnlyModeEnabled } = config.featureFlags; const showPolicies = (isPoliciesEnabled && me && me?.platformPrivileges?.managePolicies) || false; const showUsersGroups = (isIdentityManagementEnabled && me && me?.platformPrivileges?.manageIdentities) || false; const showViews = isViewsEnabled || false; const showOwnershipTypes = me && me?.platformPrivileges?.manageOwnershipTypes; - const showHomePagePosts = me && me?.platformPrivileges?.manageGlobalAnnouncements; + const showHomePagePosts = me && me?.platformPrivileges?.manageGlobalAnnouncements && !readOnlyModeEnabled; return ( diff --git a/datahub-web-react/src/app/shared/LogoCountCard.tsx b/datahub-web-react/src/app/shared/LogoCountCard.tsx index 3e2f74ebe5166..ebf0d9cd4f54e 100644 --- a/datahub-web-react/src/app/shared/LogoCountCard.tsx +++ b/datahub-web-react/src/app/shared/LogoCountCard.tsx @@ -1,27 +1,9 @@ import React from 'react'; -import { Image, Typography, Button } from 'antd'; +import { Image, Typography } from 'antd'; import styled from 'styled-components'; import { ANTD_GRAY } from '../entity/shared/constants'; import { formatNumber } from './formatNumber'; - -const Container = styled(Button)` - margin-right: 12px; - margin-left: 12px; - margin-bottom: 12px; - width: 160px; - height: 140px; - display: flex; - justify-content: center; - border-radius: 4px; - align-items: center; - flex-direction: column; - border: 1px solid ${ANTD_GRAY[4]}; - box-shadow: ${(props) => props.theme.styles['box-shadow']}; - &&:hover { - box-shadow: ${(props) => props.theme.styles['box-shadow-hover']}; - } - white-space: unset; -`; +import { HomePageButton } from './components'; const PlatformLogo = styled(Image)` max-height: 32px; @@ -53,7 +35,7 @@ type Props = { export const LogoCountCard = ({ logoUrl, logoComponent, name, count, onClick }: Props) => { return ( - + {(logoUrl && ) || logoComponent} @@ -68,6 +50,6 @@ export const LogoCountCard = ({ logoUrl, logoComponent, name, count, onClick }: {count !== undefined && {formatNumber(count)}} - + ); }; diff --git a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx index 8f84135d31025..126c43e6a6a7c 100644 --- a/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx +++ b/datahub-web-react/src/app/shared/admin/HeaderLinks.tsx @@ -5,7 +5,6 @@ import { BarChartOutlined, BookOutlined, SettingOutlined, - FolderOutlined, SolutionOutlined, DownOutlined, } from '@ant-design/icons'; @@ -17,9 +16,10 @@ import { ANTD_GRAY } from '../../entity/shared/constants'; import { HOME_PAGE_INGESTION_ID } from '../../onboarding/config/HomePageOnboardingConfig'; import { useUpdateEducationStepIdsAllowlist } from '../../onboarding/useUpdateEducationStepIdsAllowlist'; import { useUserContext } from '../../context/useUserContext'; +import DomainIcon from '../../domain/DomainIcon'; const LinkWrapper = styled.span` - margin-right: 0px; + margin-right: 0; `; const LinksWrapper = styled.div<{ areLinksHidden?: boolean }>` @@ -126,7 +126,12 @@ export function HeaderLinks(props: Props) { - + {t('adminHeader.domainsText')} {t('adminHeader.domainsDescription')} diff --git a/datahub-web-react/src/app/shared/components.tsx b/datahub-web-react/src/app/shared/components.tsx new file mode 100644 index 0000000000000..68d2fb52cfdba --- /dev/null +++ b/datahub-web-react/src/app/shared/components.tsx @@ -0,0 +1,49 @@ +import { Button } from 'antd'; +import styled from 'styled-components'; +import { ANTD_GRAY } from '../entity/shared/constants'; + +export const HomePageButton = styled(Button)` + margin-right: 12px; + margin-left: 12px; + margin-bottom: 12px; + width: 160px; + height: 140px; + display: flex; + justify-content: center; + border-radius: 4px; + align-items: center; + flex-direction: column; + border: 1px solid ${ANTD_GRAY[4]}; + box-shadow: ${(props) => props.theme.styles['box-shadow']}; + &&:hover { + box-shadow: ${(props) => props.theme.styles['box-shadow-hover']}; + } + white-space: unset; +`; + +export const BaseButton = styled(Button)` + &&& { + display: flex; + align-items: center; + justify-content: center; + border: none; + box-shadow: none; + border-radius: 50%; + } +`; + +export const RotatingButton = styled(BaseButton)<{ deg: number }>` + transform: rotate(${(props) => props.deg}deg); + transition: transform 250ms; +`; + +export const BodyGridExpander = styled.div<{ isOpen: boolean }>` + display: grid; + grid-template-rows: ${(props) => (props.isOpen ? '1fr' : '0fr')}; + transition: grid-template-rows 250ms; + overflow: hidden; +`; + +export const BodyContainer = styled.div` + min-height: 0; +`; diff --git a/datahub-web-react/src/app/shared/deleteUtils.ts b/datahub-web-react/src/app/shared/deleteUtils.ts index c1bfeac37372b..37a3758712ad6 100644 --- a/datahub-web-react/src/app/shared/deleteUtils.ts +++ b/datahub-web-react/src/app/shared/deleteUtils.ts @@ -1,3 +1,4 @@ +import { PageRoutes } from '../../conf/Global'; import { useDeleteAssertionMutation } from '../../graphql/assertion.generated'; import { useDeleteDataProductMutation } from '../../graphql/dataProduct.generated'; import { useDeleteDomainMutation } from '../../graphql/domain.generated'; @@ -18,10 +19,11 @@ export const getEntityProfileDeleteRedirectPath = (type: EntityType, entityData: switch (type) { case EntityType.CorpGroup: case EntityType.CorpUser: - case EntityType.Domain: case EntityType.Tag: // Return Home. return '/'; + case EntityType.Domain: + return `${PageRoutes.DOMAINS}`; case EntityType.GlossaryNode: case EntityType.GlossaryTerm: // Return to glossary page. diff --git a/datahub-web-react/src/app/shared/sidebar/components.tsx b/datahub-web-react/src/app/shared/sidebar/components.tsx new file mode 100644 index 0000000000000..5d123d6022790 --- /dev/null +++ b/datahub-web-react/src/app/shared/sidebar/components.tsx @@ -0,0 +1,23 @@ +import React from 'react'; +import { RightOutlined } from '@ant-design/icons'; +import styled from 'styled-components'; +import { RotatingButton } from '../components'; + +export const SidebarWrapper = styled.div<{ width: number }>` + max-height: 100%; + width: ${(props) => props.width}px; + min-width: ${(props) => props.width}px; +`; + +export function RotatingTriangle({ isOpen, onClick }: { isOpen: boolean; onClick?: () => void }) { + return ( + } + onClick={onClick} + /> + ); +} diff --git a/datahub-web-react/src/app/shared/styleUtils.ts b/datahub-web-react/src/app/shared/styleUtils.ts new file mode 100644 index 0000000000000..21bc866218cb8 --- /dev/null +++ b/datahub-web-react/src/app/shared/styleUtils.ts @@ -0,0 +1,7 @@ +export function applyOpacity(hexColor: string, opacity: number) { + if (hexColor.length !== 7) return hexColor; + + const updatedOpacity = Math.round(opacity * 2.55); + + return hexColor + updatedOpacity.toString(16).padStart(2, '0'); +} diff --git a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx index 1b4b0bde3d525..44d1ebe40d073 100644 --- a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx +++ b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx @@ -51,15 +51,15 @@ const StyleTag = styled(CustomTag)` line-height: 16px; `; -export const BrowserWrapper = styled.div<{ isHidden: boolean }>` +export const BrowserWrapper = styled.div<{ isHidden: boolean; width?: string; maxHeight?: number }>` background-color: white; border-radius: 5px; box-shadow: 0 3px 6px -4px rgb(0 0 0 / 12%), 0 6px 16px 0 rgb(0 0 0 / 8%), 0 9px 28px 8px rgb(0 0 0 / 5%); - max-height: 380px; + max-height: ${(props) => (props.maxHeight ? props.maxHeight : '380')}px; overflow: auto; position: absolute; transition: opacity 0.2s; - width: 480px; + width: ${(props) => (props.width ? props.width : '480px')}; z-index: 1051; ${(props) => props.isHidden && diff --git a/datahub-web-react/src/app/shared/tags/DomainLink.tsx b/datahub-web-react/src/app/shared/tags/DomainLink.tsx index 1c14b71369ed6..a14114ce43e43 100644 --- a/datahub-web-react/src/app/shared/tags/DomainLink.tsx +++ b/datahub-web-react/src/app/shared/tags/DomainLink.tsx @@ -3,10 +3,10 @@ import React from 'react'; import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { Domain, EntityType } from '../../../types.generated'; -import { IconStyleType } from '../../entity/Entity'; import { HoverEntityTooltip } from '../../recommendations/renderer/component/HoverEntityTooltip'; import { useEntityRegistry } from '../../useEntityRegistry'; import { ANTD_GRAY } from '../../entity/shared/constants'; +import DomainIcon from '../../domain/DomainIcon'; const DomainLinkContainer = styled(Link)` display: inline-block; @@ -39,7 +39,12 @@ function DomainContent({ domain, name, closable, onClose, tagStyle, fontSize }: return ( - {entityRegistry.getIcon(EntityType.Domain, fontSize || 10, IconStyleType.ACCENT, ANTD_GRAY[9])} + {displayName} diff --git a/datahub-web-react/src/app/shared/useToggle.ts b/datahub-web-react/src/app/shared/useToggle.ts index b020bf030f079..a73c702c4351b 100644 --- a/datahub-web-react/src/app/shared/useToggle.ts +++ b/datahub-web-react/src/app/shared/useToggle.ts @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useMemo, useState } from 'react'; const NOOP = (_: boolean) => {}; @@ -9,25 +9,39 @@ const useToggle = ({ initialValue = false, closeDelay = 0, openDelay = 0, onTogg const isClosing = transition === 'closing'; const isTransitioning = transition !== null; - const toggle = () => { - if (isOpen) { + const toggleClose = useMemo( + () => () => { setTransition('closing'); window.setTimeout(() => { setIsOpen(false); setTransition(null); onToggle(false); }, closeDelay); - } else { + }, + [closeDelay, onToggle], + ); + + const toggleOpen = useMemo( + () => () => { setTransition('opening'); window.setTimeout(() => { setIsOpen(true); setTransition(null); onToggle(true); }, openDelay); + }, + [openDelay, onToggle], + ); + + const toggle = () => { + if (isOpen) { + toggleClose(); + } else { + toggleOpen(); } }; - return { isOpen, isClosing, isOpening, isTransitioning, toggle } as const; + return { isOpen, isClosing, isOpening, isTransitioning, toggle, toggleOpen, toggleClose } as const; }; export default useToggle; diff --git a/datahub-web-react/src/app/useAppConfig.ts b/datahub-web-react/src/app/useAppConfig.ts index cdc8f92210a0d..821d00b9017c3 100644 --- a/datahub-web-react/src/app/useAppConfig.ts +++ b/datahub-web-react/src/app/useAppConfig.ts @@ -12,3 +12,8 @@ export function useIsShowAcrylInfoEnabled() { const appConfig = useAppConfig(); return appConfig.config.featureFlags.showAcrylInfo; } + +export function useIsNestedDomainsEnabled() { + const appConfig = useAppConfig(); + return appConfig.config.featureFlags.nestedDomainsEnabled; +} diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 096c2fd6ef0e5..4087ad453687c 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -49,6 +49,7 @@ export const DEFAULT_APP_CONFIG = { showBrowseV2: true, showAcrylInfo: false, showAccessManagement: false, + nestedDomainsEnabled: true, }, }; diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts index 5a8257ce4b52b..f9d650acc6d6e 100644 --- a/datahub-web-react/src/conf/Global.ts +++ b/datahub-web-react/src/conf/Global.ts @@ -24,6 +24,7 @@ export enum PageRoutes { INGESTION = '/ingestion', SETTINGS = '/settings', DOMAINS = '/domains', + DOMAIN = '/domain', GLOSSARY = '/glossary', SETTINGS_VIEWS = '/settings/views', EMBED = '/embed', diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 228fa1c9430d0..4e9bbb11d8c5a 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -64,6 +64,7 @@ query appConfig { showBrowseV2 showAcrylInfo showAccessManagement + nestedDomainsEnabled } } } diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index d72ff336bf9e7..951b93fcba9af 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -2,10 +2,14 @@ query getDomain($urn: String!) { domain(urn: $urn) { urn id + type properties { name description } + parentDomains { + ...parentDomainsFields + } ownership { ...ownershipFields } @@ -23,6 +27,9 @@ query getDomain($urn: String!) { } } } + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } } } @@ -33,16 +40,29 @@ query listDomains($input: ListDomainsInput!) { total domains { urn + id + type properties { name description } + parentDomains { + ...parentDomainsFields + } ownership { ...ownershipFields } - entities(input: { start: 0, count: 1 }) { - total - } + ...domainEntitiesFields + } + } +} + +query getDomainChildrenCount($urn: String!) { + domain(urn: $urn) { + urn + type + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total } } } @@ -51,6 +71,10 @@ mutation createDomain($input: CreateDomainInput!) { createDomain(input: $input) } +mutation moveDomain($input: MoveDomainInput!) { + moveDomain(input: $input) +} + mutation deleteDomain($urn: String!) { deleteDomain(urn: $urn) } diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index c3ac2139e687b..72474911b9310 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -82,6 +82,20 @@ fragment parentNodesFields on ParentNodesResult { } } +fragment parentDomainsFields on ParentDomainsResult { + count + domains { + urn + type + ... on Domain { + properties { + name + description + } + } + } +} + fragment ownershipFields on Ownership { owners { owner { @@ -931,6 +945,20 @@ fragment parentContainerFields on Container { } } +fragment domainEntitiesFields on Domain { + entities(input: { start: 0, count: 0 }) { + total + } + dataProducts: entities( + input: { start: 0, count: 0, filters: [{ field: "_entityType", value: "DATA_PRODUCT" }] } + ) { + total + } + children: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 0 }) { + total + } +} + fragment entityDomain on DomainAssociation { domain { urn @@ -939,6 +967,10 @@ fragment entityDomain on DomainAssociation { name description } + parentDomains { + ...parentDomainsFields + } + ...domainEntitiesFields } associatedUrn } diff --git a/datahub-web-react/src/graphql/preview.graphql b/datahub-web-react/src/graphql/preview.graphql index 03635ab1b66d5..e104d62c67074 100644 --- a/datahub-web-react/src/graphql/preview.graphql +++ b/datahub-web-react/src/graphql/preview.graphql @@ -304,7 +304,12 @@ fragment entityPreview on Entity { urn properties { name + description + } + parentDomains { + ...parentDomainsFields } + ...domainEntitiesFields } ... on Container { ...entityContainer diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 94ff263c02039..2297c2d0c1d07 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -155,6 +155,9 @@ fragment autoCompleteFields on Entity { properties { name } + parentDomains { + ...parentDomainsFields + } } ... on DataProduct { properties { @@ -671,6 +674,10 @@ fragment searchResultFields on Entity { ownership { ...ownershipFields } + parentDomains { + ...parentDomainsFields + } + ...domainEntitiesFields } ... on Container { properties { @@ -825,6 +832,9 @@ fragment facetFields on FacetMetadata { properties { name } + parentDomains { + ...parentDomainsFields + } } ... on Container { platform { diff --git a/datahub-web-react/src/images/dynamodblogo.png b/datahub-web-react/src/images/dynamodblogo.png new file mode 100644 index 0000000000000..f5beafb035772 Binary files /dev/null and b/datahub-web-react/src/images/dynamodblogo.png differ diff --git a/datahub-web-react/src/images/mlflowlogo.png b/datahub-web-react/src/images/mlflowlogo.png new file mode 100644 index 0000000000000..e724d1affbc14 Binary files /dev/null and b/datahub-web-react/src/images/mlflowlogo.png differ diff --git a/docker/build.gradle b/docker/build.gradle index ae101fe1defc5..0faea626e982d 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -38,6 +38,16 @@ task quickstart(type: Exec, dependsOn: ':metadata-ingestion:install') { // environment "ACTIONS_VERSION", 'alpine3.17-slim' // environment "DATAHUB_ACTIONS_IMAGE", 'nginx' + // Elastic + // environment "DATAHUB_SEARCH_IMAGE", 'elasticsearch' + // environment "DATAHUB_SEARCH_TAG", '7.10.1' + + // OpenSearch + environment "DATAHUB_SEARCH_IMAGE", 'opensearchproject/opensearch' + environment "DATAHUB_SEARCH_TAG", '2.9.0' + environment "XPACK_SECURITY_ENABLED", 'plugins.security.disabled=true' + environment "USE_AWS_ELASTICSEARCH", 'true' + def cmd = [ 'source ../metadata-ingestion/venv/bin/activate && ', 'datahub docker quickstart', diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index 3d47f79617370..564cc19cc9a5f 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -1,7 +1,7 @@ ARG APP_ENV=full ARG BASE_IMAGE=base -FROM golang:1-alpine3.17 AS binary +FROM golang:1-alpine3.17 AS dockerize-binary ENV DOCKERIZE_VERSION v0.6.1 WORKDIR /go/src/github.com/jwilder @@ -41,7 +41,7 @@ RUN apt-get update && apt-get install -y -qq \ && rm -rf /var/lib/apt/lists/* /var/cache/apk/* # compiled against newer golang for security fixes -COPY --from=binary /go/bin/dockerize /usr/local/bin +COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh diff --git a/docker/datahub-ingestion-base/smoke.Dockerfile b/docker/datahub-ingestion-base/smoke.Dockerfile index 276f6dbc4436e..15dc46ae5b882 100644 --- a/docker/datahub-ingestion-base/smoke.Dockerfile +++ b/docker/datahub-ingestion-base/smoke.Dockerfile @@ -20,7 +20,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk COPY . /datahub-src ARG RELEASE_VERSION RUN cd /datahub-src/metadata-ingestion && \ - sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ + sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ cat src/datahub/__init__.py && \ cd ../ && \ ./gradlew :metadata-ingestion:installAll diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 2ceff6a800ebb..0132ceaa9b1a9 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -1,7 +1,7 @@ # Defining environment ARG APP_ENV=full ARG BASE_IMAGE=acryldata/datahub-ingestion-base -ARG DOCKER_VERSION=latest +ARG DOCKER_VERSION=head FROM $BASE_IMAGE:$DOCKER_VERSION as base USER 0 @@ -11,8 +11,8 @@ COPY ./metadata-ingestion-modules/airflow-plugin /datahub-ingestion/airflow-plug ARG RELEASE_VERSION WORKDIR /datahub-ingestion -RUN sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ - sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \ +RUN sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ + sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \ cat src/datahub/__init__.py && \ chown -R datahub /datahub-ingestion diff --git a/docker/datahub-ingestion/Dockerfile-slim-only b/docker/datahub-ingestion/Dockerfile-slim-only index 678bee7e306f6..cb8c27ab463c4 100644 --- a/docker/datahub-ingestion/Dockerfile-slim-only +++ b/docker/datahub-ingestion/Dockerfile-slim-only @@ -1,6 +1,6 @@ # Defining environment ARG BASE_IMAGE=acryldata/datahub-ingestion-base -ARG DOCKER_VERSION=latest +ARG DOCKER_VERSION=head-slim FROM $BASE_IMAGE:$DOCKER_VERSION as base USER 0 @@ -9,7 +9,7 @@ COPY ./metadata-ingestion /datahub-ingestion ARG RELEASE_VERSION WORKDIR /datahub-ingestion -RUN sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ +RUN sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ cat src/datahub/__init__.py && \ chown -R datahub /datahub-ingestion diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 08f8cc1ec9c45..9543e67da07f2 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -96,6 +96,9 @@ services: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile env_file: elasticsearch-setup/env/docker.env + environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} depends_on: elasticsearch: condition: service_healthy @@ -117,13 +120,13 @@ services: elasticsearch: container_name: elasticsearch hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - 9200:9200 env_file: elasticsearch/env/docker.env environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} healthcheck: test: curl -sS --fail http://elasticsearch:9200/_cluster/health?wait_for_status=yellow&timeout=0s start_period: 5s diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 0b2e4f76b8fa9..022362782f742 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -81,6 +81,9 @@ services: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile env_file: elasticsearch-setup/env/docker.env + environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} depends_on: elasticsearch: condition: service_healthy @@ -104,13 +107,13 @@ services: elasticsearch: container_name: elasticsearch hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 env_file: elasticsearch/env/docker.env environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} deploy: resources: limits: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d07ea5fa88f8b..a486689e050a2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -83,6 +83,9 @@ services: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile env_file: elasticsearch-setup/env/docker.env + environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} depends_on: elasticsearch: condition: service_healthy @@ -109,13 +112,13 @@ services: elasticsearch: container_name: elasticsearch hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 env_file: elasticsearch/env/docker.env environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} deploy: resources: limits: diff --git a/docker/elasticsearch/env/docker.env b/docker/elasticsearch/env/docker.env index 4b1f0215ea6c8..46b5836dedd28 100644 --- a/docker/elasticsearch/env/docker.env +++ b/docker/elasticsearch/env/docker.env @@ -1 +1,2 @@ ES_JAVA_OPTS="-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" +OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" \ No newline at end of file diff --git a/docker/postgres-setup/init.sh b/docker/postgres-setup/init.sh index 6c0adc8c69bdd..afc9bdfe4c668 100755 --- a/docker/postgres-setup/init.sh +++ b/docker/postgres-setup/init.sh @@ -1,8 +1,13 @@ #!/bin/sh export PGPASSWORD=$POSTGRES_PASSWORD +POSTGRES_CREATE_DB=${POSTGRES_CREATE_DB:-true} +POSTGRES_CREATE_DB_CONNECTION_DB=${POSTGRES_CREATE_DB_CONNECTION_DB:-postgres} + # workaround create database if not exists, check https://stackoverflow.com/a/36591842 -psql -U $POSTGRES_USERNAME -h $POSTGRES_HOST -p $POSTGRES_PORT -tc "SELECT 1 FROM pg_database WHERE datname = '${DATAHUB_DB_NAME}'" | grep -q 1 || psql -U $POSTGRES_USERNAME -h $POSTGRES_HOST -p $POSTGRES_PORT -c "CREATE DATABASE ${DATAHUB_DB_NAME}" +if [ "$POSTGRES_CREATE_DB" = true ]; then + psql -d "$POSTGRES_CREATE_DB_CONNECTION_DB" -U "$POSTGRES_USERNAME" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -tc "SELECT 1 FROM pg_database WHERE datname = '${DATAHUB_DB_NAME}'" | grep -q 1 || psql -d "$POSTGRES_CREATE_DB_CONNECTION_DB" -U "$POSTGRES_USERNAME" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -c "CREATE DATABASE ${DATAHUB_DB_NAME}" +fi sed -e "s/DATAHUB_DB_NAME/${DATAHUB_DB_NAME}/g" /init.sql | tee -a /tmp/init-final.sql -psql -d $DATAHUB_DB_NAME -U $POSTGRES_USERNAME -h $POSTGRES_HOST -p $POSTGRES_PORT < /tmp/init-final.sql +psql -d "$DATAHUB_DB_NAME" -U "$POSTGRES_USERNAME" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" < /tmp/init-final.sql diff --git a/docker/quickstart/docker-compose-m1.quickstart.yml b/docker/quickstart/docker-compose-m1.quickstart.yml index 38418bc8c41b9..89e9aaa0defd6 100644 --- a/docker/quickstart/docker-compose-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-m1.quickstart.yml @@ -161,8 +161,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -170,7 +171,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -181,6 +182,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index cf879faa6a3f0..f6284edc83648 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -154,8 +154,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -163,7 +164,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -174,6 +175,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 007830078d2b4..4e3503e35c0db 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -154,8 +154,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -163,7 +164,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -174,6 +175,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 390543b92123f..e2f52064389e0 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -161,8 +161,9 @@ services: memory: 1G environment: - discovery.type=single-node - - xpack.security.enabled=false + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -170,7 +171,7 @@ services: test: curl -sS --fail http://elasticsearch:$${DATAHUB_MAPPED_ELASTIC_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=0s timeout: 5s hostname: elasticsearch - image: elasticsearch:7.10.1 + image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: @@ -181,6 +182,8 @@ services: elasticsearch: condition: service_healthy environment: + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http diff --git a/docs-website/build.gradle b/docs-website/build.gradle index 851c10d9ea97f..a213ec1ae8194 100644 --- a/docs-website/build.gradle +++ b/docs-website/build.gradle @@ -89,7 +89,7 @@ task fastReload(type: YarnTask) { args = ['run', 'generate-rsync'] } -task yarnLint(type: YarnTask, dependsOn: [yarnInstall]) { +task yarnLint(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) { inputs.files(projectMdFiles) args = ['run', 'lint-check'] outputs.dir("dist") @@ -119,6 +119,10 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnLint, yarnGenerate, downloadHisto outputs.dir("dist") // tell gradle to apply the build cache outputs.cacheIf { true } + // See https://stackoverflow.com/questions/53230823/fatal-error-ineffective-mark-compacts-near-heap-limit-allocation-failed-java + // and https://github.com/facebook/docusaurus/issues/8329. + // TODO: As suggested in https://github.com/facebook/docusaurus/issues/4765, try switching to swc-loader. + environment = ['NODE_OPTIONS': '--max-old-space-size=10248'] args = ['run', 'build'] } diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 9bdba5f317542..c1ecf0283cf63 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -12,13 +12,11 @@ module.exports = { organizationName: "datahub-project", // Usually your GitHub org/user name. projectName: "datahub", // Usually your repo name. staticDirectories: ["static", "genStatic"], - stylesheets: [ - "https://fonts.googleapis.com/css2?family=Manrope:wght@400;600&display=swap", - ], + stylesheets: ["https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap"], noIndex: isSaas, customFields: { isSaas: isSaas, - markpromptProjectKey: process.env.DOCUSAURUS_MARKPROMPT_PROJECT_KEY || 'IeF3CUFCUQWuouZ8MP5Np9nES52QAtaA', + markpromptProjectKey: process.env.DOCUSAURUS_MARKPROMPT_PROJECT_KEY || "IeF3CUFCUQWuouZ8MP5Np9nES52QAtaA", }, themeConfig: { ...(!isSaas && { @@ -35,12 +33,8 @@ module.exports = { title: null, logo: { alt: "DataHub Logo", - src: `img/${ - isSaas ? "acryl" : "datahub" - }-logo-color-light-horizontal.svg`, - srcDark: `img/${ - isSaas ? "acryl" : "datahub" - }-logo-color-dark-horizontal.svg`, + src: `img/${isSaas ? "acryl" : "datahub"}-logo-color-light-horizontal.svg`, + srcDark: `img/${isSaas ? "acryl" : "datahub"}-logo-color-dark-horizontal.svg`, }, items: [ { @@ -50,7 +44,8 @@ module.exports = { position: "right", }, { - href: "/integrations", + to: "/integrations", + activeBasePath: "integrations", label: "Integrations", position: "right", }, @@ -70,8 +65,8 @@ module.exports = { position: "right", }, { - type: 'docsVersionDropdown', - position: 'right', + type: "docsVersionDropdown", + position: "right", dropdownActiveClassDisabled: true, }, { @@ -201,9 +196,7 @@ module.exports = { blog: false, theme: { customCss: [ - isSaas - ? require.resolve("./src/styles/acryl.scss") - : require.resolve("./src/styles/datahub.scss"), + isSaas ? require.resolve("./src/styles/acryl.scss") : require.resolve("./src/styles/datahub.scss"), require.resolve("./src/styles/global.scss"), require.resolve("./src/styles/sphinx.scss"), require.resolve("./src/styles/config-table.scss"), @@ -217,10 +210,7 @@ module.exports = { ], ], plugins: [ - [ - "@docusaurus/plugin-ideal-image", - { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }, - ], + ["@docusaurus/plugin-ideal-image", { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }], "docusaurus-plugin-sass", [ "docusaurus-graphql-plugin", diff --git a/docs-website/markdown-link-check-config.json b/docs-website/markdown-link-check-config.json new file mode 100644 index 0000000000000..2f5a51ada324e --- /dev/null +++ b/docs-website/markdown-link-check-config.json @@ -0,0 +1,41 @@ +{ + "ignorePatterns": [ + { + "pattern": "^https?://demo\\.datahubproject\\.io" + }, + { + "pattern": "^http://localhost" + }, + { + "pattern": "^/docs" + }, + { + "pattern": "^/integrations" + }, + { + "pattern": "^https?://www.linkedin.com" + }, + { + "pattern": "\\.md(#.*)?$" + }, + { + "pattern": "\\.json$" + }, + { + "pattern": "\\.txt$" + }, + { + "pattern": "\\.java$" + }, + { + "pattern": "^https://oauth2.googleapis.com/token" + }, + { + "pattern": "^https://login.microsoftonline.com/common/oauth2/na$" + }, + { + "pattern": "^https://github.com/datahub-project/datahub/assets/15873986/2f47d033-6c2b-483a-951d-e6d6b807f0d0%22%3E$" + } + ], + "aliveStatusCodes": [200, 206, 0, 999] +} diff --git a/docs-website/package.json b/docs-website/package.json index 400ef4143c786..eca6e5814d3c6 100644 --- a/docs-website/package.json +++ b/docs-website/package.json @@ -18,7 +18,9 @@ "generate-rsync": "mkdir -p genDocs genStatic && yarn _generate-docs && rsync -v --checksum -r -h -i --delete docs/ genDocs && rm -rf docs", "lint": "prettier -w generateDocsDir.ts sidebars.js src/pages/index.js", "lint-check": "prettier -l generateDocsDir.ts sidebars.js src/pages/index.js", - "lint-fix": "prettier --write generateDocsDir.ts sidebars.js src/pages/index.js" + "lint-fix": "prettier --write generateDocsDir.ts sidebars.js src/pages/index.js", + "_list-link-check-files": "find ./genDocs -name '*.md' -not \\( -path './genDocs/python-sdk/*' -o -path './genDocs/releases.md' \\)", + "check-links": "yarn run -s _list-link-check-files -print0 | xargs -0 -n1 -t markdown-link-check -q -c markdown-link-check-config.json" }, "dependencies": { "@ant-design/icons": "^4.7.0", @@ -37,6 +39,7 @@ "docusaurus-graphql-plugin": "0.5.0", "docusaurus-plugin-sass": "^0.2.1", "dotenv": "^16.0.1", + "markdown-link-check": "^3.11.2", "markprompt": "^0.1.7", "react": "^18.2.0", "react-dom": "18.2.0", diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index fcf82b786a1b9..b07cd0b03ce11 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -81,6 +81,13 @@ module.exports = { "docs/quick-ingestion-guides/powerbi/configuration", ], }, + { + Looker: [ + "docs/quick-ingestion-guides/looker/overview", + "docs/quick-ingestion-guides/looker/setup", + "docs/quick-ingestion-guides/looker/configuration", + ], + }, ], }, { @@ -437,6 +444,7 @@ module.exports = { Observability: [ "docs/managed-datahub/observe/freshness-assertions", "docs/managed-datahub/observe/volume-assertions", + "docs/managed-datahub/observe/custom-sql-assertions", ], }, ], @@ -597,6 +605,7 @@ module.exports = { }, { "Managed DataHub Release History": [ + "docs/managed-datahub/release-notes/v_0_2_11", "docs/managed-datahub/release-notes/v_0_2_10", "docs/managed-datahub/release-notes/v_0_2_9", "docs/managed-datahub/release-notes/v_0_2_8", diff --git a/docs-website/sphinx/Makefile b/docs-website/sphinx/Makefile index 00ece7ae25331..c01b45e322c67 100644 --- a/docs-website/sphinx/Makefile +++ b/docs-website/sphinx/Makefile @@ -22,7 +22,7 @@ $(VENV_SENTINEL): requirements.txt $(VENV_DIR)/bin/pip install -r requirements.txt touch $(VENV_SENTINEL) -.PHONY: help html doctest linkcheck clean serve md +.PHONY: help html doctest linkcheck clean clean_all serve md # Not using Python's http.server because it enables caching headers. serve: @@ -35,3 +35,6 @@ md: html # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). html doctest linkcheck clean: venv Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean_all: clean + -rm -rf $(VENV_DIR) diff --git a/docs-website/sphinx/requirements.txt b/docs-website/sphinx/requirements.txt index a63fd05853259..94ddd40579f0e 100644 --- a/docs-website/sphinx/requirements.txt +++ b/docs-website/sphinx/requirements.txt @@ -1,4 +1,4 @@ --e ../../metadata-ingestion[datahub-rest] +-e ../../metadata-ingestion[datahub-rest,sql-parsing] beautifulsoup4==4.11.2 Sphinx==6.1.3 sphinx-click==4.4.0 diff --git a/docs-website/src/components/Feedback/styles.module.scss b/docs-website/src/components/Feedback/styles.module.scss index b0fa3d7d1bd2b..ee22f6b055012 100644 --- a/docs-website/src/components/Feedback/styles.module.scss +++ b/docs-website/src/components/Feedback/styles.module.scss @@ -37,11 +37,11 @@ } .feedbackText { + font-family: var(--ifm-font-family-base); width: 100%; border: var(--ifm-hr-border-color) 1px solid; border-radius: 0.4rem; padding: 0.4rem; - font-family: "Manrope", sans-serif; } .feedbackButton { diff --git a/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss b/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss index 270877cd04a9f..0d874cad11790 100644 --- a/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss +++ b/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss @@ -325,7 +325,6 @@ button { padding-left: 1.5714286em; } .MarkpromptAnswer ol > li::marker { - font-weight: 400; color: var(--markprompt-foreground); } .MarkpromptAnswer ul > li::marker { @@ -454,7 +453,6 @@ button { background-color: var(--markprompt-muted); border: 1px solid var(--markprompt-border); overflow-x: auto; - font-weight: 400; font-size: 0.8571429em; line-height: 1.6666667; margin-top: 1.6666667em; diff --git a/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss b/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss new file mode 100644 index 0000000000000..fcd3666d03ddc --- /dev/null +++ b/docs-website/src/pages/_components/CardCTAs/cardCTAs.module.scss @@ -0,0 +1,24 @@ +.flexCol { + display: flex; +} + +.ctaCard { + flex-direction: row; + align-items: flex-start; + justify-content: space-between; + row-gap: 1rem; + padding: 1rem; + &:hover { + text-decoration: none; + border: 1px solid var(--ifm-color-primary); + background-color: var(--ifm-background-surface-color); + } + margin-bottom: 1rem; + flex: 1; +} + +.ctaHeading { + margin-bottom: 0; + display: flex; + align-items: center; +} diff --git a/docs-website/src/pages/_components/CardCTAs/index.js b/docs-website/src/pages/_components/CardCTAs/index.js new file mode 100644 index 0000000000000..d87c803b42818 --- /dev/null +++ b/docs-website/src/pages/_components/CardCTAs/index.js @@ -0,0 +1,52 @@ +import React from "react"; +import clsx from "clsx"; +import styles from "./cardCTAs.module.scss"; +import useBaseUrl from "@docusaurus/useBaseUrl"; +import { ArrowRightOutlined } from "@ant-design/icons"; + +const cardsContent = [ + { + label: "Data Mesh", + title: "Data Products, Delivered", + url: "https://www.acryldata.io/blog/data-products-in-datahub-everything-you-need-to-know", + }, + { + label: "Data Contracts", + title: "End-to-end Reliability in Data", + url: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management", + }, + { + label: "Shift Left", + title: "Developer-friendly Data Governance", + url: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2", + }, +]; + +const Card = ({ label, title, url }) => { + return ( + + ); +}; + +const CardCTAs = () => + cardsContent?.length > 0 ? ( +
+
+
+ {cardsContent.map((props, idx) => ( + + ))} +
+
+
+ ) : null; + +export default CardCTAs; diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss index c2103bb0782bd..6e4a623f469d5 100644 --- a/docs-website/src/pages/_components/Hero/hero.module.scss +++ b/docs-website/src/pages/_components/Hero/hero.module.scss @@ -74,3 +74,26 @@ margin-right: 0.5rem; } } + +.quickstartContent { + text-align: center; + padding: 2rem 0; + height: 100%; + margin: 2rem 0; + background: #34394d; + border-radius: var(--ifm-card-border-radius); +} + +.quickstartTitle { + color: #fafafa; +} + +.quickstartSubtitle { + font-size: 1.1rem; + color: gray; +} + +.quickstartCodeblock { + text-align: left; + padding: 0 20vh; +} diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js index b5fa04c80faee..ffa298b27a822 100644 --- a/docs-website/src/pages/_components/Hero/index.js +++ b/docs-website/src/pages/_components/Hero/index.js @@ -7,6 +7,7 @@ import { useColorMode } from "@docusaurus/theme-common"; import { QuestionCircleOutlined } from "@ant-design/icons"; import styles from "./hero.module.scss"; import CodeBlock from "@theme/CodeBlock"; +import CardCTAs from "../CardCTAs"; const HeroAnnouncement = ({ message, linkUrl, linkText }) => (
@@ -33,7 +34,11 @@ const Hero = ({}) => { complexity of your data ecosystem.

-Built with ❤️ by Acryl Data and LinkedIn. + Built with ❤️ by {" "} + + Acryl Data + {" "} + and LinkedIn.

Get Started → @@ -43,11 +48,12 @@ Built with ❤️ by DataHub Flow Diagram -
-

Get Started Now

-

Run the following command to get started with DataHub.

-
+
+

Get Started Now

+

Run the following command to get started with DataHub.

+
python3 -m pip install --upgrade pip wheel setuptools
python3 -m pip install --upgrade acryl-datahub
diff --git a/docs-website/src/pages/_components/Section/section.module.scss b/docs-website/src/pages/_components/Section/section.module.scss index 4b68ce5533d4d..7a39a60b6fa4c 100644 --- a/docs-website/src/pages/_components/Section/section.module.scss +++ b/docs-website/src/pages/_components/Section/section.module.scss @@ -9,7 +9,6 @@ .sectionTitle { font-size: 2.5rem; margin-bottom: 3rem; - font-weight: normal; text-align: center; } diff --git a/docs-website/src/pages/docs/_components/DropDownFilter/search.module.scss b/docs-website/src/pages/docs/_components/DropDownFilter/search.module.scss index 17e5f22490664..2ae0f5c849ba9 100644 --- a/docs-website/src/pages/docs/_components/DropDownFilter/search.module.scss +++ b/docs-website/src/pages/docs/_components/DropDownFilter/search.module.scss @@ -74,7 +74,6 @@ } .searchResultItemHeading { - font-weight: 400; margin-bottom: 0; } diff --git a/docs-website/src/pages/docs/_components/GuideList/guidelist.module.scss b/docs-website/src/pages/docs/_components/GuideList/guidelist.module.scss index a8f279e74ef87..46b1b01408592 100644 --- a/docs-website/src/pages/docs/_components/GuideList/guidelist.module.scss +++ b/docs-website/src/pages/docs/_components/GuideList/guidelist.module.scss @@ -16,6 +16,9 @@ display: block; margin-bottom: 0.25rem; } + strong { + font-weight: 600; + } span { font-size: 0.875rem; line-height: 1.25em; diff --git a/docs-website/src/pages/docs/_components/GuideList/index.jsx b/docs-website/src/pages/docs/_components/GuideList/index.jsx index 3a47e1691aeea..9d4b50b5f5159 100644 --- a/docs-website/src/pages/docs/_components/GuideList/index.jsx +++ b/docs-website/src/pages/docs/_components/GuideList/index.jsx @@ -19,7 +19,7 @@ const GuideList = ({ title, content, seeMoreLink }) => content?.length > 0 ? (
-

{title}

+

{title}

{content.map((props, idx) => ( diff --git a/docs-website/src/pages/docs/_components/QuickLinkCard/quicklinkcard.module.scss b/docs-website/src/pages/docs/_components/QuickLinkCard/quicklinkcard.module.scss index cf239ff8643ba..4fbbc4583d662 100644 --- a/docs-website/src/pages/docs/_components/QuickLinkCard/quicklinkcard.module.scss +++ b/docs-website/src/pages/docs/_components/QuickLinkCard/quicklinkcard.module.scss @@ -20,6 +20,10 @@ display: block; margin-bottom: 0.25rem; } + strong { + font-weight: 600; + } + span { font-size: 0.875rem; line-height: 1.25em; diff --git a/docs-website/src/pages/docs/_components/SearchBar/search.module.scss b/docs-website/src/pages/docs/_components/SearchBar/search.module.scss index 6faaf19c7e603..d85607b08e4e7 100644 --- a/docs-website/src/pages/docs/_components/SearchBar/search.module.scss +++ b/docs-website/src/pages/docs/_components/SearchBar/search.module.scss @@ -91,7 +91,6 @@ } .searchResultItemHeading { - font-weight: 400; margin-bottom: 0; } diff --git a/docs-website/src/pages/docs/index.js b/docs-website/src/pages/docs/index.js index a0462091a046d..0e8bfdcf3b9d7 100644 --- a/docs-website/src/pages/docs/index.js +++ b/docs-website/src/pages/docs/index.js @@ -180,8 +180,8 @@ const quickLinkContent = [ { title: "Developer Guides", icon: , - description: "Interact with DataHub programmatically ", - to: "/docs/cli", + description: "Interact with DataHub programmatically", + to: "/docs/api/datahub-apis", }, { title: "Feature Guides", diff --git a/docs-website/src/styles/acryl.scss b/docs-website/src/styles/acryl.scss index 8eb9b375830bf..8bb25ca28cb38 100644 --- a/docs-website/src/styles/acryl.scss +++ b/docs-website/src/styles/acryl.scss @@ -7,4 +7,7 @@ --ifm-color-primary-light: #13beb0; --ifm-color-primary-lighter: #14c7b8; --ifm-color-primary-lightest: #16e1d0; + + // Custom + --ifm-color-primary-opaque: rgba(17, 173, 160, 0.1); } diff --git a/docs-website/src/styles/datahub.scss b/docs-website/src/styles/datahub.scss index a41359c24b833..8d8f0bdd6daa9 100644 --- a/docs-website/src/styles/datahub.scss +++ b/docs-website/src/styles/datahub.scss @@ -7,4 +7,6 @@ --ifm-color-primary-light: #349dff; --ifm-color-primary-lighter: #42a4ff; --ifm-color-primary-lightest: #6cb8ff; + + --ifm-color-primary-opaque: rgba(24, 144, 255, 0.1); } diff --git a/docs-website/src/styles/global.scss b/docs-website/src/styles/global.scss index 013e9fb9f0d9a..55a54876b41ac 100644 --- a/docs-website/src/styles/global.scss +++ b/docs-website/src/styles/global.scss @@ -7,16 +7,23 @@ /* You can override the default Infima variables here. */ :root { - font-family: "Manrope", sans-serif; + // Global --ifm-background-color: #ffffff; + --ifm-global-spacing: 1rem; /* Typography */ - --ifm-heading-font-weight: 600; - --ifm-font-weight-semibold: 600; + --ifm-font-size-base: 95%; + --ifm-heading-font-weight: 700; --ifm-code-font-size: 0.9em; --ifm-heading-color: #000000; + --ifm-heading-font-family: "Manrope", sans-serif; + --ifm-font-family-base: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif, "Apple Color Emoji", + "Segoe UI Emoji", "Segoe UI Symbol"; + --ifm-font-family-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + /* Buttons */ --ifm-button-border-radius: 1000em; + --ifm-button-font-weight: 600; /* Navbar */ --ifm-navbar-background-color: var(--ifm-background-color); @@ -30,6 +37,7 @@ --ifm-hr-border-width: 1px 0 0 0; --ifm-hr-border-color: #e3e3e3; --ifm-hr-background-color: #e3e3e3; + /* More Colors */ --ifm-hero-background-color: var(--ifm-background-color); --ifm-background-surface-color: #fafafa; @@ -37,6 +45,19 @@ /* Cards */ --ifm-card-background-color: --ifm-background-color; --ifm-card-border-radius: calc(var(--ifm-global-radius) * 1.5); + + /* Menu */ + --ifm-menu-link-padding-vertical: 0.6rem; + --ifm-menu-link-padding-horizontal: 1rem; + --ifm-menu-link-sublist-icon: url('data:image/svg+xml;utf8,'); + --ifm-menu-color-background-hover: var(--ifm-color-primary-opaque); + --ifm-menu-color-background-active: var(--ifm-color-primary-opaque); + + /* TOC */ + --ifm-toc-padding-vertical: 1.25rem; + + /* Breadcrumbs */ + --ifm-breadcrumb-item-background-active: var(--ifm-color-primary-opaque); } html[data-theme="dark"] { @@ -49,16 +70,29 @@ html[data-theme="dark"] { .button--primary { color: #fff; } + + .invert-on-dark { + filter: invert(1); + } +} + +/* Main Docs Content Area */ + +main { + padding-top: 1rem; } -h1 { - font-weight: 400; +.markdown, +main > h1 { + margin-top: 1rem; } -html[data-theme="dark"] .invert-on-dark { - filter: invert(1); +[class*="docItemCol"] { + padding: 0 2rem; } +/* Custom Utility */ + .row--centered { align-items: center; } @@ -67,6 +101,8 @@ html[data-theme="dark"] .invert-on-dark { padding: 5vh 0; } +/* Announcement Bar */ + div[class^="announcementBar"] { z-index: calc(var(--ifm-z-index-fixed) - 1); div { @@ -106,6 +142,8 @@ div[class^="announcementBar"] { } } +/** Navbar */ + @media only screen and (max-width: 1050px) { .navbar__toggle { display: inherit; @@ -158,16 +196,7 @@ div[class^="announcementBar"] { } } -.footer { - .footer__copyright { - text-align: left; - font-size: 0.8em; - opacity: 0.5; - } - &.footer--dark { - --ifm-footer-background-color: #000000; - } -} +/* Misc */ .button { white-space: initial; @@ -192,64 +221,93 @@ div[class^="announcementBar"] { } } +.footer { + .footer__copyright { + text-align: left; + font-size: 0.8em; + opacity: 0.5; + } + &.footer--dark { + --ifm-footer-background-color: #000000; + } +} + +/* Hero */ + .hero { padding: 5vh 0; -} -.hero__subtitle { - font-size: 1.25em; - margin: 1rem auto 3rem; - max-width: 800px; -} -.hero__content { - text-align: center; - padding: 2rem 0; - height: 100%; -} + .hero__subtitle { + font-size: 1.25em; + margin: 1rem auto 3rem; + max-width: 800px; -.quickstart__content { - text-align: center; - padding: 2rem 0; - height: 100%; - margin: 2rem 0; - background: #34394d; - border-radius: var(--ifm-card-border-radius); + img { + vertical-align: middle; + margin-top: -0.3em; + } + } + .hero__content { + text-align: center; + padding: 2rem 0; + height: 100%; + } } -.quickstart__title { - color: #fafafa; -} +/* Sidebar Menu */ + +.menu .theme-doc-sidebar-menu { + ul li.saasOnly a.menu__link { + &:after { + content: ""; + display: block; + width: 20px; + height: 20px; + flex-shrink: 0; + margin-right: auto; + margin-left: 10px; + opacity: 0.5; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='64 64 896 896' focusable='false' data-icon='cloud' width='1em' height='1em' fill='currentColor' aria-hidden='true'%3E%3Cpath d='M811.4 418.7C765.6 297.9 648.9 212 512.2 212S258.8 297.8 213 418.6C127.3 441.1 64 519.1 64 612c0 110.5 89.5 200 199.9 200h496.2C870.5 812 960 722.5 960 612c0-92.7-63.1-170.7-148.6-193.3zm36.3 281a123.07 123.07 0 01-87.6 36.3H263.9c-33.1 0-64.2-12.9-87.6-36.3A123.3 123.3 0 01140 612c0-28 9.1-54.3 26.2-76.3a125.7 125.7 0 0166.1-43.7l37.9-9.9 13.9-36.6c8.6-22.8 20.6-44.1 35.7-63.4a245.6 245.6 0 0152.4-49.9c41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.2c19.9 14 37.5 30.8 52.4 49.9 15.1 19.3 27.1 40.7 35.7 63.4l13.8 36.5 37.8 10c54.3 14.5 92.1 63.8 92.1 120 0 33.1-12.9 64.3-36.3 87.7z'%3E%3C/path%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: 50% 50%; + background-size: 20px 20px; + [data-theme="dark"] & { + filter: invert(1); + } + } + } -.quickstart__subtitle { - font-size: 1.1rem; - color: gray; -} + .theme-doc-sidebar-item-category-level-1 .menu__link { + font-weight: 400; + } + + .theme-doc-sidebar-item-category-level-1 .menu__link--active { + font-weight: 600; + } -.quickstart__codeblock { - text-align: left; - padding: 0 20vh; + .theme-doc-sidebar-item-category-level-1 > div > a:first-child { + color: var(--ifm-navbar-link-color); + font-weight: 600; + padding: calc(var(--ifm-menu-link-padding-vertical) + 0.2rem) var(--ifm-menu-link-padding-horizontal); + } + .theme-doc-sidebar-item-category-level-1 > div > a.menu__link--active { + color: var(--ifm-navbar-link-color); + font-weight: 600; + } } -.theme-doc-sidebar-menu ul li.saasOnly a.menu__link { - &:after { - content: ""; - display: block; - width: 20px; - height: 20px; - flex-shrink: 0; - margin-right: auto; - margin-left: 10px; - opacity: 0.5; - background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='64 64 896 896' focusable='false' data-icon='cloud' width='1em' height='1em' fill='currentColor' aria-hidden='true'%3E%3Cpath d='M811.4 418.7C765.6 297.9 648.9 212 512.2 212S258.8 297.8 213 418.6C127.3 441.1 64 519.1 64 612c0 110.5 89.5 200 199.9 200h496.2C870.5 812 960 722.5 960 612c0-92.7-63.1-170.7-148.6-193.3zm36.3 281a123.07 123.07 0 01-87.6 36.3H263.9c-33.1 0-64.2-12.9-87.6-36.3A123.3 123.3 0 01140 612c0-28 9.1-54.3 26.2-76.3a125.7 125.7 0 0166.1-43.7l37.9-9.9 13.9-36.6c8.6-22.8 20.6-44.1 35.7-63.4a245.6 245.6 0 0152.4-49.9c41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.2c19.9 14 37.5 30.8 52.4 49.9 15.1 19.3 27.1 40.7 35.7 63.4l13.8 36.5 37.8 10c54.3 14.5 92.1 63.8 92.1 120 0 33.1-12.9 64.3-36.3 87.7z'%3E%3C/path%3E%3C/svg%3E"); - background-repeat: no-repeat; - background-position: 50% 50%; - background-size: 20px 20px; - [data-theme="dark"] & { - filter: invert(1); - } +/* TOC */ +.table-of-contents { + padding-left: 1.5rem; + font-size: 0.9rem; + line-height: 1rem; + .table-of-contents__link--active { + font-weight: 600; } } +/* Search */ + [data-theme="light"] .DocSearch { /* --docsearch-primary-color: var(--ifm-color-primary); */ /* --docsearch-text-color: var(--ifm-font-color-base); */ @@ -285,18 +343,3 @@ div[class^="announcementBar"] { --docsearch-footer-background: var(--ifm-background-surface-color); --docsearch-key-gradient: linear-gradient(-26.5deg, var(--ifm-color-emphasis-200) 0%, var(--ifm-color-emphasis-100) 100%); } - -.theme-doc-sidebar-item-category-level-1 > div > a:first-child { - color: var(--ifm-navbar-link-color); - font-size: 17px; -} - -.theme-doc-sidebar-item-category-level-1 > div > a.menu__link--active { - color: var(--ifm-menu-color-active); - font-size: 17px; -} - -/* Increase padding for levels greater than 1 */ -[class^="theme-doc-sidebar-item"][class*="-level-"]:not(.theme-doc-sidebar-item-category-level-1) { - padding-left: 8px; -} diff --git a/docs-website/static/img/acryl-logo-transparent-mark.svg b/docs-website/static/img/acryl-logo-transparent-mark.svg new file mode 100644 index 0000000000000..87c9904baaf8c --- /dev/null +++ b/docs-website/static/img/acryl-logo-transparent-mark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs-website/versions.json b/docs-website/versions.json index 0b79ac9498e06..a5493c26a4c65 100644 --- a/docs-website/versions.json +++ b/docs-website/versions.json @@ -1,3 +1,4 @@ [ + "0.11.0", "0.10.5" ] diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index 209a57a43dab0..5698029bff70a 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -2986,6 +2986,13 @@ dependencies: "@types/node" "*" +"@types/websocket@^1.0.3": + version "1.0.6" + resolved "https://registry.yarnpkg.com/@types/websocket/-/websocket-1.0.6.tgz#ec8dce5915741632ac3a4b1f951b6d4156e32d03" + integrity sha512-JXkliwz93B2cMWOI1ukElQBPN88vMg3CruvW4KVSKpflt3NyNCJImnhIuB/f97rG7kakqRJGFiwkA895Kn02Dg== + dependencies: + "@types/node" "*" + "@types/ws@^8.5.5": version "8.5.5" resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.5.tgz#af587964aa06682702ee6dcbc7be41a80e4b28eb" @@ -3407,6 +3414,11 @@ async-validator@^4.1.0: resolved "https://registry.yarnpkg.com/async-validator/-/async-validator-4.2.5.tgz#c96ea3332a521699d0afaaceed510a54656c6339" integrity sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg== +async@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -3758,6 +3770,11 @@ chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: ansi-styles "^4.1.0" supports-color "^7.1.0" +chalk@^5.2.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385" + integrity sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w== + character-entities-legacy@^1.0.0: version "1.1.4" resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz#94bc1845dce70a5bb9d2ecc748725661293d8fc1" @@ -3790,7 +3807,7 @@ cheerio-select@^2.1.0: domhandler "^5.0.3" domutils "^3.0.1" -cheerio@^1.0.0-rc.12: +cheerio@^1.0.0-rc.10, cheerio@^1.0.0-rc.12: version "1.0.0-rc.12" resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.12.tgz#788bf7466506b1c6bf5fae51d24a2c4d62e47683" integrity sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q== @@ -3977,6 +3994,11 @@ comma-separated-tokens@^2.0.0: resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== +commander@^10.0.1: + version "10.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" + integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== + commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -4378,6 +4400,13 @@ debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1: dependencies: ms "2.1.2" +debug@^3.2.6: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + decode-named-character-reference@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz#daabac9690874c394c81e4162a0304b35d824f0e" @@ -5544,6 +5573,13 @@ html-entities@^2.3.2: resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.4.0.tgz#edd0cee70402584c8c76cc2c0556db09d1f45061" integrity sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ== +html-link-extractor@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/html-link-extractor/-/html-link-extractor-1.0.5.tgz#a4be345cb13b8c3352d82b28c8b124bb7bf5dd6f" + integrity sha512-ADd49pudM157uWHwHQPUSX4ssMsvR/yHIswOR5CUfBdK9g9ZYGMhVSE6KZVHJ6kCkR0gH4htsfzU6zECDNVwyw== + dependencies: + cheerio "^1.0.0-rc.10" + html-minifier-terser@^6.0.2, html-minifier-terser@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz#bfc818934cc07918f6b3669f5774ecdfd48f32ab" @@ -5666,6 +5702,13 @@ iconv-lite@0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" +iconv-lite@^0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + icss-utils@^5.0.0, icss-utils@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" @@ -5788,6 +5831,11 @@ ipaddr.js@^2.0.1: resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.1.0.tgz#2119bc447ff8c257753b196fc5f1ce08a4cdf39f" integrity sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ== +is-absolute-url@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-4.0.1.tgz#16e4d487d4fded05cfe0685e53ec86804a5e94dc" + integrity sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A== + is-alphabetical@1.0.4, is-alphabetical@^1.0.0: version "1.0.4" resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.4.tgz#9e7d6b94916be22153745d184c298cbf986a686d" @@ -5956,6 +6004,13 @@ is-regexp@^1.0.0: resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" integrity sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA== +is-relative-url@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/is-relative-url/-/is-relative-url-4.0.0.tgz#4d8371999ff6033b76e4d9972fb5bf496fddfa97" + integrity sha512-PkzoL1qKAYXNFct5IKdKRH/iBQou/oCC85QhXj6WKtUQBliZ4Yfd3Zk27RHu9KQG8r6zgvAA2AQKC9p+rqTszg== + dependencies: + is-absolute-url "^4.0.1" + is-root@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-root/-/is-root-2.1.0.tgz#809e18129cf1129644302a4f8544035d51984a9c" @@ -6003,6 +6058,13 @@ isarray@~1.0.0: resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== +isemail@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.2.0.tgz#59310a021931a9fb06bbb51e155ce0b3f236832c" + integrity sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg== + dependencies: + punycode "2.x.x" + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -6198,6 +6260,16 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +link-check@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/link-check/-/link-check-5.2.0.tgz#595a339d305900bed8c1302f4342a29c366bf478" + integrity sha512-xRbhYLaGDw7eRDTibTAcl6fXtmUQ13vkezQiTqshHHdGueQeumgxxmQMIOmJYsh2p8BF08t8thhDQ++EAOOq3w== + dependencies: + is-relative-url "^4.0.0" + isemail "^3.2.0" + ms "^2.1.3" + needle "^3.1.0" + loader-runner@^4.2.0: version "4.3.0" resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1" @@ -6359,6 +6431,28 @@ markdown-escapes@^1.0.0: resolved "https://registry.yarnpkg.com/markdown-escapes/-/markdown-escapes-1.0.4.tgz#c95415ef451499d7602b91095f3c8e8975f78535" integrity sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg== +markdown-link-check@^3.11.2: + version "3.11.2" + resolved "https://registry.yarnpkg.com/markdown-link-check/-/markdown-link-check-3.11.2.tgz#303a8a03d4a34c42ef3158e0b245bced26b5d904" + integrity sha512-zave+vI4AMeLp0FlUllAwGbNytSKsS3R2Zgtf3ufVT892Z/L6Ro9osZwE9PNA7s0IkJ4onnuHqatpsaCiAShJw== + dependencies: + async "^3.2.4" + chalk "^5.2.0" + commander "^10.0.1" + link-check "^5.2.0" + lodash "^4.17.21" + markdown-link-extractor "^3.1.0" + needle "^3.2.0" + progress "^2.0.3" + +markdown-link-extractor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/markdown-link-extractor/-/markdown-link-extractor-3.1.0.tgz#0d5a703630d791a9e2017449e1a9b294f2d2b676" + integrity sha512-r0NEbP1dsM+IqB62Ru9TXLP/HDaTdBNIeylYXumuBi6Xv4ufjE1/g3TnslYL8VNqNcGAGbMptQFHrrdfoZ/Sug== + dependencies: + html-link-extractor "^1.0.5" + marked "^4.1.0" + markdown-table@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-3.0.3.tgz#e6331d30e493127e031dd385488b5bd326e4a6bd" @@ -6369,6 +6463,11 @@ marked@^2.0.3: resolved "https://registry.yarnpkg.com/marked/-/marked-2.1.3.tgz#bd017cef6431724fd4b27e0657f5ceb14bff3753" integrity sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA== +marked@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-4.3.0.tgz#796362821b019f734054582038b116481b456cf3" + integrity sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A== + markprompt@^0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/markprompt/-/markprompt-0.1.7.tgz#fa049e11109d93372c45c38b3ca40bd5fdf751ea" @@ -6971,7 +7070,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3: +ms@2.1.3, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -6994,6 +7093,15 @@ napi-build-utils@^1.0.1: resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== +needle@^3.1.0, needle@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/needle/-/needle-3.2.0.tgz#07d240ebcabfd65c76c03afae7f6defe6469df44" + integrity sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ== + dependencies: + debug "^3.2.6" + iconv-lite "^0.6.3" + sax "^1.2.4" + negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" @@ -7053,7 +7161,6 @@ node-forge@^1: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== - node-gyp-build@^4.3.0: version "4.6.1" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.1.tgz#24b6d075e5e391b8d5539d98c7fc5c210cac8a3e" @@ -7747,6 +7854,11 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== +progress@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + promise@^7.1.1: version "7.3.1" resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf" @@ -7799,16 +7911,16 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" +punycode@2.x.x, punycode@^2.1.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f" + integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== + punycode@^1.3.2: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" integrity sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ== -punycode@^2.1.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f" - integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== - pupa@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" @@ -8783,7 +8895,7 @@ safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -"safer-buffer@>= 2.1.2 < 3": +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -9903,6 +10015,10 @@ use-sidecar@^1.1.2: detect-node-es "^1.1.0" tslib "^2.0.0" +use-sync-external-store@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a" + integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== utf-8-validate@^5.0.2: version "5.0.10" @@ -9911,12 +10027,6 @@ utf-8-validate@^5.0.2: dependencies: node-gyp-build "^4.3.0" -use-sync-external-store@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a" - integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== - - util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" diff --git a/docs/advanced/no-code-modeling.md b/docs/advanced/no-code-modeling.md index d76b776d3dddb..172e63f821eab 100644 --- a/docs/advanced/no-code-modeling.md +++ b/docs/advanced/no-code-modeling.md @@ -100,10 +100,9 @@ Currently, there are various models in GMS: 1. [Urn](https://github.com/datahub-project/datahub/blob/master/li-utils/src/main/pegasus/com/linkedin/common/DatasetUrn.pdl) - Structs composing primary keys 2. [Root] [Snapshots](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl) - Container of aspects 3. [Aspects](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DashboardAspect.pdl) - Optional container of fields -4. [Values](https://github.com/datahub-project/datahub/blob/master/gms/api/src/main/pegasus/com/linkedin/dataset/Dataset.pdl), [Keys](https://github.com/datahub-project/datahub/blob/master/gms/api/src/main/pegasus/com/linkedin/dataset/DatasetKey.pdl) - Model returned by GMS [Rest.li](http://rest.li) API (public facing) -5. [Entities](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/entity/DatasetEntity.pdl) - Records with fields derived from the URN. Used only in graph / relationships -6. [Relationships](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/relationship/Relationship.pdl) - Edges between 2 entities with optional edge properties -7. [Search Documents](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/search/ChartDocument.pdl) - Flat documents for indexing within Elastic index +4. [Keys](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DatasetKey.pdl) - Model returned by GMS [Rest.li](http://rest.li) API (public facing) +5. [Relationships](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/EntityRelationship.pdl) - Edges between 2 entities with optional edge properties +6. Search Documents - Flat documents for indexing within Elastic index - And corresponding index [mappings.json](https://github.com/datahub-project/datahub/blob/master/gms/impl/src/main/resources/index/chart/mappings.json), [settings.json](https://github.com/datahub-project/datahub/blob/master/gms/impl/src/main/resources/index/chart/settings.json) Various components of GMS depend on / make assumptions about these model types: diff --git a/docs/api/graphql/how-to-set-up-graphql.md b/docs/api/graphql/how-to-set-up-graphql.md index 584bf34ad3f92..2be2f935b12b1 100644 --- a/docs/api/graphql/how-to-set-up-graphql.md +++ b/docs/api/graphql/how-to-set-up-graphql.md @@ -68,7 +68,7 @@ In the request body, select the `GraphQL` option and enter your GraphQL query in

-Please refer to [Querying with GraphQL](https://learning.postman.com/docs/sending-requests/graphql/graphql/) in the Postman documentation for more information. +Please refer to [Querying with GraphQL](https://learning.postman.com/docs/sending-requests/graphql/graphql-overview/) in the Postman documentation for more information. ### Authentication + Authorization diff --git a/docs/api/tutorials/ml.md b/docs/api/tutorials/ml.md index cb77556d48ebf..e88c941c90467 100644 --- a/docs/api/tutorials/ml.md +++ b/docs/api/tutorials/ml.md @@ -7,11 +7,12 @@ import TabItem from '@theme/TabItem'; Machine learning systems have become a crucial feature in modern data stacks. However, the relationships between the different components of a machine learning system, such as features, models, and feature tables, can be complex. -Thus, it is essential for these systems to be discoverable to facilitate easy access and utilization by other members of the organization. +DataHub makes these relationships discoverable and facilitate utilization by other members of the organization. -For more information on ML entities, please refer to the following docs: +For technical details on ML entities, please refer to the following docs: - [MlFeature](/docs/generated/metamodel/entities/mlFeature.md) +- [MlPrimaryKey](/docs/generated/metamodel/entities/mlPrimaryKey.md) - [MlFeatureTable](/docs/generated/metamodel/entities/mlFeatureTable.md) - [MlModel](/docs/generated/metamodel/entities/mlModel.md) - [MlModelGroup](/docs/generated/metamodel/entities/mlModelGroup.md) @@ -20,9 +21,11 @@ For more information on ML entities, please refer to the following docs: This guide will show you how to -- Create ML entities: MlFeature, MlFeatureTable, MlModel, MlModelGroup -- Read ML entities: MlFeature, MlFeatureTable, MlModel, MlModelGroup -- Attach MlFeatureTable or MlModel to MlFeature +- Create ML entities: MlFeature, MlFeatureTable, MlModel, MlModelGroup, MlPrimaryKey +- Read ML entities: MlFeature, MlFeatureTable, MlModel, MlModelGroup, MlPrimaryKey +- Attach MlModel to MlFeature +- Attach MlFeatures to MlFeatureTable +- Attached MlFeatures to upstream Datasets that power them ## Prerequisites @@ -33,6 +36,8 @@ For detailed steps, please refer to [Datahub Quickstart Guide](/docs/quickstart. ### Create MlFeature +An ML Feature represents an instance of a feature that can be used across different machine learning models. Features are organized into Feature Tables to be consumed by machine learning models. For example, if we were modeling features for a Users Feature Table, the Features would be `age`, `sign_up_date`, `active_in_past_30_days` and so forth.Using Features in DataHub allows users to see the sources a feature was generated from and how a feature is used to train models. + @@ -40,13 +45,31 @@ For detailed steps, please refer to [Datahub Quickstart Guide](/docs/quickstart. {{ inline /metadata-ingestion/examples/library/create_mlfeature.py show_path_as_comment }} ``` -Note that when creating a feature, you can access a list of data sources using `sources`. +Note that when creating a feature, you create upstream lineage to the data warehouse using `sources`. + + + + +### Create MlPrimaryKey + +An ML Primary Key represents a specific element of a Feature Table that indicates what group the other features belong to. For example, if a Feature Table contained features for Users, the ML Primary Key would likely be `user_id` or some similar unique identifier for a user. Using ML Primary Keys in DataHub allow users to indicate how ML Feature Tables are structured. + + + + +```python +{{ inline /metadata-ingestion/examples/library/create_mlprimarykey.py show_path_as_comment }} +``` + +Note that when creating a primary key, you create upstream lineage to the data warehouse using `sources`. ### Create MlFeatureTable +A feature table represents a group of similar Features that can all be used together to train a model. For example, if there was a Users Feature Table, it would contain documentation around how to use the Users collection of Features and references to each Feature and ML Primary Key contained within it. + @@ -54,14 +77,14 @@ Note that when creating a feature, you can access a list of data sources using ` {{ inline /metadata-ingestion/examples/library/create_mlfeature_table.py show_path_as_comment }} ``` -Note that when creating a feature table, you can access a list of features using `mlFeatures`. +Note that when creating a feature table, you connect the table to its features and primary key using `mlFeatures` and `mlPrimaryKeys`. ### Create MlModel -Please note that an MlModel represents the outcome of a single training run for a model, not the collective results of all model runs. +An ML Model in Acryl represents an individual version of a trained Machine Learning Model. Another way to think about the ML Model entity is as an istance of a training run. An ML Model entity tracks the exact ML Features used in that instance of training, along with the training results. This entity does not represents all versions of a ML Model. For example, if we train a model for homepage customization on a certain day, that would be a ML Model in DataHub. If you re-train the model the next day off of new data or with different parameters, that would produce a second ML Model entity. @@ -70,15 +93,15 @@ Please note that an MlModel represents the outcome of a single training run for {{ inline /metadata-ingestion/examples/library/create_mlmodel.py show_path_as_comment }} ``` -Note that when creating a model, you can access a list of features using `mlFeatures`. -Additionally, you can access the relationship to model groups with `groups`. +Note that when creating a model, you link it to a list of features using `mlFeatures`. This indicates how the individual instance of the model was trained. +Additionally, you can access the relationship to model groups with `groups`. An ML Model is connected to the warehouse tables it depends on via its dependency on the ML Features it reads from. ### Create MlModelGroup -Please note that an MlModelGroup serves as a container for all the runs of a single ML model. +An ML Model Group represents the grouping of all training runs of a single Machine Learning model category. It will store documentation about the group of ML Models, along with references to each individual ML Model instance. @@ -94,18 +117,14 @@ Please note that an MlModelGroup serves as a container for all the runs of a sin You can search the entities in DataHub UI. -

- -

- ## Read ML Entities ### Read MLFeature @@ -192,6 +211,93 @@ Expected response: +### Read MlPrimaryKey + + + + +```json +query { + mlPrimaryKey(urn: "urn:li:mlPrimaryKey:(user_features,user_id)"){ + name + featureNamespace + description + dataType + properties { + description + dataType + version { + versionTag + } + } + } +} +``` + +Expected response: + +```json +{ + "data": { + "mlPrimaryKey": { + "name": "user_id", + "featureNamespace": "user_features", + "description": "User's internal ID", + "dataType": "ORDINAL", + "properties": { + "description": "User's internal ID", + "dataType": "ORDINAL", + "version": null + } + } + }, + "extensions": {} +} +``` + + + + +```json +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "query { mlPrimaryKey(urn: \"urn:li:mlPrimaryKey:(user_features,user_id)\"){ name featureNamespace description dataType properties { description dataType version { versionTag } } }}" +}' +``` + +Expected response: + +```json +{ + "data": { + "mlPrimaryKey": { + "name": "user_id", + "featureNamespace": "user_features", + "description": "User's internal ID", + "dataType": "ORDINAL", + "properties": { + "description": "User's internal ID", + "dataType": "ORDINAL", + "version": null + } + } + }, + "extensions": {} +} +``` + + + + +```python +{{ inline /metadata-ingestion/examples/library/read_mlprimarykey.py show_path_as_comment }} +``` + + + + ### Read MLFeatureTable @@ -232,8 +338,7 @@ Expected Response: { "name": "test_BOOL_LIST_feature" }, - ... - { + ...{ "name": "test_STRING_feature" } ] @@ -273,8 +378,7 @@ Expected Response: { "name": "test_BOOL_LIST_feature" }, - ... - { + ...{ "name": "test_STRING_feature" } ] @@ -507,14 +611,10 @@ Expected Response: (Note that this entity does not exist in the sample ingestion You can access to `Features` or `Group` Tab of each entity to view the added entities. -

- -

- diff --git a/docs/architecture/architecture.md b/docs/architecture/architecture.md index 6a9c1860d71b0..20f18f09d949b 100644 --- a/docs/architecture/architecture.md +++ b/docs/architecture/architecture.md @@ -17,7 +17,7 @@ The figures below describe the high-level architecture of DataHub.

- +

diff --git a/docs/authentication/guides/add-users.md b/docs/authentication/guides/add-users.md index f5dfc83201083..d380cacd6665e 100644 --- a/docs/authentication/guides/add-users.md +++ b/docs/authentication/guides/add-users.md @@ -19,13 +19,13 @@ To do so, navigate to the **Users & Groups** section inside of Settings page. He do not have the correct privileges to invite users, this button will be disabled.

- +

To invite new users, simply share the link with others inside your organization.

- +

When a new user visits the link, they will be directed to a sign up screen where they can create their DataHub account. @@ -37,13 +37,13 @@ and click **Reset user password** inside the menu dropdown on the right hand sid `Manage User Credentials` [Platform Privilege](../../authorization/access-policies-guide.md) in order to reset passwords.

- +

To reset the password, simply share the password reset link with the user who needs to change their password. Password reset links expire after 24 hours.

- +

# Configuring Single Sign-On with OpenID Connect diff --git a/docs/authentication/guides/sso/configure-oidc-react.md b/docs/authentication/guides/sso/configure-oidc-react.md index d27792ce3967b..512d6adbf916f 100644 --- a/docs/authentication/guides/sso/configure-oidc-react.md +++ b/docs/authentication/guides/sso/configure-oidc-react.md @@ -26,7 +26,7 @@ please see [this guide](../jaas.md) to mount a custom user.props file for a JAAS To configure OIDC in React, you will most often need to register yourself as a client with your identity provider (Google, Okta, etc). Each provider may have their own instructions. Provided below are links to examples for Okta, Google, Azure AD, & Keycloak. -- [Registering an App in Okta](https://developer.okta.com/docs/guides/add-an-external-idp/apple/register-app-in-okta/) +- [Registering an App in Okta](https://developer.okta.com/docs/guides/add-an-external-idp/openidconnect/main/) - [OpenID Connect in Google Identity](https://developers.google.com/identity/protocols/oauth2/openid-connect) - [OpenID Connect authentication with Azure Active Directory](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/auth-oidc) - [Keycloak - Securing Applications and Services Guide](https://www.keycloak.org/docs/latest/securing_apps/) diff --git a/docs/authorization/policies.md b/docs/authorization/policies.md index 27d8b15e5a73a..e3606f2a3e48d 100644 --- a/docs/authorization/policies.md +++ b/docs/authorization/policies.md @@ -145,28 +145,31 @@ For example, the following resource filter will apply the policy to datasets, ch ```json { - "resource": { - "criteria": [ - { - "field": "resource_type", - "values": [ - "dataset", - "chart", - "dashboard" - ], - "condition": "EQUALS" - }, - { - "field": "domain", - "values": [ - "urn:li:domain:domain1" - ], - "condition": "EQUALS" + "resources": { + "filter": { + "criteria": [ + { + "field": "RESOURCE_TYPE", + "condition": "EQUALS", + "values": [ + "dataset", + "chart", + "dashboard" + ] + }, + { + "field": "DOMAIN", + "values": [ + "urn:li:domain:domain1" + ], + "condition": "EQUALS" + } + ] } - ] - } + } } ``` +Where `resources` is inside the `info` aspect of a Policy. Supported fields are as follows diff --git a/docs/cli.md b/docs/cli.md index eb8bb406b0107..267f289d9f54a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -547,7 +547,7 @@ Old Entities Migrated = {'urn:li:dataset:(urn:li:dataPlatform:hive,logging_event ### Using docker [![Docker Hub](https://img.shields.io/docker/pulls/acryldata/datahub-ingestion?style=plastic)](https://hub.docker.com/r/acryldata/datahub-ingestion) -[![datahub-ingestion docker](https://github.com/acryldata/datahub/actions/workflows/docker-ingestion.yml/badge.svg)](https://github.com/acryldata/datahub/actions/workflows/docker-ingestion.yml) +[![datahub-ingestion docker](https://github.com/acryldata/datahub/workflows/datahub-ingestion%20docker/badge.svg)](https://github.com/acryldata/datahub/actions/workflows/docker-ingestion.yml) If you don't want to install locally, you can alternatively run metadata ingestion within a Docker container. We have prebuilt images available on [Docker hub](https://hub.docker.com/r/acryldata/datahub-ingestion). All plugins will be installed and enabled automatically. diff --git a/docs/datahub_lite.md b/docs/datahub_lite.md index 3918b8cee7830..de0a20eed1d01 100644 --- a/docs/datahub_lite.md +++ b/docs/datahub_lite.md @@ -7,7 +7,6 @@ import TabItem from '@theme/TabItem'; DataHub Lite is a lightweight embeddable version of DataHub with no external dependencies. It is intended to enable local developer tooling use-cases such as simple access to metadata for scripts and other tools. DataHub Lite is compatible with the DataHub metadata format and all the ingestion connectors that DataHub supports. -It was built as a reaction to [recap](https://github.com/recap-cloud/recap) to prove that a similar lightweight system could be built within DataHub quite easily. Currently DataHub Lite uses DuckDB under the covers as its default storage layer, but that might change in the future. ## Features diff --git a/docs/domains.md b/docs/domains.md index c846a753417c5..1b2ebc9d47f39 100644 --- a/docs/domains.md +++ b/docs/domains.md @@ -22,20 +22,20 @@ You can create this privileges by creating a new [Metadata Policy](./authorizati To create a Domain, first navigate to the **Domains** tab in the top-right menu of DataHub.

- +

Once you're on the Domains page, you'll see a list of all the Domains that have been created on DataHub. Additionally, you can view the number of entities inside each Domain.

- +

To create a new Domain, click '+ New Domain'.

- +

Inside the form, you can choose a name for your Domain. Most often, this will align with your business units or groups, for example @@ -48,7 +48,7 @@ for the Domain. This option is useful if you intend to refer to Domains by a com key to be human-readable. Proceed with caution: once you select a custom id, it cannot be easily changed.

- +

By default, you don't need to worry about this. DataHub will auto-generate a unique Domain id for you. @@ -64,7 +64,7 @@ To assign an asset to a Domain, simply navigate to the asset's profile page. At see a 'Domain' section. Click 'Set Domain', and then search for the Domain you'd like to add to. When you're done, click 'Add'.

- +

To remove an asset from a Domain, click the 'x' icon on the Domain tag. @@ -149,27 +149,27 @@ source: Once you've created a Domain, you can use the search bar to find it.

- +

Clicking on the search result will take you to the Domain's profile, where you can edit its description, add / remove owners, and view the assets inside the Domain.

- +

Once you've added assets to a Domain, you can filter search results to limit to those Assets within a particular Domain using the left-side search filters.

- +

On the homepage, you'll also find a list of the most popular Domains in your organization.

- +

## Additional Resources @@ -242,7 +242,6 @@ DataHub supports Tags, Glossary Terms, & Domains as distinct types of Metadata t - **Tags**: Informal, loosely controlled labels that serve as a tool for search & discovery. Assets may have multiple tags. No formal, central management. - **Glossary Terms**: A controlled vocabulary, with optional hierarchy. Terms are typically used to standardize types of leaf-level attributes (i.e. schema fields) for governance. E.g. (EMAIL_PLAINTEXT) - **Domains**: A set of top-level categories. Usually aligned to business units / disciplines to which the assets are most relevant. Central or distributed management. Single Domain assignment per data asset. - *Need more help? Join the conversation in [Slack](http://slack.datahubproject.io)!* ### Related Features diff --git a/docs/how/add-new-aspect.md b/docs/how/add-new-aspect.md index 6ea7256ed75cc..d1fe567018903 100644 --- a/docs/how/add-new-aspect.md +++ b/docs/how/add-new-aspect.md @@ -1,20 +1,20 @@ # How to add a new metadata aspect? Adding a new metadata [aspect](../what/aspect.md) is one of the most common ways to extend an existing [entity](../what/entity.md). -We'll use the [CorpUserEditableInfo](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserEditableInfo.pdl) as an example here. +We'll use the CorpUserEditableInfo as an example here. 1. Add the aspect model to the corresponding namespace (e.g. [`com.linkedin.identity`](https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/identity)) -2. Extend the entity's aspect union to include the new aspect (e.g. [`CorpUserAspect`](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/CorpUserAspect.pdl)) +2. Extend the entity's aspect union to include the new aspect. 3. Rebuild the rest.li [IDL & snapshot](https://linkedin.github.io/rest.li/modeling/compatibility_check) by running the following command from the project root ``` ./gradlew :metadata-service:restli-servlet-impl:build -Prest.model.compatibility=ignore ``` -4. To surface the new aspect at the top-level [resource endpoint](https://linkedin.github.io/rest.li/user_guide/restli_server#writing-resources), extend the resource data model (e.g. [`CorpUser`](https://github.com/datahub-project/datahub/blob/master/gms/api/src/main/pegasus/com/linkedin/identity/CorpUser.pdl)) with an optional field (e.g. [`editableInfo`](https://github.com/datahub-project/datahub/blob/master/gms/api/src/main/pegasus/com/linkedin/identity/CorpUser.pdl#L21)). You'll also need to extend the `toValue` & `toSnapshot` methods of the top-level resource (e.g. [`CorpUsers`](https://github.com/datahub-project/datahub/blob/master/gms/impl/src/main/java/com/linkedin/metadata/resources/identity/CorpUsers.java)) to convert between the snapshot & value models. +4. To surface the new aspect at the top-level [resource endpoint](https://linkedin.github.io/rest.li/user_guide/restli_server#writing-resources), extend the resource data model with an optional field. You'll also need to extend the `toValue` & `toSnapshot` methods of the top-level resource (e.g. [`CorpUsers`](https://github.com/datahub-project/datahub/blob/master/gms/impl/src/main/java/com/linkedin/metadata/resources/identity/CorpUsers.java)) to convert between the snapshot & value models. -5. (Optional) If there's need to update the aspect via API (instead of/in addition to MCE), add a [sub-resource](https://linkedin.github.io/rest.li/user_guide/restli_server#sub-resources) endpoint for the new aspect (e.g. [`CorpUsersEditableInfoResource`](https://github.com/datahub-project/datahub/blob/master/gms/impl/src/main/java/com/linkedin/metadata/resources/identity/CorpUsersEditableInfoResource.java)). The sub-resource endpiont also allows you to retrieve previous versions of the aspect as well as additional metadata such as the audit stamp. +5. (Optional) If there's need to update the aspect via API (instead of/in addition to MCE), add a [sub-resource](https://linkedin.github.io/rest.li/user_guide/restli_server#sub-resources) endpoint for the new aspect (e.g. `CorpUsersEditableInfoResource`). The sub-resource endpiont also allows you to retrieve previous versions of the aspect as well as additional metadata such as the audit stamp. -6. After rebuilding & restarting [gms](https://github.com/datahub-project/datahub/tree/master/gms), [mce-consumer-job](https://github.com/datahub-project/datahub/tree/master/metadata-jobs/mce-consumer-job) & [mae-consumer-job](https://github.com/datahub-project/datahub/tree/master/metadata-jobs/mae-consumer-job), +6. After rebuilding & restarting gms, [mce-consumer-job](https://github.com/datahub-project/datahub/tree/master/metadata-jobs/mce-consumer-job) & [mae-consumer-job](https://github.com/datahub-project/datahub/tree/master/metadata-jobs/mae-consumer-job),z you should be able to start emitting [MCE](../what/mxe.md) with the new aspect and have it automatically ingested & stored in DB. diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 7ba516c82cf1b..9b19291ee246a 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -5,19 +5,42 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ## Next ### Breaking Changes +- #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now. ### Potential Downtime +### Deprecations + +### Other Notable Changes + +## 0.11.0 + +### Breaking Changes + +### Potential Downtime +- #8611 Search improvements requires reindexing indices. A `system-update` job will run which will set indices to read-only and create a backup/clone of each index. During the reindexing new components will be prevented from start-up until the reindex completes. The logs of this job will indicate a % complete per index. Depending on index sizes and infrastructure this process can take 5 minutes to hours however as a rough estimate 1 hour for every 2.3 million entities. + ### Deprecations - #8525: In LDAP ingestor, the `manager_pagination_enabled` changed to general `pagination_enabled` +- MAE Events are no longer produced. MAE events have been deprecated for over a year. ### Other Notable Changes +- In this release we now enable you to create and delete pinned announcements on your DataHub homepage! If you have the “Manage Home Page Posts” platform privilege you’ll see a new section in settings called “Home Page Posts” where you can create and delete text posts and link posts that your users see on the home page. +- The new search and browse experience, which was first made available in the previous release behind a feature flag, is now on by default. Check out our release notes for v0.10.5 to get more information and documentation on this new Browse experience. +- In addition to the ranking changes mentioned above, this release includes changes to the highlighting of search entities to understand why they match your query. You can also sort your results alphabetically or by last updated times, in addition to relevance. In this release, we suggest a correction if your query has a typo in it. - #8300: Clickhouse source now inherited from TwoTierSQLAlchemy. In old way we have platform_instance -> container -> co container db (None) -> container schema and now we have platform_instance -> container database. - #8300: Added `uri_opts` argument; now we can add any options for clickhouse client. - #8659: BigQuery ingestion no longer creates DataPlatformInstance aspects by default. This will only affect users that were depending on this aspect for custom functionality, and can be enabled via the `include_data_platform_instance` config option. +- OpenAPI entity and aspect endpoints expanded to improve developer experience when using this API with additional aspects to be added in the near future. +- The CLI now supports recursive deletes. +- Batching of default aspects on initial ingestion (SQL) +- Improvements to multi-threading. Ingestion recipes, if previously reduced to 1 thread, can be restored to the 15 thread default. +- Gradle 7 upgrade moderately improves build speed +- DataHub Ingestion slim images reduced in size by 2GB+ +- Glue Schema Registry fixed ## 0.10.5 diff --git a/docs/managed-datahub/observe/custom-sql-assertions.md b/docs/managed-datahub/observe/custom-sql-assertions.md new file mode 100644 index 0000000000000..d4a09b434ca79 --- /dev/null +++ b/docs/managed-datahub/observe/custom-sql-assertions.md @@ -0,0 +1,315 @@ +--- +description: This page provides an overview of working with DataHub SQL Assertions +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + + +# Custom SQL Assertions + + + + +> ⚠️ The **Custom SQL Assertions** feature is currently in private beta, part of the **Acryl Observe** module, and may only be available to a +> limited set of design partners. +> +> If you are interested in trying it and providing feedback, please reach out to your Acryl Customer Success +> representative. + +## Introduction + +Can you remember a time when the meaning of Data Warehouse Table that you depended on fundamentally changed, with little or no notice? +If the answer is yes, how did you find out? We'll take a guess - someone looking at an internal reporting dashboard or worse, a user using your your product, sounded an alarm when +a number looked a bit out of the ordinary. Perhaps your table initially tracked purchases made on your company's e-commerce web store, but suddenly began to include purchases made +through your company's new mobile app. + +There are many reasons why an important Table on Snowflake, Redshift, or BigQuery may change in its meaning - application code bugs, new feature rollouts, +changes to key metric definitions, etc. Often times, these changes break important assumptions made about the data used in building key downstream data products +like reporting dashboards or data-driven product features. + +What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data +issues _before_ anyone else? With Acryl DataHub **Custom SQL Assertions**, you can. + +Acryl DataHub allows users to define complex expectations about a particular warehouse Table through custom SQL queries, and then monitor those expectations over time as the table grows and changes. + +In this article, we'll cover the basics of monitoring Custom SQL Assertions - what they are, how to configure them, and more - so that you and your team can +start building trust in your most important data assets. + +Let's get started! + +## Support + +Custom SQL Assertions are currently supported for: + +1. Snowflake +2. Redshift +3. BigQuery + +Note that an Ingestion Source _must_ be configured with the data platform of your choice in Acryl DataHub's **Ingestion** +tab. + +> Note that SQL Assertions are not yet supported if you are connecting to your warehouse +> using the DataHub CLI or a Remote Ingestion Executor. + +## What is a Custom SQL Assertion? + +A **Custom SQL Assertion** is a highly configurable Data Quality rule used to monitor a Data Warehouse Table +for unexpected or sudden changes in its meaning. Custom SQL Assertions are defined through a raw SQL query that is evaluated against +the Table. You have full control over the SQL query, and can use any SQL features supported by your Data Warehouse. +Custom SQL Assertions can be particularly useful when you have complex tables or relationships +that are used to generate important metrics or reports, and where the meaning of the table is expected to be stable over time. +If you have existing SQL queries that you already use to monitor your data, you may find that Custom SQL Assertions are an easy way to port them +to Acryl DataHub to get started. + +For example, imagine that you have a Table that tracks the number of purchases made on your company's e-commerce web store. +You have a SQL query that you use to calculate the number of purchases made in the past 24 hours, and you'd like to monitor this +metric over time to ensure that it is always greater than 1000. You can use a Custom SQL Assertion to do this! + + +### Anatomy of a Custom SQL Assertion + +At the most basic level, **Custom SQL Assertions** consist of a few important parts: + +1. An **Evaluation Schedule** +2. A **Query** +3. An **Condition Type** +4. An **Assertion Description** + +In this section, we'll give an overview of each. + +#### 1. Evaluation Schedule + +The **Evaluation Schedule**: This defines how often to query the given warehouse Table. This should usually +be configured to match the expected change frequency of the Table, although it can also be less frequently depending +on the requirements. You can also specify specific days of the week, hours in the day, or even +minutes in an hour. + + +#### 2. Query + +The **Query**: This is the SQL query that will be used to evaluate the Table. The query should return a **single row** containing a **single numeric column** (integers, floats). +The query can be as simple or as complex as you'd like, and can use any SQL features supported by your Data Warehouse. This requires that the configured user account has read access to the asset. Make sure to use the fully qualified name of the Table in your query. + +Use the "Try it out" button to test your query and ensure that it returns a single row with a single column. The query will be run against the Table in the context of the configured user account, so ensure that the user has read access to the Table. + + +#### 3. Condition Type + +The **Condition Type**: This defines the conditions under which the Assertion will **fail**. The list of supported operations is: +- **Is Equal To**: The assertion will fail if the query result is equal to the configured value +- **Is Not Equal To**: The assertion will fail if the query result is not equal to the configured value +- **Is Greater Than**: The assertion will fail if the query result is greater than the configured value +- **Is Less Than**: The assertion will fail if the query result is less than the configured value +- **Is Outside a Range**: The assertion will fail if the query result is outside the configured range +- **Grows More Than**: The assertion will fail if the query result grows more than the configured range. This can be either a percentage (**Percentage**) or a number (**Value**). +- **Grows Less Than**: The assertion will fail if the query result grows less than the configured percentage. This can be either a percentage (**Percentage**) or a number (**Value**). +- **Growth is outside a range**: The assertion will fail if the query result growth is outside the configured range. This can be either a percentage (**Percentage**) or a number (**Value**). + +Custom SQL Assertions also have an off switch: they can be started or stopped at any time with the click of button. + +#### 4. Assertion Description + +The **Assertion Description**: This is a human-readable description of the Assertion. It should be used to describe the meaning of the Assertion, and can be used to provide additional context to users who are viewing the Assertion. + + +## Creating a Custom SQL Assertion + +### Prerequisites + +1. **Permissions**: To create or delete Custom SQL Assertions for a specific entity on DataHub, you'll need to be granted the + `Edit Assertions` and `Edit Monitors` privileges for the entity. This is granted to Entity owners by default. + +2. **Data Platform Connection**: In order to create a Custom SQL Assertion, you'll need to have an **Ingestion Source** configured to your + Data Platform: Snowflake, BigQuery, or Redshift under the **Integrations** tab. + +Once these are in place, you're ready to create your Custom SQL Assertions! + +### Steps + +1. Navigate to the Table you want to monitor +2. Click the **Validations** tab + +

+ +

+ +3. Click **+ Create Assertion** + +

+ +

+ +4. Choose **Custom** + +5. Configure the evaluation **schedule**. This is the frequency at which the assertion will be evaluated to produce a pass or fail result, and the times + when the query will be executed. + +6. Provide a SQL **query** that will be used to evaluate the Table. The query should return a single row with a single column. Currently only numeric values are supported (integer and floats). The query can be as simple or as complex as you'd like, and can use any SQL features supported by your Data Warehouse. Make sure to use the fully qualified name of the Table in your query. + +

+ +

+ +7. Configure the evaluation **condition type**. This determines the cases in which the new assertion will fail when it is evaluated. + +

+ +

+ +8. Add a **description** for the assertion. This is a human-readable description of the Assertion. It should be used to describe the meaning of the Assertion, and can be used to provide additional context to users who are viewing the Assertion. + +

+ +

+ +9. (Optional) Use the **Try it out** button to test your query and ensure that it returns a single row with a single column, and passes the configured condition type. + +

+ +

+ +10. Click **Next** +11. Configure actions that should be taken when the Custom SQL Assertion passes or fails + +

+ +

+ +- **Raise incident**: Automatically raise a new DataHub Incident for the Table whenever the Custom SQL Assertion is failing. This + may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when + an incident is created due to an Assertion failure. +- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Custom SQL Assertion. Note that + any other incidents will not be impacted. + +1. Click **Save**. + +And that's it! DataHub will now begin to monitor your Custom SQL Assertion for the table. + +To view the time of the next Custom SQL Assertion evaluation, simply click **Custom** and then click on your +new Assertion: + +

+ +

+ +Once your assertion has run, you will begin to see Success or Failure status for the Table + +

+ +

+ + +## Stopping a Custom SQL Assertion + +In order to temporarily stop the evaluation of a Custom SQL Assertion: + +1. Navigate to the **Validations** tab of the Table with the assertion +2. Click **Custom** to open the Custom SQL Assertions list +3. Click the three-dot menu on the right side of the assertion you want to disable +4. Click **Stop** + +

+ +

+ +To resume the Custom SQL Assertion, simply click **Turn On**. + +

+ +

+ + +## Creating Custom SQL Assertions via API + +Under the hood, Acryl DataHub implements Custom SQL Assertion Monitoring using two "entity" concepts: + +- **Assertion**: The specific expectation for the custom assertion, e.g. "The table was changed in the past 7 hours" + or "The table is changed on a schedule of every day by 8am". This is the "what". + +- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific + mechanisms. This is the "how". + +Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the +`Edit Assertions` and `Edit Monitors` privileges for it. + +#### GraphQL + +In order to create a Custom SQL Assertion that is being monitored on a specific **Evaluation Schedule**, you'll need to use 2 +GraphQL mutation queries to create a Custom SQL Assertion entity and create an Assertion Monitor entity responsible for evaluating it. + +Start by creating the Custom SQL Assertion entity using the `createSqlAssertion` query and hang on to the 'urn' field of the Assertion entity +you get back. Then continue by creating a Monitor entity using the `createAssertionMonitor`. + +##### Examples + +To create a Custom SQL Assertion Entity that checks whether a query result is greater than 100: + +```json +mutation createSqlAssertion { + createSqlAssertion( + input: { + entityUrn: "", + type: METRIC, + description: "", + statement: "", + operator: GREATER_THAN, + parameters: { + value: { + value: "100", + type: NUMBER + } + } + } + ) { + urn + } +} +``` + +The supported assertion types are `METRIC` and `METRIC_CHANGE`. If you choose `METRIC_CHANGE`, +you will need to provide a `changeType` parameter with either `ABSOLUTE` or `PERCENTAGE` values. +The supported operator types are `EQUAL_TO`, `NOT_EQUAL_TO`, `GREATER_THAN`, `GREATER_THAN_OR_EQUAL_TO`, `LESS_THAN`, `LESS_THAN_OR_EQUAL_TO`, and `BETWEEN` (requires minValue, maxValue). +The supported parameter types are `NUMBER`. + +To create an Assertion Monitor Entity that evaluates the custom assertion every 8 hours: + +```json +mutation createAssertionMonitor { + createAssertionMonitor( + input: { + entityUrn: "", + assertionUrn: "", + schedule: { + cron: "0 */8 * * *", + timezone: "America/Los_Angeles" + }, + parameters: { + type: DATASET_SQL + } + } + ) { + urn + } +} +``` + +This entity defines _when_ to run the check (Using CRON format - every 8th hour) and _how_ to run the check (using the Information Schema). + +After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone. + +You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. + +### Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the Acryl GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: diff --git a/docs/managed-datahub/observe/freshness-assertions.md b/docs/managed-datahub/observe/freshness-assertions.md index c5d4ca9081b43..82de423f6f2de 100644 --- a/docs/managed-datahub/observe/freshness-assertions.md +++ b/docs/managed-datahub/observe/freshness-assertions.md @@ -125,7 +125,7 @@ Change Source types vary by the platform, but generally fall into these categori - **DataHub Operation**: A DataHub "Operation" aspect contains timeseries information used to describe changes made to an entity. Using this option avoids contacting your data platform, and instead uses the DataHub Operation metadata to evaluate Freshness Assertions. This relies on Operations being reported to DataHub, either via ingestion or via use of the DataHub APIs (see [Report Operation via API](#reporting-operations-via-api)). - Note if you have not configured an ingestion source through DataHub, then this may be the only option available. + Note if you have not configured an ingestion source through DataHub, then this may be the only option available. By default, any operation type found will be considered a valid change. Use the **Operation Types** dropdown when selecting this option to specify which operation types should be considered valid changes. You may choose from one of DataHub's standard Operation Types, or specify a "Custom" Operation Type by typing in the name of the Operation Type. Using either of the column value approaches (**Last Modified Column** or **High Watermark Column**) to determine whether a Table has changed can be useful because it can be customized to determine whether specific types of important changes have been made to a given Table. Because it does not involve system warehouse tables, it is also easily portable across Data Warehouse and Data Lake providers. diff --git a/docs/managed-datahub/release-notes/v_0_2_11.md b/docs/managed-datahub/release-notes/v_0_2_11.md new file mode 100644 index 0000000000000..1f42090848712 --- /dev/null +++ b/docs/managed-datahub/release-notes/v_0_2_11.md @@ -0,0 +1,73 @@ +# v0.2.11 +--- + +Release Availability Date +--- +14-Sep-2023 + +Recommended CLI/SDK +--- +- `v0.11.0` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.10.5.5 +- [Deprecation] In LDAP ingestor, the manager_pagination_enabled changed to general pagination_enabled + +If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better. + +Special Notes +--- +- Deployment process for this release is going to have a downtime when systme will be in a read only mode. A rough estimate 1 hour for every 2.3 million entities (includes soft-deleted entities). + + +## Release Changelog +--- +- Since `v0.2.10` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/2b0952195b7895df0a2bf92b28e71aac18217781...75252a3d9f6a576904be5a0790d644b9ae2df6ac have been pulled in. +- Misc fixes & features + - Proposals + - Group names shown correctly for proposal Inbox + - Metadata tests + - Deprecate/Un-deprecate actions available in Metadata tests + - Last Observed (in underlying sql) available as a filter in metadata tests + - [Breaking change] Renamed `__lastUpdated` -> `__created` as a filter to correctly represent what it was. This was not surfaced in the UI. But if you were using it then this needs to be renamed. Acryl Customer Success team will keep an eye out to pro-actively find and bring this up if you are affected by this. + - Robustness improvements to metadata test runs + - Copy urn for metadata tests to allow for easier filtering for iteration over metadata test results via our APIs. + - A lot more fixes to subscriptions, notifications and Observability (Beta). + - Some performance improvements to lineage queries + +## Some notable features in this SaaS release +- We now enable you to create and delete pinned announcements on your DataHub homepage! If you have the “Manage Home Page Posts” platform privilege you’ll see a new section in settings called “Home Page Posts” where you can create and delete text posts and link posts that your users see on the home page. +- Improvements to search experience +
+