Merge branch 'master' into feature/cus-3571-fix-model-DashboardInfo-DashboardContainsDashboard

Author: sgomezvillamor

.github/ISSUE_TEMPLATE/--bug-report.md

@@ -29,4 +29,4 @@ If applicable, add screenshots to help explain your problem.
  - Version [e.g. 22]
 
 **Additional context**
-Add any other context about the problem here.
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/datahub-v1-0-rc-bug-report.md

@@ -0,0 +1,32 @@
+---
+name: DataHub v1.0-rc Bug Report
+about: Report issues found in DataHub v1.0 Release Candidates
+title: "[v1.0-rc/bug] Description of Bug"
+labels: bug, datahub-v1.0-rc
+assignees: chriscollins3456, david-leifker, maggiehays
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots and/or Screen Recordings**
+If applicable, add screenshots and/or screen recordings to help explain the issue.
+
+**System details (please complete the following information):**
+ - DataHub Version Tag [e.g. v1.0-rc1]
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/docker-unified.yml

@@ -1039,12 +1039,12 @@ jobs:
           cypress_matrix=$(printf "{\"test_strategy\":\"cypress\",\"batch\":\"0\",\"batch_count\":\"$cypress_batch_count\"}"; for ((i=1;i<cypress_batch_count;i++)); do printf ",{\"test_strategy\":\"cypress\", \"batch_count\":\"$cypress_batch_count\",\"batch\":\"%d\"}" $i; done)
           
           includes=''
-          if [[  "${{ needs.setup.outputs.frontend_only }}" == 'true' ]]; then
-            includes=$cypress_matrix
-          elif [ "${{ needs.setup.outputs.ingestion_only }}" == 'true' ]; then
-            includes=$python_matrix
-          elif [[ "${{ needs.setup.outputs.backend_change }}" == 'true' || "${{ needs.setup.outputs.smoke_test_change }}" == 'true' ]]; then
+          if [[ "${{ needs.setup.outputs.backend_change }}" == 'true' || "${{ needs.setup.outputs.smoke_test_change }}" == 'true' || "${{ needs.setup.outputs.publish }}" == 'true' ]]; then
             includes="$python_matrix,$cypress_matrix"
+          elif [[  "${{ needs.setup.outputs.frontend_only }}" == 'true' ]]; then
+            includes="$cypress_matrix"
+          elif [[ "${{ needs.setup.outputs.ingestion_only }}" == 'true' ]]; then
+            includes="$python_matrix"
           fi
           echo "matrix={\"include\":[$includes] }" >> "$GITHUB_OUTPUT"
 

.github/workflows/metadata-model.yml

@@ -60,4 +60,5 @@ jobs:
         env:
           DATAHUB_SERVER: ${{ secrets.DataHubServer }}
           DATAHUB_TOKEN: ${{ secrets.DataHubToken }}
+          DATAHUB_TELEMETRY_ENABLED: false
         run: ./gradlew :metadata-ingestion:modelDocUpload

build.gradle

@@ -38,7 +38,7 @@ buildscript {
   ext.springVersion = '6.1.14'
   ext.springBootVersion = '3.2.9'
   ext.springKafkaVersion = '3.1.6'
-  ext.openTelemetryVersion = '1.18.0'
+  ext.openTelemetryVersion = '1.45.0'
   ext.neo4jVersion = '5.20.0'
   ext.neo4jTestVersion = '5.20.0'
   ext.neo4jApocVersion = '5.20.0'
@@ -220,7 +220,10 @@ project.ext.externalDependency = [
     'neo4jApocCore': 'org.neo4j.procedure:apoc-core:' + neo4jApocVersion,
     'neo4jApocCommon': 'org.neo4j.procedure:apoc-common:' + neo4jApocVersion,
     'opentelemetryApi': 'io.opentelemetry:opentelemetry-api:' + openTelemetryVersion,
-    'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
+    'opentelemetrySdk': 'io.opentelemetry:opentelemetry-sdk:' + openTelemetryVersion,
+    'opentelemetrySdkTrace': 'io.opentelemetry:opentelemetry-sdk-trace:' + openTelemetryVersion,
+    'opentelemetryAutoConfig': 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:' + openTelemetryVersion,
+    'opentelemetryAnnotations': 'io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations:2.11.0',
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
     'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java

@@ -29,6 +29,7 @@ private Constants() {}
   public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql";
   public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql";
   public static final String VERSION_SCHEMA_FILE = "versioning.graphql";
+  public static final String QUERY_SCHEMA_FILE = "query.graphql";
   public static final String BROWSE_PATH_DELIMITER = "/";
   public static final String BROWSE_PATH_V2_DELIMITER = "␟";
   public static final String VERSION_STAMP_FIELD_NAME = "versionStamp";

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -257,6 +257,7 @@
 import com.linkedin.datahub.graphql.resolvers.mutate.RemoveTagResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.RemoveTermResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateDescriptionResolver;
+import com.linkedin.datahub.graphql.resolvers.mutate.UpdateDisplayPropertiesResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver;
 import com.linkedin.datahub.graphql.resolvers.mutate.UpdateUserSettingResolver;
@@ -297,6 +298,7 @@
 import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver;
 import com.linkedin.datahub.graphql.resolvers.settings.view.GlobalViewsSettingsResolver;
 import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver;
+import com.linkedin.datahub.graphql.resolvers.siblings.SiblingsSearchResolver;
 import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver;
 import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver;
 import com.linkedin.datahub.graphql.resolvers.structuredproperties.CreateStructuredPropertyResolver;
@@ -314,6 +316,7 @@
 import com.linkedin.datahub.graphql.resolvers.test.UpdateTestResolver;
 import com.linkedin.datahub.graphql.resolvers.timeline.GetSchemaBlameResolver;
 import com.linkedin.datahub.graphql.resolvers.timeline.GetSchemaVersionListResolver;
+import com.linkedin.datahub.graphql.resolvers.timeline.GetTimelineResolver;
 import com.linkedin.datahub.graphql.resolvers.type.AspectInterfaceTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.EntityInterfaceTypeResolver;
 import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver;
@@ -376,6 +379,7 @@
 import com.linkedin.datahub.graphql.types.notebook.NotebookType;
 import com.linkedin.datahub.graphql.types.ownership.OwnershipType;
 import com.linkedin.datahub.graphql.types.policy.DataHubPolicyType;
+import com.linkedin.datahub.graphql.types.post.PostType;
 import com.linkedin.datahub.graphql.types.query.QueryType;
 import com.linkedin.datahub.graphql.types.restricted.RestrictedType;
 import com.linkedin.datahub.graphql.types.role.DataHubRoleType;
@@ -389,6 +393,7 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.metadata.client.UsageStatsJavaClient;
+import com.linkedin.metadata.config.ChromeExtensionConfiguration;
 import com.linkedin.metadata.config.DataHubConfiguration;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
@@ -496,6 +501,7 @@ public class GmsGraphQLEngine {
   private final TestsConfiguration testsConfiguration;
   private final DataHubConfiguration datahubConfiguration;
   private final ViewsConfiguration viewsConfiguration;
+  private final ChromeExtensionConfiguration chromeExtensionConfiguration;
 
   private final DatasetType datasetType;
 
@@ -540,6 +546,7 @@ public class GmsGraphQLEngine {
   private final FormType formType;
   private final IncidentType incidentType;
   private final RestrictedType restrictedType;
+  private final PostType postType;
   private final DataProcessInstanceType dataProcessInstanceType;
   private final VersionSetType versionSetType;
 
@@ -620,6 +627,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.datahubConfiguration = args.datahubConfiguration;
     this.viewsConfiguration = args.viewsConfiguration;
     this.featureFlags = args.featureFlags;
+    this.chromeExtensionConfiguration = args.chromeExtensionConfiguration;
 
     this.datasetType = new DatasetType(entityClient);
     this.roleType = new RoleType(entityClient);
@@ -662,6 +670,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.formType = new FormType(entityClient);
     this.incidentType = new IncidentType(entityClient);
     this.restrictedType = new RestrictedType(entityClient, restrictedService);
+    this.postType = new PostType(entityClient);
     this.dataProcessInstanceType = new DataProcessInstanceType(entityClient, featureFlags);
     this.versionSetType = new VersionSetType(entityClient);
 
@@ -713,6 +722,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
                 entityTypeType,
                 formType,
                 incidentType,
+                postType,
                 versionSetType,
                 restrictedType,
                 businessAttributeType,
@@ -873,7 +883,8 @@ public GraphQLEngine.Builder builder() {
         .addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE))
         .addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE))
         .addSchema(fileBasedSchema(COMMON_SCHEMA_FILE))
-        .addSchema(fileBasedSchema(VERSION_SCHEMA_FILE));
+        .addSchema(fileBasedSchema(VERSION_SCHEMA_FILE))
+        .addSchema(fileBasedSchema(QUERY_SCHEMA_FILE));
 
     for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
       List<String> pluginSchemaFiles = plugin.getSchemaFiles();
@@ -1002,7 +1013,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
                         this.testsConfiguration,
                         this.datahubConfiguration,
                         this.viewsConfiguration,
-                        this.featureFlags))
+                        this.featureFlags,
+                        this.chromeExtensionConfiguration))
                 .dataFetcher("me", new MeResolver(this.entityClient, featureFlags))
                 .dataFetcher("search", new SearchResolver(this.entityClient))
                 .dataFetcher(
@@ -1088,6 +1100,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher(
                     "executionRequest", new GetIngestionExecutionRequestResolver(this.entityClient))
                 .dataFetcher("getSchemaBlame", new GetSchemaBlameResolver(this.timelineService))
+                .dataFetcher("getTimeline", new GetTimelineResolver(this.timelineService))
                 .dataFetcher(
                     "getSchemaVersionList", new GetSchemaVersionListResolver(this.timelineService))
                 .dataFetcher("test", getResolver(testType))
@@ -1352,6 +1365,9 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
                   "createOwnershipType", new CreateOwnershipTypeResolver(this.ownershipTypeService))
               .dataFetcher(
                   "updateOwnershipType", new UpdateOwnershipTypeResolver(this.ownershipTypeService))
+              .dataFetcher(
+                  "updateDisplayProperties",
+                  new UpdateDisplayPropertiesResolver(this.entityService))
               .dataFetcher(
                   "deleteOwnershipType", new DeleteOwnershipTypeResolver(this.ownershipTypeService))
               .dataFetcher("submitFormPrompt", new SubmitFormPromptResolver(this.formService))
@@ -1735,7 +1751,10 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
                     .dataFetcher("exists", new EntityExistsResolver(entityService))
                     .dataFetcher("runs", new EntityRunsResolver(entityClient))
                     .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
-                    .dataFetcher("parentContainers", new ParentContainersResolver(entityClient)))
+                    .dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
+                    .dataFetcher(
+                        "siblingsSearch",
+                        new SiblingsSearchResolver(this.entityClient, this.viewService)))
         .type(
             "Owner",
             typeWiring ->
@@ -1768,6 +1787,13 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
                         datasetType,
                         (env) ->
                             ((ForeignKeyConstraint) env.getSource()).getForeignDataset().getUrn())))
+        .type(
+            "Deprecation",
+            typeWiring ->
+                typeWiring.dataFetcher(
+                    "replacement",
+                    new EntityTypeResolver(
+                        entityTypes, (env) -> ((Deprecation) env.getSource()).getReplacement())))
         .type(
             "SiblingProperties",
             typeWiring ->
@@ -1878,10 +1904,16 @@ private void configureSchemaFieldResolvers(final RuntimeWiring.Builder builder)
     builder.type(
         "SchemaFieldEntity",
         typeWiring ->
-            typeWiring.dataFetcher(
-                "parent",
-                new EntityTypeResolver(
-                    entityTypes, (env) -> ((SchemaFieldEntity) env.getSource()).getParent())));
+            typeWiring
+                .dataFetcher(
+                    "parent",
+                    new EntityTypeResolver(
+                        entityTypes, (env) -> ((SchemaFieldEntity) env.getSource()).getParent()))
+                .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
+                .dataFetcher(
+                    "lineage",
+                    new EntityLineageResultResolver(
+                        siblingGraphService, restrictedService, this.authorizationConfiguration)));
   }
 
   private void configureEntityPathResolvers(final RuntimeWiring.Builder builder) {
@@ -1935,10 +1967,12 @@ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) {
                 new LoadableTypeBatchResolver<>(
                     dataPlatformType,
                     (env) ->
-                        ((CorpUserEditableProperties) env.getSource())
-                            .getPlatforms().stream()
-                                .map(DataPlatform::getUrn)
-                                .collect(Collectors.toList()))));
+                        ((CorpUserEditableProperties) env.getSource()).getPlatforms() != null
+                            ? ((CorpUserEditableProperties) env.getSource())
+                                .getPlatforms().stream()
+                                    .map(DataPlatform::getUrn)
+                                    .collect(Collectors.toList())
+                            : ImmutableList.of())));
   }
 
   /**
@@ -3065,11 +3099,20 @@ private void configureQueryEntityResolvers(final RuntimeWiring.Builder builder)
         .type(
             "QuerySubject",
             typeWiring ->
-                typeWiring.dataFetcher(
-                    "dataset",
-                    new LoadableTypeResolver<>(
-                        datasetType,
-                        (env) -> ((QuerySubject) env.getSource()).getDataset().getUrn())));
+                typeWiring
+                    .dataFetcher(
+                        "dataset",
+                        new LoadableTypeResolver<>(
+                            datasetType,
+                            (env) -> ((QuerySubject) env.getSource()).getDataset().getUrn()))
+                    .dataFetcher(
+                        "schemaField",
+                        new LoadableTypeResolver<>(
+                            schemaFieldType,
+                            (env) ->
+                                ((QuerySubject) env.getSource()).getSchemaField() != null
+                                    ? ((QuerySubject) env.getSource()).getSchemaField().getUrn()
+                                    : null)));
   }
 
   private void configureOwnershipTypeResolver(final RuntimeWiring.Builder builder) {
@@ -3098,13 +3141,26 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b
         "DataProcessInstance",
         typeWiring ->
             typeWiring
+                .dataFetcher("exists", new EntityExistsResolver(entityService))
+                .dataFetcher(
+                    "platform",
+                    new LoadableTypeResolver<>(
+                        dataPlatformType,
+                        (env) -> {
+                          final DataProcessInstance dataProcessInstance = env.getSource();
+                          return dataProcessInstance != null
+                                  && dataProcessInstance.getPlatform() != null
+                              ? dataProcessInstance.getPlatform().getUrn()
+                              : null;
+                        }))
                 .dataFetcher(
                     "dataPlatformInstance",
                     new LoadableTypeResolver<>(
                         dataPlatformInstanceType,
                         (env) -> {
                           final DataProcessInstance dataProcessInstance = env.getSource();
-                          return dataProcessInstance.getDataPlatformInstance() != null
+                          return dataProcessInstance != null
+                                  && dataProcessInstance.getDataPlatformInstance() != null
                               ? dataProcessInstance.getDataPlatformInstance().getUrn()
                               : null;
                         }))
@@ -3117,6 +3173,11 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b
                           final DataProcessInstance dpi = env.getSource();
                           return dpi.getContainer() != null ? dpi.getContainer().getUrn() : null;
                         }))
+                .dataFetcher(
+                    "parentTemplate",
+                    new EntityTypeResolver(
+                        entityTypes,
+                        (env) -> ((DataProcessInstance) env.getSource()).getParentTemplate()))
                 .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))
                 .dataFetcher(
                     "lineage",

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java

@@ -13,6 +13,7 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.entity.client.SystemEntityClient;
 import com.linkedin.metadata.client.UsageStatsJavaClient;
+import com.linkedin.metadata.config.ChromeExtensionConfiguration;
 import com.linkedin.metadata.config.DataHubConfiguration;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.config.TestsConfiguration;
@@ -87,6 +88,7 @@ public class GmsGraphQLEngineArgs {
   int graphQLQueryDepthLimit;
   boolean graphQLQueryIntrospectionEnabled;
   BusinessAttributeService businessAttributeService;
+  ChromeExtensionConfiguration chromeExtensionConfiguration;
   ConnectionService connectionService;
   AssertionService assertionService;
   EntityVersioningService entityVersioningService;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -375,5 +375,21 @@ public static boolean isViewDatasetUsageAuthorized(
         new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString()));
   }
 
+  public static boolean isViewDatasetProfileAuthorized(
+      final QueryContext context, final Urn resourceUrn) {
+    return AuthUtil.isAuthorized(
+        context.getOperationContext(),
+        PoliciesConfig.VIEW_DATASET_PROFILE_PRIVILEGE,
+        new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString()));
+  }
+
+  public static boolean isViewDatasetOperationsAuthorized(
+      final QueryContext context, final Urn resourceUrn) {
+    return AuthUtil.isAuthorized(
+        context.getOperationContext(),
+        PoliciesConfig.VIEW_DATASET_OPERATIONS_PRIVILEGE,
+        new EntitySpec(resourceUrn.getEntityType(), resourceUrn.toString()));
+  }
+
   private AuthorizationUtils() {}
 }