Merge branch 'master' into Qlik-Connector-Integration

Author: shubhamjagtap639

.github/workflows/docker-unified.yml

@@ -3,17 +3,9 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - "docs-website/**"
-      - "docs/**"
-      - "**.md"
   pull_request:
     branches:
       - "**"
-    paths-ignore:
-      - "docs-website/**"
-      - "docs/**"
-      - "**.md"
   release:
     types: [published]
 

build.gradle

@@ -73,7 +73,7 @@ plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.1'
   id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
   id 'com.palantir.docker' version '0.35.0' apply false
-  id 'com.avast.gradle.docker-compose' version '0.17.5'
+  id 'com.avast.gradle.docker-compose' version '0.17.6'
   id "com.diffplug.spotless" version "6.23.3"
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -141,6 +141,7 @@
 import com.linkedin.datahub.graphql.resolvers.entity.EntityExistsResolver;
 import com.linkedin.datahub.graphql.resolvers.entity.EntityPrivilegesResolver;
 import com.linkedin.datahub.graphql.resolvers.form.BatchAssignFormResolver;
+import com.linkedin.datahub.graphql.resolvers.form.BatchRemoveFormResolver;
 import com.linkedin.datahub.graphql.resolvers.form.CreateDynamicFormAssignmentResolver;
 import com.linkedin.datahub.graphql.resolvers.form.IsFormAssignedToMeResolver;
 import com.linkedin.datahub.graphql.resolvers.form.SubmitFormPromptResolver;
@@ -1214,6 +1215,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
                     new CreateDynamicFormAssignmentResolver(this.formService))
                 .dataFetcher(
                     "verifyForm", new VerifyFormResolver(this.formService, this.groupService))
+                .dataFetcher("batchRemoveForm", new BatchRemoveFormResolver(this.formService))
                 .dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient))
                 .dataFetcher(
                     "updateIncidentStatus",

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/BatchRemoveFormResolver.java

@@ -0,0 +1,54 @@
+package com.linkedin.datahub.graphql.resolvers.form;
+
+import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
+
+import com.datahub.authentication.Authentication;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.BatchAssignFormInput;
+import com.linkedin.metadata.service.FormService;
+import graphql.schema.DataFetcher;
+import graphql.schema.DataFetchingEnvironment;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+
+public class BatchRemoveFormResolver implements DataFetcher<CompletableFuture<Boolean>> {
+
+  private final FormService _formService;
+
+  public BatchRemoveFormResolver(@Nonnull final FormService formService) {
+    _formService = Objects.requireNonNull(formService, "formService must not be null");
+  }
+
+  @Override
+  public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
+      throws Exception {
+    final QueryContext context = environment.getContext();
+
+    final BatchAssignFormInput input =
+        bindArgument(environment.getArgument("input"), BatchAssignFormInput.class);
+    final Urn formUrn = UrnUtils.getUrn(input.getFormUrn());
+    final List<String> entityUrns = input.getEntityUrns();
+    final Authentication authentication = context.getAuthentication();
+
+    // TODO: (PRD-1062) Add permission check once permission exists
+
+    return CompletableFuture.supplyAsync(
+        () -> {
+          try {
+            _formService.batchUnassignFormForEntities(
+                entityUrns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()),
+                formUrn,
+                authentication);
+            return true;
+          } catch (Exception e) {
+            throw new RuntimeException(
+                String.format("Failed to perform update against input %s", input), e);
+          }
+        });
+  }
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java

@@ -6,6 +6,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.generated.AndFilterInput;
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 import com.linkedin.datahub.graphql.generated.LineageDirection;
@@ -92,9 +93,14 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
 
     final int start = input.getStart() != null ? input.getStart() : DEFAULT_START;
     final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT;
-    final List<FacetFilterInput> filters =
-        input.getFilters() != null ? input.getFilters() : new ArrayList<>();
-    final Integer maxHops = getMaxHops(filters);
+    final List<AndFilterInput> filters =
+        input.getOrFilters() != null ? input.getOrFilters() : new ArrayList<>();
+    final List<FacetFilterInput> facetFilters =
+        filters.stream()
+            .map(AndFilterInput::getAnd)
+            .flatMap(List::stream)
+            .collect(Collectors.toList());
+    final Integer maxHops = getMaxHops(facetFilters);
 
     @Nullable
     final Long startTimeMillis =
@@ -117,7 +123,8 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
                 start,
                 count);
 
-            final Filter filter = ResolverUtils.buildFilter(filters, input.getOrFilters());
+            final Filter filter =
+                ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters());
             SearchFlags searchFlags = null;
             com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags();
             if (inputFlags != null) {

datahub-graphql-core/src/main/resources/forms.graphql

@@ -1,3 +1,10 @@
+extend type Mutation {
+    """
+    Remove a form from a given list of entities.
+    """
+    batchRemoveForm(input: BatchRemoveFormInput!): Boolean!
+}
+
 """
 Requirements forms that are assigned to an entity.
 """
@@ -376,3 +383,18 @@ input VerifyFormInput {
     """
     entityUrn: String!
 }
+
+"""
+Input for batch removing a form from different entities
+"""
+input BatchRemoveFormInput {
+    """
+    The urn of the form being removed from entities
+    """
+    formUrn: String!
+
+    """
+    The entities that this form is being removed from
+    """
+    entityUrns: [String!]!
+}

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java

@@ -13,7 +13,7 @@
 import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
-import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
+import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
 import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.mxe.MetadataChangeProposal;
@@ -22,14 +22,14 @@
 
 public class TestUtils {
 
-  public static EntityService<MCPUpsertBatchItem> getMockEntityService() {
+  public static EntityService<ChangeItemImpl> getMockEntityService() {
     PathSpecBasedSchemaAnnotationVisitor.class
         .getClassLoader()
         .setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false);
     EntityRegistry registry =
         new ConfigEntityRegistry(TestUtils.class.getResourceAsStream("/test-entity-registry.yaml"));
-    EntityService<MCPUpsertBatchItem> mockEntityService =
-        (EntityService<MCPUpsertBatchItem>) Mockito.mock(EntityService.class);
+    EntityService<ChangeItemImpl> mockEntityService =
+        (EntityService<ChangeItemImpl>) Mockito.mock(EntityService.class);
     Mockito.when(mockEntityService.getEntityRegistry()).thenReturn(registry);
     return mockEntityService;
   }
@@ -111,14 +111,14 @@ public static QueryContext getMockDenyContext(String actorUrn, AuthorizationRequ
   }
 
   public static void verifyIngestProposal(
-      EntityService<MCPUpsertBatchItem> mockService,
+      EntityService<ChangeItemImpl> mockService,
       int numberOfInvocations,
       MetadataChangeProposal proposal) {
     verifyIngestProposal(mockService, numberOfInvocations, List.of(proposal));
   }
 
   public static void verifyIngestProposal(
-      EntityService<MCPUpsertBatchItem> mockService,
+      EntityService<ChangeItemImpl> mockService,
       int numberOfInvocations,
       List<MetadataChangeProposal> proposals) {
     AspectsBatchImpl batch =
@@ -128,29 +128,29 @@ public static void verifyIngestProposal(
   }
 
   public static void verifySingleIngestProposal(
-      EntityService<MCPUpsertBatchItem> mockService,
+      EntityService<ChangeItemImpl> mockService,
       int numberOfInvocations,
       MetadataChangeProposal proposal) {
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
         .ingestProposal(Mockito.eq(proposal), Mockito.any(AuditStamp.class), Mockito.eq(false));
   }
 
   public static void verifyIngestProposal(
-      EntityService<MCPUpsertBatchItem> mockService, int numberOfInvocations) {
+      EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
         .ingestProposal(Mockito.any(AspectsBatchImpl.class), Mockito.eq(false));
   }
 
   public static void verifySingleIngestProposal(
-      EntityService<MCPUpsertBatchItem> mockService, int numberOfInvocations) {
+      EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
     Mockito.verify(mockService, Mockito.times(numberOfInvocations))
         .ingestProposal(
             Mockito.any(MetadataChangeProposal.class),
             Mockito.any(AuditStamp.class),
             Mockito.eq(false));
   }
 
-  public static void verifyNoIngestProposal(EntityService<MCPUpsertBatchItem> mockService) {
+  public static void verifyNoIngestProposal(EntityService<ChangeItemImpl> mockService) {
     Mockito.verify(mockService, Mockito.times(0))
         .ingestProposal(Mockito.any(AspectsBatchImpl.class), Mockito.anyBoolean());
   }

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/BatchRemoveFormResolverTest.java

@@ -0,0 +1,83 @@
+package com.linkedin.datahub.graphql.resolvers.form;
+
+import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
+
+import com.datahub.authentication.Authentication;
+import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.BatchAssignFormInput;
+import com.linkedin.metadata.service.FormService;
+import graphql.com.google.common.collect.ImmutableList;
+import graphql.schema.DataFetchingEnvironment;
+import java.util.concurrent.CompletionException;
+import org.mockito.Mockito;
+import org.testng.annotations.Test;
+
+public class BatchRemoveFormResolverTest {
+
+  private static final String TEST_DATASET_URN =
+      "urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)";
+  private static final String TEST_FORM_URN = "urn:li:form:1";
+
+  private static final BatchAssignFormInput TEST_INPUT =
+      new BatchAssignFormInput(TEST_FORM_URN, ImmutableList.of(TEST_DATASET_URN));
+
+  @Test
+  public void testGetSuccess() throws Exception {
+    FormService mockFormService = initMockFormService(true);
+    BatchRemoveFormResolver resolver = new BatchRemoveFormResolver(mockFormService);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    boolean success = resolver.get(mockEnv).get();
+
+    assertTrue(success);
+
+    // Validate that we called unassign on the service
+    Mockito.verify(mockFormService, Mockito.times(1))
+        .batchUnassignFormForEntities(
+            Mockito.eq(ImmutableList.of(UrnUtils.getUrn(TEST_DATASET_URN))),
+            Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)),
+            Mockito.any(Authentication.class));
+  }
+
+  @Test
+  public void testThrowsError() throws Exception {
+    FormService mockFormService = initMockFormService(false);
+    BatchRemoveFormResolver resolver = new BatchRemoveFormResolver(mockFormService);
+
+    // Execute resolver
+    QueryContext mockContext = getMockAllowContext();
+    DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
+    Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
+    Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
+
+    assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
+
+    // Validate that we called unassign on the service - but it throws an error
+    Mockito.verify(mockFormService, Mockito.times(1))
+        .batchUnassignFormForEntities(
+            Mockito.eq(ImmutableList.of(UrnUtils.getUrn(TEST_DATASET_URN))),
+            Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)),
+            Mockito.any(Authentication.class));
+  }
+
+  private FormService initMockFormService(final boolean shouldSucceed) throws Exception {
+    FormService service = Mockito.mock(FormService.class);
+
+    if (!shouldSucceed) {
+      Mockito.doThrow(new RuntimeException())
+          .when(service)
+          .batchUnassignFormForEntities(
+              Mockito.any(), Mockito.any(), Mockito.any(Authentication.class));
+    }
+
+    return service;
+  }
+}

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/tag/AddTagsResolverTest.java

@@ -18,7 +18,7 @@
 import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
-import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
+import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
 import com.linkedin.mxe.MetadataChangeProposal;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.concurrent.CompletionException;
@@ -221,7 +221,7 @@ public void testGetUnauthorized() throws Exception {
 
   @Test
   public void testGetEntityClientException() throws Exception {
-    EntityService<MCPUpsertBatchItem> mockService = getMockEntityService();
+    EntityService<ChangeItemImpl> mockService = getMockEntityService();
 
     Mockito.doThrow(RuntimeException.class)
         .when(mockService)

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java

@@ -178,8 +178,9 @@ private void readerExecutable(ReaderWrapper reader, UpgradeContext context) {
       final RecordTemplate aspectRecord;
       try {
         aspectRecord =
-            EntityUtils.toAspectRecord(
-                entityName, aspectName, aspect.getMetadata(), _entityRegistry);
+            EntityUtils.toSystemAspect(aspect.toEntityAspect(), _entityService)
+                .get()
+                .getRecordTemplate();
       } catch (Exception e) {
         context
             .report()