Merge branch 'master' into feature/cus-3452-graphql-dataflowjob-container-aspect

Author: sgomezvillamor

.github/workflows/build-and-test.yml

@@ -109,8 +109,6 @@ jobs:
         if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         run: |
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
-        env:
-          NODE_OPTIONS: "--max-old-space-size=4096"
       - name: Gradle compile (jdk8) for legacy Spark
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
@@ -157,4 +155,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: Event File
-          path: ${{ github.event_path }}
+          path: ${{ github.event_path }}

.github/workflows/contributor-open-pr-comment.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Get and Format Username (PR only)
         if: github.event_name == 'pull_request'

.github/workflows/dagster-plugin.yml

@@ -30,11 +30,11 @@ jobs:
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.11"]
         include:
           - python-version: "3.9"
             extraPythonRequirement: "dagster>=1.3.3"
-          - python-version: "3.10"
+          - python-version: "3.11"
             extraPythonRequirement: "dagster>=1.3.3"
       fail-fast: false
     steps:
@@ -57,7 +57,7 @@ jobs:
         if: always()
         run: source metadata-ingestion-modules/dagster-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v4
-        if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
+        if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'dagster>=1.3.3' }}
         with:
           name: Test Results (dagster Plugin ${{ matrix.python-version}})
           path: |

.github/workflows/docker-unified.yml

@@ -445,8 +445,6 @@ jobs:
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
           mv ./datahub-frontend/build/distributions/datahub-frontend-*.zip datahub-frontend.zip
-        env:
-          NODE_OPTIONS: "--max-old-space-size=4096"
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

.github/workflows/metadata-ingestion.yml

@@ -33,7 +33,7 @@ jobs:
       # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
     strategy:
       matrix:
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.8", "3.11"]
         command:
           [
             "testQuick",
@@ -43,7 +43,7 @@ jobs:
           ]
         include:
           - python-version: "3.8"
-          - python-version: "3.10"
+          - python-version: "3.11"
       fail-fast: false
     steps:
       - name: Free up disk space

.github/workflows/prefect-plugin.yml

@@ -30,7 +30,7 @@ jobs:
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
       fail-fast: false
     steps:
       - name: Set up JDK 17
@@ -52,7 +52,7 @@ jobs:
         if: always()
         run: source metadata-ingestion-modules/prefect-plugin/venv/bin/activate && uv pip freeze
       - uses: actions/upload-artifact@v4
-        if: ${{ always() && matrix.python-version == '3.10'}}
+        if: ${{ always() && matrix.python-version == '3.11'}}
         with:
           name: Test Results (Prefect Plugin ${{ matrix.python-version}})
           path: |

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataTransformLogicMapper.java

@@ -0,0 +1,73 @@
+package com.linkedin.datahub.graphql.types.common.mappers;
+
+import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.DataTransform;
+import com.linkedin.datahub.graphql.generated.DataTransformLogic;
+import com.linkedin.datahub.graphql.generated.QueryLanguage;
+import com.linkedin.datahub.graphql.generated.QueryStatement;
+import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public class DataTransformLogicMapper
+    implements ModelMapper<
+        com.linkedin.common.DataTransformLogic,
+        com.linkedin.datahub.graphql.generated.DataTransformLogic> {
+
+  public static final DataTransformLogicMapper INSTANCE = new DataTransformLogicMapper();
+
+  public static DataTransformLogic map(
+      @Nullable final QueryContext context,
+      @Nonnull final com.linkedin.common.DataTransformLogic input) {
+    return INSTANCE.apply(context, input);
+  }
+
+  @Override
+  public DataTransformLogic apply(
+      @Nullable final QueryContext context,
+      @Nonnull final com.linkedin.common.DataTransformLogic input) {
+
+    final DataTransformLogic result = new DataTransformLogic();
+
+    // Map transforms array using DataTransformMapper
+    result.setTransforms(
+        input.getTransforms().stream()
+            .map(transform -> DataTransformMapper.map(context, transform))
+            .collect(Collectors.toList()));
+
+    return result;
+  }
+}
+
+class DataTransformMapper
+    implements ModelMapper<
+        com.linkedin.common.DataTransform, com.linkedin.datahub.graphql.generated.DataTransform> {
+
+  public static final DataTransformMapper INSTANCE = new DataTransformMapper();
+
+  public static DataTransform map(
+      @Nullable final QueryContext context,
+      @Nonnull final com.linkedin.common.DataTransform input) {
+    return INSTANCE.apply(context, input);
+  }
+
+  @Override
+  public DataTransform apply(
+      @Nullable final QueryContext context,
+      @Nonnull final com.linkedin.common.DataTransform input) {
+
+    final DataTransform result = new DataTransform();
+
+    // Map query statement if present
+    if (input.hasQueryStatement()) {
+      QueryStatement statement =
+          new QueryStatement(
+              input.getQueryStatement().getValue(),
+              QueryLanguage.valueOf(input.getQueryStatement().getLanguage().toString()));
+      result.setQueryStatement(statement);
+    }
+
+    return result;
+  }
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/QueryPropertiesMapper.java

@@ -0,0 +1,61 @@
+package com.linkedin.datahub.graphql.types.common.mappers;
+
+import com.linkedin.data.template.GetMode;
+import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.*;
+import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
+import com.linkedin.query.QueryProperties;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public class QueryPropertiesMapper
+    implements ModelMapper<
+        QueryProperties, com.linkedin.datahub.graphql.generated.QueryProperties> {
+
+  public static final QueryPropertiesMapper INSTANCE = new QueryPropertiesMapper();
+
+  public static com.linkedin.datahub.graphql.generated.QueryProperties map(
+      @Nullable final QueryContext context, @Nonnull final QueryProperties input) {
+    return INSTANCE.apply(context, input);
+  }
+
+  @Override
+  public com.linkedin.datahub.graphql.generated.QueryProperties apply(
+      @Nullable final QueryContext context, @Nonnull final QueryProperties input) {
+
+    final com.linkedin.datahub.graphql.generated.QueryProperties result =
+        new com.linkedin.datahub.graphql.generated.QueryProperties();
+
+    // Map Query Source
+    result.setSource(QuerySource.valueOf(input.getSource().toString()));
+
+    // Map Query Statement
+    result.setStatement(
+        new QueryStatement(
+            input.getStatement().getValue(),
+            QueryLanguage.valueOf(input.getStatement().getLanguage().toString())));
+
+    // Map optional fields
+    result.setName(input.getName(GetMode.NULL));
+    result.setDescription(input.getDescription(GetMode.NULL));
+
+    // Map origin if present
+    if (input.hasOrigin() && input.getOrigin() != null) {
+      result.setOrigin(UrnToEntityMapper.map(context, input.getOrigin()));
+    }
+
+    // Map created audit stamp
+    AuditStamp created = new AuditStamp();
+    created.setTime(input.getCreated().getTime());
+    created.setActor(input.getCreated().getActor(GetMode.NULL).toString());
+    result.setCreated(created);
+
+    // Map last modified audit stamp
+    AuditStamp lastModified = new AuditStamp();
+    lastModified.setTime(input.getLastModified().getTime());
+    lastModified.setActor(input.getLastModified().getActor(GetMode.NULL).toString());
+    result.setLastModified(lastModified);
+
+    return result;
+  }
+}

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java

@@ -79,7 +79,8 @@ public class DataJobType
           BROWSE_PATHS_V2_ASPECT_NAME,
           SUB_TYPES_ASPECT_NAME,
           STRUCTURED_PROPERTIES_ASPECT_NAME,
-          FORMS_ASPECT_NAME);
+          FORMS_ASPECT_NAME,
+          DATA_TRANSFORM_LOGIC_ASPECT_NAME);
   private static final Set<String> FACET_FIELDS = ImmutableSet.of("flow");
   private final EntityClient _entityClient;
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java

@@ -4,16 +4,7 @@
 import static com.linkedin.metadata.Constants.*;
 
 import com.google.common.collect.ImmutableList;
-import com.linkedin.common.BrowsePathsV2;
-import com.linkedin.common.DataPlatformInstance;
-import com.linkedin.common.Deprecation;
-import com.linkedin.common.Forms;
-import com.linkedin.common.GlobalTags;
-import com.linkedin.common.GlossaryTerms;
-import com.linkedin.common.InstitutionalMemory;
-import com.linkedin.common.Ownership;
-import com.linkedin.common.Status;
-import com.linkedin.common.SubTypes;
+import com.linkedin.common.*;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.DataMap;
 import com.linkedin.datahub.graphql.QueryContext;
@@ -27,15 +18,7 @@
 import com.linkedin.datahub.graphql.generated.DataJobProperties;
 import com.linkedin.datahub.graphql.generated.Dataset;
 import com.linkedin.datahub.graphql.generated.EntityType;
-import com.linkedin.datahub.graphql.types.common.mappers.BrowsePathsV2Mapper;
-import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.DeprecationMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.FineGrainedLineagesMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper;
-import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.*;
 import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils;
 import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
 import com.linkedin.datahub.graphql.types.form.FormsMapper;
@@ -148,6 +131,9 @@ public DataJob apply(
                         context, new StructuredProperties(data), entityUrn));
               } else if (FORMS_ASPECT_NAME.equals(name)) {
                 result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString()));
+              } else if (DATA_TRANSFORM_LOGIC_ASPECT_NAME.equals(name)) {
+                result.setDataTransformLogic(
+                    DataTransformLogicMapper.map(context, new DataTransformLogic(data)));
               }
             });
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/query/QueryMapper.java

@@ -5,18 +5,13 @@
 import com.linkedin.common.DataPlatformInstance;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.DataMap;
-import com.linkedin.data.template.GetMode;
 import com.linkedin.datahub.graphql.QueryContext;
-import com.linkedin.datahub.graphql.generated.AuditStamp;
 import com.linkedin.datahub.graphql.generated.DataPlatform;
 import com.linkedin.datahub.graphql.generated.Dataset;
 import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.generated.QueryEntity;
-import com.linkedin.datahub.graphql.generated.QueryLanguage;
-import com.linkedin.datahub.graphql.generated.QuerySource;
-import com.linkedin.datahub.graphql.generated.QueryStatement;
 import com.linkedin.datahub.graphql.generated.QuerySubject;
-import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
+import com.linkedin.datahub.graphql.types.common.mappers.QueryPropertiesMapper;
 import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper;
 import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
 import com.linkedin.entity.EntityResponse;
@@ -48,7 +43,10 @@ public QueryEntity apply(
     result.setType(EntityType.QUERY);
     EnvelopedAspectMap aspectMap = entityResponse.getAspects();
     MappingHelper<QueryEntity> mappingHelper = new MappingHelper<>(aspectMap, result);
-    mappingHelper.mapToResult(context, QUERY_PROPERTIES_ASPECT_NAME, this::mapQueryProperties);
+    mappingHelper.mapToResult(
+        QUERY_PROPERTIES_ASPECT_NAME,
+        (entity, dataMap) ->
+            entity.setProperties(QueryPropertiesMapper.map(context, new QueryProperties(dataMap))));
     mappingHelper.mapToResult(QUERY_SUBJECTS_ASPECT_NAME, this::mapQuerySubjects);
     mappingHelper.mapToResult(DATA_PLATFORM_INSTANCE_ASPECT_NAME, this::mapPlatform);
     return mappingHelper.getResult();
@@ -64,37 +62,6 @@ private void mapPlatform(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
     }
   }
 
-  private void mapQueryProperties(
-      @Nullable final QueryContext context, @Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
-    QueryProperties queryProperties = new QueryProperties(dataMap);
-    com.linkedin.datahub.graphql.generated.QueryProperties res =
-        new com.linkedin.datahub.graphql.generated.QueryProperties();
-
-    // Query Source must be kept in sync.
-    res.setSource(QuerySource.valueOf(queryProperties.getSource().toString()));
-    res.setStatement(
-        new QueryStatement(
-            queryProperties.getStatement().getValue(),
-            QueryLanguage.valueOf(queryProperties.getStatement().getLanguage().toString())));
-    res.setName(queryProperties.getName(GetMode.NULL));
-    res.setDescription(queryProperties.getDescription(GetMode.NULL));
-    if (queryProperties.hasOrigin() && queryProperties.getOrigin() != null) {
-      res.setOrigin(UrnToEntityMapper.map(context, queryProperties.getOrigin()));
-    }
-
-    AuditStamp created = new AuditStamp();
-    created.setTime(queryProperties.getCreated().getTime());
-    created.setActor(queryProperties.getCreated().getActor(GetMode.NULL).toString());
-    res.setCreated(created);
-
-    AuditStamp lastModified = new AuditStamp();
-    lastModified.setTime(queryProperties.getLastModified().getTime());
-    lastModified.setActor(queryProperties.getLastModified().getActor(GetMode.NULL).toString());
-    res.setLastModified(lastModified);
-
-    query.setProperties(res);
-  }
-
   @Nonnull
   private void mapQuerySubjects(@Nonnull QueryEntity query, @Nonnull DataMap dataMap) {
     QuerySubjects querySubjects = new QuerySubjects(dataMap);

datahub-graphql-core/src/main/resources/entity.graphql

@@ -6589,6 +6589,11 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity {
     The forms associated with the Dataset
     """
     forms: Forms
+
+    """
+    Data Transform Logic associated with the Data Job
+    """
+    dataTransformLogic: DataTransformLogic
 }
 
 """
@@ -6806,6 +6811,26 @@ type DataJobInputOutput {
     fineGrainedLineages: [FineGrainedLineage!]
 }
 
+"""
+Information about a transformation applied to data assets
+"""
+type DataTransform {
+  """
+  The transformation may be defined by a query statement
+  """
+  queryStatement: QueryStatement
+}
+
+"""
+Information about transformations applied to data assets
+"""
+type DataTransformLogic {
+  """
+  List of transformations applied
+  """
+  transforms: [DataTransform!]!
+}
+
 """
 Information about individual user usage of a Dataset
 """