Merge branch 'master' into feature/ing-804-feat-snowflake-is-temp-table

Author: sgomezvillamor

.github/workflows/docker-unified.yml

@@ -1018,8 +1018,8 @@ jobs:
         # python_batch_count is used to split pytests in the smoke-test (batches of actual test functions)
         # cypress_batch_count is used to split the collection of cypress test specs into batches.
         run: |
-          echo "cypress_batch_count=11" >> "$GITHUB_OUTPUT"
-          echo "python_batch_count=5" >> "$GITHUB_OUTPUT"
+          echo "cypress_batch_count=5" >> "$GITHUB_OUTPUT"
+          echo "python_batch_count=3" >> "$GITHUB_OUTPUT"
 
       - id: set-matrix
         # For m batches for python and n batches for cypress, we need a test matrix of python x m + cypress x n.

build.gradle

@@ -39,13 +39,13 @@ buildscript {
   ext.springBootVersion = '3.2.9'
   ext.springKafkaVersion = '3.1.6'
   ext.openTelemetryVersion = '1.18.0'
-  ext.neo4jVersion = '5.14.0'
-  ext.neo4jTestVersion = '5.14.0'
-  ext.neo4jApocVersion = '5.14.0'
+  ext.neo4jVersion = '5.20.0'
+  ext.neo4jTestVersion = '5.20.0'
+  ext.neo4jApocVersion = '5.20.0'
   ext.testContainersVersion = '1.17.4'
   ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
-  ext.jettyVersion = '11.0.21'
+  ext.jettyVersion = '12.0.16'
   // see also datahub-frontend/play.gradle
   ext.playVersion = '2.8.22'
   ext.playScalaVersion = '2.13'
@@ -136,7 +136,8 @@ project.ext.externalDependency = [
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
     'datastaxOssCore': 'com.datastax.oss:java-driver-core:4.14.1',
     'datastaxOssQueryBuilder': 'com.datastax.oss:java-driver-query-builder:4.14.1',
-    'dgraph4j' : 'io.dgraph:dgraph4j:21.12.0',
+    'dgraph4j' : 'io.dgraph:dgraph4j:24.1.1',
+    'dgraphNetty': 'io.grpc:grpc-netty-shaded:1.69.0',
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
@@ -176,8 +177,9 @@ project.ext.externalDependency = [
     'jakartaValidation': 'jakarta.validation:jakarta.validation-api:3.1.0-M2',
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.41',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
-    'jettyJaas': "org.eclipse.jetty:jetty-jaas:$jettyVersion",
+    'jettySecurity': "org.eclipse.jetty:jetty-security:$jettyVersion",
     'jettyClient': "org.eclipse.jetty:jetty-client:$jettyVersion",
+    'jettyJmx': "org.eclipse.jetty:jetty-jmx:$jettyVersion",
     'jettison': 'org.codehaus.jettison:jettison:1.5.4',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
     'jna': 'net.java.dev.jna:jna:5.12.1',
@@ -380,6 +382,13 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
     resolutionStrategy.force externalDependency.antlr4Runtime
     resolutionStrategy.force externalDependency.antlr4
     resolutionStrategy.force 'org.apache.mina:mina-core:2.2.4'
+    resolutionStrategy {
+      force "org.eclipse.jetty:jetty-security:${jettyVersion}"
+      force "org.eclipse.jetty:jetty-server:${jettyVersion}"
+      force "org.eclipse.jetty:jetty-ee10-servlet:${jettyVersion}"
+      force "org.eclipse.jetty:jetty-ee10-webapp:${jettyVersion}"
+      force "org.eclipse.jetty:jetty-xml:${jettyVersion}"
+    }
   }
 }
 
@@ -407,7 +416,7 @@ subprojects {
       implementation externalDependency.annotationApi
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
-        implementation('io.netty:netty-all:4.1.115.Final')
+        implementation('io.netty:netty-all:4.1.116.Final')
         implementation('org.apache.commons:commons-compress:1.27.1')
         implementation('org.apache.velocity:velocity-engine-core:2.4')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')

datahub-frontend/app/security/AuthenticationManager.java

@@ -1,68 +1,33 @@
 package security;
 
 import com.google.common.base.Preconditions;
-import java.util.Collections;
 import javax.annotation.Nonnull;
 import javax.naming.AuthenticationException;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.auth.login.LoginContext;
-import javax.security.auth.login.LoginException;
 import org.apache.commons.lang3.StringUtils;
-import org.eclipse.jetty.jaas.JAASLoginService;
-import org.eclipse.jetty.jaas.PropertyUserStoreManager;
-import play.Logger;
+import org.eclipse.jetty.security.UserPrincipal;
+import org.eclipse.jetty.util.security.Credential;
 
 public class AuthenticationManager {
-
-  private AuthenticationManager(boolean verbose) {}
+  private AuthenticationManager() {} // Prevent instantiation
 
   public static void authenticateJaasUser(@Nonnull String userName, @Nonnull String password)
       throws Exception {
     Preconditions.checkArgument(!StringUtils.isAnyEmpty(userName), "Username cannot be empty");
-    JAASLoginService jaasLoginService = new JAASLoginService("WHZ-Authentication");
-    PropertyUserStoreManager propertyUserStoreManager = new PropertyUserStoreManager();
-    propertyUserStoreManager.start();
-    jaasLoginService.setBeans(Collections.singletonList(propertyUserStoreManager));
-    JAASLoginService.INSTANCE.set(jaasLoginService);
-    try {
-      LoginContext lc =
-          new LoginContext("WHZ-Authentication", new WHZCallbackHandler(userName, password));
-      lc.login();
-    } catch (LoginException le) {
-      AuthenticationException authenticationException =
-          new AuthenticationException(le.getMessage());
-      authenticationException.setRootCause(le);
-      throw authenticationException;
-    }
-  }
 
-  private static class WHZCallbackHandler implements CallbackHandler {
-    private String password;
-    private String username;
-
-    private WHZCallbackHandler(@Nonnull String username, @Nonnull String password) {
-      this.username = username;
-      this.password = password;
-    }
+    try {
+      // Create and configure credentials for authentication
+      UserPrincipal userPrincipal = new UserPrincipal(userName, Credential.getCredential(password));
 
-    @Override
-    public void handle(@Nonnull Callback[] callbacks) {
-      NameCallback nc = null;
-      PasswordCallback pc = null;
-      for (Callback callback : callbacks) {
-        Logger.debug(
-            "The submitted callback is of type: " + callback.getClass() + " : " + callback);
-        if (callback instanceof NameCallback) {
-          nc = (NameCallback) callback;
-          nc.setName(this.username);
-        } else if (callback instanceof PasswordCallback) {
-          pc = (PasswordCallback) callback;
-          pc.setPassword(this.password.toCharArray());
-        }
+      // Verify credentials
+      if (!userPrincipal.authenticate(password)) {
+        throw new AuthenticationException("Invalid credentials for user: " + userName);
       }
+
+    } catch (Exception e) {
+      AuthenticationException authenticationException =
+          new AuthenticationException("Authentication failed");
+      authenticationException.setRootCause(e);
+      throw authenticationException;
     }
   }
 }

datahub-frontend/play.gradle

@@ -50,7 +50,7 @@ dependencies {
   implementation externalDependency.springBeans
   implementation externalDependency.springContext
   implementation externalDependency.springBootAutoconfigure
-  implementation externalDependency.jettyJaas
+  implementation externalDependency.jettySecurity
   implementation externalDependency.graphqlJava
   implementation externalDependency.antlr4Runtime
   implementation externalDependency.antlr4

datahub-web-react/package.json

@@ -152,7 +152,7 @@
         "prettier": "^2.8.8",
         "source-map-explorer": "^2.5.2",
         "storybook": "^8.1.11",
-        "vite": "^4.5.5",
+        "vite": "^4.5.6",
         "vite-plugin-babel-macros": "^1.0.6",
         "vite-plugin-static-copy": "^0.17.0",
         "vite-plugin-svgr": "^4.1.0",

datahub-web-react/src/app/entity/domain/DataProductsTab/DataProductsTab.tsx

@@ -68,6 +68,7 @@ export default function DataProductsTab() {
                 searchFlags: { skipCache: true },
             },
         },
+        fetchPolicy: 'no-cache',
     });
     const totalResults = data?.searchAcrossEntities?.total || 0;
     const searchResults = data?.searchAcrossEntities?.searchResults?.map((r) => r.entity) || [];

datahub-web-react/yarn.lock

@@ -12066,10 +12066,10 @@ vite-plugin-svgr@^4.1.0:
     "@svgr/core" "^8.1.0"
     "@svgr/plugin-jsx" "^8.1.0"
 
-"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.5:
-  version "4.5.5"
-  resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.5.tgz#639b9feca5c0a3bfe3c60cb630ef28bf219d742e"
-  integrity sha512-ifW3Lb2sMdX+WU91s3R0FyQlAyLxOzCSCP37ujw0+r5POeHPwe6udWVIElKQq8gk3t7b8rkmvqC6IHBpCff4GQ==
+"vite@^3.0.0 || ^4.0.0 || ^5.0.0-0", "vite@^3.1.0 || ^4.0.0 || ^5.0.0-0", vite@^4.5.6:
+  version "4.5.6"
+  resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.6.tgz#48bbd97fe06e8241df2e625b31c581707e10b57d"
+  integrity sha512-ElBNuVvJKslxcfY2gMmae5IjaKGqCYGicCNZ+8R56sAznobeE3pI9ctzI17cBS/6OJh5YuQNMSN4BP4dRjugBg==
   dependencies:
     esbuild "^0.18.10"
     postcss "^8.4.27"

docker/datahub-gms/Dockerfile

@@ -26,7 +26,6 @@ RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
 FROM alpine:3.21 AS base
 
 ENV JMX_VERSION=0.18.0
-ENV JETTY_VERSION=11.0.21
 
 # Re-declaring args from above to make them available in this stage (will inherit default values)
 ARG ALPINE_REPO_URL
@@ -42,9 +41,6 @@ RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat snappy \
     && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
     && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/${JETTY_VERSION}/jetty-runner-${JETTY_VERSION}.jar --output jetty-runner.jar \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/${JETTY_VERSION}/jetty-jmx-${JETTY_VERSION}.jar --output jetty-jmx.jar \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/${JETTY_VERSION}/jetty-util-${JETTY_VERSION}.jar --output jetty-util.jar \
     && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
     && wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
     && cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks
@@ -56,8 +52,6 @@ FROM base AS prod-install
 COPY war.war /datahub/datahub-gms/bin/war.war
 COPY metadata-models/src/main/resources/entity-registry.yml /datahub/datahub-gms/resources/entity-registry.yml
 COPY docker/datahub-gms/start.sh /datahub/datahub-gms/scripts/start.sh
-COPY docker/datahub-gms/jetty.xml /datahub/datahub-gms/scripts/jetty.xml
-COPY docker/datahub-gms/jetty-jmx.xml /datahub/datahub-gms/scripts/jetty-jmx.xml
 COPY docker/monitoring/client-prometheus-config.yaml /datahub/datahub-gms/scripts/prometheus-config.yaml
 RUN chmod +x /datahub/datahub-gms/scripts/start.sh
 
@@ -70,7 +64,7 @@ FROM ${APP_ENV}-install AS final
 RUN mkdir -p /etc/datahub/plugins/auth/resources
 
 RUN addgroup -S datahub && adduser -S datahub -G datahub
-RUN chown -R datahub:datahub /etc/datahub
+RUN chown -R datahub:datahub /etc/datahub /datahub
 USER datahub
 
 ENV JMX_OPTS=""

docker/datahub-gms/jetty-jmx.xml

@@ -62,13 +62,8 @@ COMMON="
     java $JAVA_OPTS $JMX_OPTS \
     $OTEL_AGENT \
     $PROMETHEUS_AGENT \
-    -jar /jetty-runner.jar \
-    --stats unsecure \
-    --jar jetty-util.jar \
-    --jar jetty-jmx.jar \
-    --config /datahub/datahub-gms/scripts/jetty.xml \
-    --config /datahub/datahub-gms/scripts/jetty-jmx.xml \
-    /datahub/datahub-gms/bin/war.war"
+    -Dstats=unsecure \
+    -jar /datahub/datahub-gms/bin/war.war"
 
 if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
   exec dockerize \

docker/datahub-gms/jetty.xml

@@ -20,9 +20,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 COPY . /datahub-src
 ARG RELEASE_VERSION
 RUN cd /datahub-src && \
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion/src/datahub/__init__.py && \
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/__init__.py && \
-    cat metadata-ingestion/src/datahub/__init__.py && \
+    sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion/src/datahub/_version.py && \
+    sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_version.py && \
+    cat metadata-ingestion/src/datahub/_version.py && \
     ./gradlew :metadata-ingestion:codegen && \
     pip install file:metadata-ingestion-modules/airflow-plugin#egg=acryl-datahub-airflow-plugin file:metadata-ingestion#egg=acryl-datahub
 

docker/datahub-gms/start.sh

@@ -26,10 +26,10 @@ COPY --chown=datahub ./metadata-ingestion-modules/airflow-plugin /metadata-inges
 
 ARG RELEASE_VERSION
 WORKDIR /metadata-ingestion
-RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \
-    cat src/datahub/__init__.py | grep __version__ && \
-    cat airflow-plugin/src/datahub_airflow_plugin/__init__.py | grep __version__
+RUN sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/_version.py && \
+    sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/_version.py && \
+    cat src/datahub/_version.py | grep __version__ && \
+    cat airflow-plugin/src/datahub_airflow_plugin/_version.py | grep __version__
 
 FROM base AS slim-install
 

docker/datahub-ingestion-base/smoke.Dockerfile

@@ -15,8 +15,8 @@ COPY --chown=datahub ./metadata-ingestion /metadata-ingestion
 
 ARG RELEASE_VERSION
 WORKDIR /metadata-ingestion
-RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \
-    cat src/datahub/__init__.py
+RUN sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/_version.py && \
+    cat src/datahub/_version.py
 
 FROM base as slim-install
 

docker/datahub-ingestion/Dockerfile

@@ -49,8 +49,6 @@ services:
     - KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR=${KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR:-true}
     volumes:
     - ./datahub-gms/start.sh:/datahub/datahub-gms/scripts/start.sh
-    - ./datahub-gms/jetty.xml:/datahub/datahub-gms/scripts/jetty.xml
-    - ./datahub-gms/jetty-jmx.xml:/datahub/datahub-gms/scripts/jetty-jmx.xml
     - ./monitoring/client-prometheus-config.yaml:/datahub/datahub-gms/scripts/prometheus-config.yaml
     - ../metadata-models/src/main/resources/:/datahub/datahub-gms/resources
     - ../metadata-service/war/build/libs/:/datahub/datahub-gms/bin

docker/datahub-ingestion/Dockerfile-slim-only

@@ -26,6 +26,7 @@ x-datahub-frontend-service-dev: &datahub-frontend-service-dev
     DATAHUB_ANALYTICS_ENABLED: ${DATAHUB_ANALYTICS_ENABLED:-true}
   volumes:
     - ../../datahub-frontend/build/stage/main:/datahub-frontend
+    - ./monitoring/client-prometheus-config.yaml:/datahub-frontend/client-prometheus-config.yaml
 
 services:
   frontend-quickstart: