Merge branch 'master' into master

Author: mkamalas

.github/.codecov.yml

@@ -1,21 +1,19 @@
 comment:
-  layout: "header, files, footer" # remove "new" from "header" and "footer"
-  hide_project_coverage: true # set to false
+  layout: "condensed_header, condensed_files, condensed_footer"
+  hide_project_coverage: true
   require_changes: false  # if true: only post the comment if coverage changes
 
 codecov:
   #due to ci-optimization, reports for modules that have not changed may be quite old
   max_report_age: off
 
+github_checks:
+  #Hide annotations that show up in github PR reviews. There still is a red bar next to lines not covered
+  annotations: false
+
 flag_management:
   default_rules: # the rules that will be followed for any flag added, generally
     carryforward: true
-    statuses:
-      - type: project
-        target: auto
-        threshold: 0%  #Not enforcing project coverage yet.
-      - type: patch
-        target: 90%
   individual_flags: # exceptions to the default rules above, stated flag by flag
     - name: frontend
       paths:
@@ -55,11 +53,8 @@ flag_management:
         - "metadata-ingestion-modules/prefect-plugin/**"
 coverage:
   status:
-    project:
-      default:
-        target: 0% # no threshold enforcement yet
-        only_pulls: true
+    project: false
     patch:
       default:
-        target: 90% # for new code added in the patch
-        only_pulls: true
+        target: 75% # for new code added in the patch
+        only_pulls: true

.github/workflows/docker-unified.yml

@@ -1274,6 +1274,11 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
   deploy_datahub_head:
     name: Deploy to Datahub HEAD
     runs-on: ubuntu-latest

.github/workflows/metadata-ingestion.yml

@@ -27,7 +27,7 @@ concurrency:
 jobs:
   metadata-ingestion:
     runs-on: ubuntu-latest
-    timeout-minutes: 40
+    timeout-minutes: 60
     env:
       DATAHUB_TELEMETRY_ENABLED: false
       # TODO: Enable this once the test is fixed.

.github/workflows/pr-labeler.yml

@@ -51,7 +51,8 @@ jobs:
                 "acrylJonny",
                 "chakru-r",
                 "brock-acryl",
-                "mminichino"
+                "mminichino",
+                "jayacryl"
               ]'), 
               github.actor
             ) 

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-17 16:43:31 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-02-11 10:00:11 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -53,13 +53,6 @@ repos:
         files: ^metadata-dao-impl/kafka-producer/.*\.java$
         pass_filenames: false
 
-      - id: metadata-events-mxe-avro-spotless
-        name: metadata-events/mxe-avro Spotless Apply
-        entry: ./gradlew :metadata-events:mxe-avro:spotlessApply
-        language: system
-        files: ^metadata-events/mxe-avro/.*\.java$
-        pass_filenames: false
-
       - id: metadata-events-mxe-registration-spotless
         name: metadata-events/mxe-registration Spotless Apply
         entry: ./gradlew :metadata-events:mxe-registration:spotlessApply
@@ -312,6 +305,20 @@ repos:
         files: ^metadata-service/graphql-servlet-impl/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-iceberg-catalog-spotless
+        name: metadata-service/iceberg-catalog Spotless Apply
+        entry: ./gradlew :metadata-service:iceberg-catalog:spotlessApply
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-iceberg-catalog-lint-fix
+        name: metadata-service/iceberg-catalog Lint Fix
+        entry: ./gradlew :metadata-service:iceberg-catalog:lintFix
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.py$
+        pass_filenames: false
+
       - id: metadata-service-openapi-analytics-servlet-spotless
         name: metadata-service/openapi-analytics-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-analytics-servlet:spotlessApply
@@ -410,6 +417,13 @@ repos:
         files: ^metadata-service/servlet/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-war-spotless
+        name: metadata-service/war Spotless Apply
+        entry: ./gradlew :metadata-service:war:spotlessApply
+        language: system
+        files: ^metadata-service/war/.*\.java$
+        pass_filenames: false
+
       - id: metadata-utils-spotless
         name: metadata-utils Spotless Apply
         entry: ./gradlew :metadata-utils:spotlessApply

README.md

@@ -18,27 +18,38 @@ export const Logo = (props) => {
 <!--
 HOSTED_DOCS_ONLY-->
 <p align="center">
+<a href="https://datahubproject.io">
 <img alt="DataHub" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" height="150" />
+</a>
 </p>
 <!-- -->
 
 # DataHub: The Data Discovery Platform for the Modern Data Stack
-## Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="25"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="25"/> [LinkedIn](https://engineering.linkedin.com)
-[![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest)
-[![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
-[![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
-[![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
-[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme)
-[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
-[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)
-[![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE)
-[![YouTube](https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=social)](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w)
-[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://medium.com/datahub-project)
-[![Follow](https://img.shields.io/twitter/follow/datahubproject?label=Follow&style=social)](https://twitter.com/datahubproject)
-### 🏠 Hosted DataHub Docs (Courtesy of Acryl Data): [datahubproject.io](https://datahubproject.io/docs)
+
+### Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="20"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="20"/> [LinkedIn](https://engineering.linkedin.com)
+
+<div>
+  <a target="_blank" href="https://github.com/datahub-project/datahub/blob/master/LICENSE">
+    <img alt="Apache 2.0 License" src="https://img.shields.io/badge/License-Apache_2.0-blue.svg?label=license&labelColor=133554&color=1890ff" /></a>
+  <a target="_blank" href="https://pypi.org/project/acryl-datahub/">
+    <img alt="PyPI" src="https://img.shields.io/pypi/dm/acryl-datahub?label=downloads&labelColor=133554&color=1890ff" /></a>
+  <a target="_blank" href="https://github.com/datahub-project/datahub/pulse">
+    <img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/datahub-project/datahub?label=commits&labelColor=133554&color=1890ff" /></a>
+  <br />
+  <a target="_blank" href="https://pages.acryl.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme">
+    <img alt="Slack" src="https://img.shields.io/badge/slack-join_community-red.svg?logo=slack&labelColor=133554&color=1890ff" /></a>
+  <a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w">
+    <img alt="YouTube" src="https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=flat&logo=youtube&label=subscribers&labelColor=133554&color=1890ff"/></a>
+  <a href="https://blog.datahubproject.io/">
+    <img alt="Medium" src="https://img.shields.io/badge/blog-DataHub-red.svg?style=flat&logo=medium&logoColor=white&labelColor=133554&color=1890ff" /></a>
+  <a href="https://x.com/datahubproject">
+    <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/badge/follow-datahubproject-red.svg?style=flat&logo=x&labelColor=133554&color=1890ff" /></a>
+</div>
 
 ---
 
+### 🏠 Docs: [datahubproject.io](https://datahubproject.io/docs)
+
 [Quickstart](https://datahubproject.io/docs/quickstart) |
 [Features](https://datahubproject.io/docs/) |
 [Roadmap](https://feature-requests.datahubproject.io/roadmap) |
@@ -47,6 +58,7 @@ HOSTED_DOCS_ONLY-->
 [Town Hall](https://datahubproject.io/docs/townhalls)
 
 ---
+
 > 📣 DataHub Town Hall is the 4th Thursday at 9am US PT of every month - [add it to your calendar!](https://rsvp.datahubproject.io/)
 >
 > - Town-hall Zoom link: [zoom.datahubproject.io](https://zoom.datahubproject.io)
@@ -70,11 +82,11 @@ Check out DataHub's [Features](docs/features.md) & [Roadmap](https://feature-req
 
 ## Demo and Screenshots
 
-There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally
+There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally.
 
 ## Quickstart
 
-Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to get a copy of DataHub up & running locally using [Docker](https://docker.com). As the guide assumes some basic knowledge of Docker, we'd recommend you to go through the "Hello World" example of [A Docker Tutorial for Beginners](https://docker-curriculum.com) if Docker is completely foreign to you.
+Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to run DataHub locally using [Docker](https://docker.com).
 
 ## Development
 
@@ -106,7 +118,7 @@ We welcome contributions from the community. Please refer to our [Contributing G
 
 ## Community
 
-Join our [Slack workspace](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
+Join our [Slack workspace](https://pages.acryl.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
 
 ## Security
 
@@ -154,12 +166,11 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Udemy](https://www.udemy.com/)
 - [Uphold](https://uphold.com)
 - [Viasat](https://viasat.com)
+- [Wealthsimple](https://www.wealthsimple.com)
 - [Wikimedia](https://www.wikimedia.org)
 - [Wolt](https://wolt.com)
 - [Zynga](https://www.zynga.com)
 
-
-
 ## Select Articles & Talks
 
 - [DataHub Blog](https://blog.datahubproject.io/)

build.gradle

@@ -186,7 +186,7 @@ project.ext.externalDependency = [
     'jsonPatch': 'jakarta.json:jakarta.json-api:2.1.3',
     'jsonPathImpl': 'org.eclipse.parsson:parsson:1.1.6',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
-    'jsonSmart': 'net.minidev:json-smart:2.4.9',
+    'jsonSmart': 'net.minidev:json-smart:2.5.2',
     'json': 'org.json:json:20231013',
     'jsonSchemaValidator': 'com.github.java-json-tools:json-schema-validator:2.2.14',
     'junit': 'junit:junit:4.13.2',
@@ -419,7 +419,7 @@ subprojects {
       implementation externalDependency.annotationApi
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
-        implementation('io.netty:netty-all:4.1.116.Final')
+        implementation('io.netty:netty-all:4.1.118.Final')
         implementation('org.apache.commons:commons-compress:1.27.1')
         implementation('org.apache.velocity:velocity-engine-core:2.4')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')

datahub-frontend/app/auth/GuestAuthenticationConfigs.java

@@ -0,0 +1,40 @@
+package auth;
+
+public class GuestAuthenticationConfigs {
+  public static final String GUEST_ENABLED_CONFIG_PATH = "auth.guest.enabled";
+  public static final String GUEST_USER_CONFIG_PATH = "auth.guest.user";
+  public static final String GUEST_PATH_CONFIG_PATH = "auth.guest.path";
+  public static final String DEFAULT_GUEST_USER_NAME = "guest";
+  public static final String DEFAULT_GUEST_PATH = "/public";
+
+  private Boolean isEnabled = false;
+  private String guestUser =
+      DEFAULT_GUEST_USER_NAME; // Default if not defined but guest auth is enabled.
+  private String guestPath =
+      DEFAULT_GUEST_PATH; // The path for initial access to login as guest and bypass login page.
+
+  public GuestAuthenticationConfigs(final com.typesafe.config.Config configs) {
+    if (configs.hasPath(GUEST_ENABLED_CONFIG_PATH)
+        && configs.getBoolean(GUEST_ENABLED_CONFIG_PATH)) {
+      isEnabled = true;
+    }
+    if (configs.hasPath(GUEST_USER_CONFIG_PATH)) {
+      guestUser = configs.getString(GUEST_USER_CONFIG_PATH);
+    }
+    if (configs.hasPath(GUEST_PATH_CONFIG_PATH)) {
+      guestPath = configs.getString(GUEST_PATH_CONFIG_PATH);
+    }
+  }
+
+  public boolean isGuestEnabled() {
+    return isEnabled;
+  }
+
+  public String getGuestUser() {
+    return guestUser;
+  }
+
+  public String getGuestPath() {
+    return guestPath;
+  }
+}

datahub-frontend/app/controllers/AuthenticationController.java

@@ -6,6 +6,7 @@
 
 import auth.AuthUtils;
 import auth.CookieConfigs;
+import auth.GuestAuthenticationConfigs;
 import auth.JAASConfigs;
 import auth.NativeAuthenticationConfigs;
 import auth.sso.SsoManager;
@@ -58,6 +59,8 @@ public class AuthenticationController extends Controller {
   private final CookieConfigs cookieConfigs;
   private final JAASConfigs jaasConfigs;
   private final NativeAuthenticationConfigs nativeAuthenticationConfigs;
+  private final GuestAuthenticationConfigs guestAuthenticationConfigs;
+
   private final boolean verbose;
 
   @Inject private org.pac4j.core.config.Config ssoConfig;
@@ -73,6 +76,7 @@ public AuthenticationController(@Nonnull Config configs) {
     cookieConfigs = new CookieConfigs(configs);
     jaasConfigs = new JAASConfigs(configs);
     nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs);
+    guestAuthenticationConfigs = new GuestAuthenticationConfigs(configs);
     verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING);
   }
 
@@ -110,6 +114,23 @@ public Result authenticate(Http.Request request) {
       return Results.redirect(redirectPath);
     }
 
+    if (guestAuthenticationConfigs.isGuestEnabled()
+        && guestAuthenticationConfigs.getGuestPath().equals(redirectPath)) {
+      final String accessToken =
+          authClient.generateSessionTokenForUser(guestAuthenticationConfigs.getGuestUser());
+      redirectPath =
+          "/"; // We requested guest login by accessing {guestPath} URL. It is not really a target.
+      CorpuserUrn guestUserUrn = new CorpuserUrn(guestAuthenticationConfigs.getGuestUser());
+      return Results.redirect(redirectPath)
+          .withSession(createSessionMap(guestUserUrn.toString(), accessToken))
+          .withCookies(
+              createActorCookie(
+                  guestUserUrn.toString(),
+                  cookieConfigs.getTtlInHours(),
+                  cookieConfigs.getAuthCookieSameSite(),
+                  cookieConfigs.getAuthCookieSecure()));
+    }
+
     // 1. If SSO is enabled, redirect to IdP if not authenticated.
     if (ssoManager.isSsoEnabled()) {
       return redirectToIdentityProvider(request, redirectPath)

datahub-frontend/conf/application.conf

@@ -203,6 +203,11 @@ auth.oidc.grantType = ${?AUTH_OIDC_GRANT_TYPE}
 #
 auth.jaas.enabled = ${?AUTH_JAAS_ENABLED}
 auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
+auth.guest.enabled = ${?GUEST_AUTHENTICATION_ENABLED}
+# The name of the guest user id
+auth.guest.user = ${?GUEST_AUTHENTICATION_USER}
+# The path to bypass login page and get logged in as guest
+auth.guest.path = ${?GUEST_AUTHENTICATION_PATH}
 
 # Enforces the usage of a valid email for user sign up
 auth.native.signUp.enforceValidEmail = true

datahub-frontend/play.gradle

@@ -32,7 +32,7 @@ dependencies {
     play(externalDependency.jacksonDataBind)
     play("com.typesafe.akka:akka-actor_$playScalaVersion:2.6.20")
     play(externalDependency.jsonSmart)
-    play('io.netty:netty-all:4.1.114.Final')
+    play('io.netty:netty-all:4.1.118.Final')
 
     implementation(externalDependency.commonsText) {
       because("previous versions are vulnerable to CVE-2022-42889")