From 82140ed613153a83ac7f15468a529924f3be142c Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 8 Jan 2025 17:06:22 +0530
Subject: [PATCH 1/6] dev: add pre-commit hooks

---
 .github/scripts/generate_pre_commit.py | 239 +++++++++++++
 .pre-commit-config.yaml                | 465 +++++++++++++++++++++++--
 build.gradle                           |   4 -
 settings.gradle                        |  36 ++
 4 files changed, 716 insertions(+), 28 deletions(-)
 create mode 100755 .github/scripts/generate_pre_commit.py

diff --git a/.github/scripts/generate_pre_commit.py b/.github/scripts/generate_pre_commit.py
new file mode 100755
index 00000000000000..054893c2822750
--- /dev/null
+++ b/.github/scripts/generate_pre_commit.py
@@ -0,0 +1,239 @@
+"""Generate pre-commit hooks for Java and Python projects.
+
+This script scans a repository for Java and Python projects and generates appropriate
+pre-commit hooks for linting and formatting.
+"""
+
+import os
+from dataclasses import dataclass
+from enum import Enum, auto
+from pathlib import Path
+
+import yaml
+
+
+class ProjectType(Enum):
+    """Types of projects supported for hook generation."""
+
+    JAVA = auto()
+    PYTHON = auto()
+
+
+@dataclass
+class Project:
+    """Represents a project found in the repository."""
+
+    path: str
+    type: ProjectType
+
+    @property
+    def gradle_path(self) -> str:
+        """Convert path to Gradle task format."""
+        return ":" + self.path.replace("/", ":")
+
+    @property
+    def project_id(self) -> str:
+        """Generate a unique identifier for the project."""
+        return self.path.replace("/", "-").replace(".", "-")
+
+
+class ProjectFinder:
+    """Find Java and Python projects in a repository."""
+
+    JAVA_PATTERNS = [
+        "plugins.hasPlugin('java')",
+        "apply plugin: 'java'",
+        "id 'java'",
+        "id 'java-library'",
+        "plugins.hasPlugin('java-library')",
+        "apply plugin: 'java-library'",
+        "plugins.hasPlugin('pegasus')",
+        "org.springframework.boot",
+    ]
+
+    EXCLUDED_DIRS = {".git", "build", "node_modules", ".tox", "venv"}
+    SOURCE_EXTENSIONS = {".java", ".kt", ".groovy"}
+
+    def __init__(self, root_dir: str):
+        self.root_path = Path(root_dir)
+
+    def find_all_projects(self) -> list[Project]:
+        """Find all Java and Python projects in the repository."""
+        java_projects = self._find_java_projects()
+        python_projects = self._find_python_projects()
+
+        all_projects = []
+        all_projects.extend(
+            Project(path=p, type=ProjectType.JAVA) for p in java_projects
+        )
+        all_projects.extend(
+            Project(path=p, type=ProjectType.PYTHON) for p in python_projects
+        )
+
+        return sorted(all_projects, key=lambda p: p.path)
+
+    def _find_java_projects(self) -> set[str]:
+        """Find all Java projects by checking build.gradle files."""
+        java_projects = set()
+
+        # Search both build.gradle and build.gradle.kts
+        for pattern in ["build.gradle", "build.gradle.kts"]:
+            for gradle_file in self.root_path.rglob(pattern):
+                if self._should_skip_directory(gradle_file.parent):
+                    continue
+
+                if self._is_java_project(gradle_file):
+                    java_projects.add(self._get_relative_path(gradle_file.parent))
+
+        return {
+            p
+            for p in java_projects
+            if "buildSrc" not in p and "spark-smoke-test" not in p
+        }
+
+    def _find_python_projects(self) -> set[str]:
+        """Find all Python projects by checking for setup.py or pyproject.toml."""
+        python_projects = set()
+
+        for file_name in ["setup.py", "pyproject.toml"]:
+            for path in self.root_path.rglob(file_name):
+                if self._should_skip_directory(path.parent):
+                    continue
+
+                rel_path = self._get_relative_path(path.parent)
+                if "examples" not in rel_path:
+                    python_projects.add(rel_path)
+
+        return python_projects
+
+    def _should_skip_directory(self, path: Path) -> bool:
+        """Check if directory should be skipped."""
+        return any(
+            part in self.EXCLUDED_DIRS or part.startswith(".") for part in path.parts
+        )
+
+    def _is_java_project(self, gradle_file: Path) -> bool:
+        """Check if a Gradle file represents a Java project."""
+        try:
+            content = gradle_file.read_text()
+            has_java_plugin = any(pattern in content for pattern in self.JAVA_PATTERNS)
+
+            if has_java_plugin:
+                # Verify presence of source files
+                return any(
+                    list(gradle_file.parent.rglob(f"*{ext}"))
+                    for ext in self.SOURCE_EXTENSIONS
+                )
+            return False
+
+        except Exception as e:
+            print(f"Warning: Error reading {gradle_file}: {e}")
+            return False
+
+    def _get_relative_path(self, path: Path) -> str:
+        """Get relative path from root, normalized with forward slashes."""
+        return str(path.relative_to(self.root_path)).replace("\\", "/")
+
+
+class HookGenerator:
+    """Generate pre-commit hooks for projects."""
+
+    def __init__(self, projects: list[Project]):
+        self.projects = projects
+
+    def generate_config(self) -> dict:
+        """Generate the complete pre-commit config."""
+        hooks = []
+
+        for project in self.projects:
+            if project.type == ProjectType.PYTHON:
+                hooks.append(self._generate_lint_fix_hook(project))
+            else:  # ProjectType.JAVA
+                hooks.append(self._generate_spotless_hook(project))
+
+        return {"repos": [{"repo": "local", "hooks": hooks}]}
+
+    def _generate_lint_fix_hook(self, project: Project) -> dict:
+        """Generate a lint-fix hook for Python projects."""
+        return {
+            "id": f"{project.project_id}-lint-fix",
+            "name": f"{project.path} Lint Fix",
+            "entry": f"./gradlew {project.gradle_path}:lintFix",
+            "language": "system",
+            "files": f"^{project.path}/.*\\.py$",
+            "pass_filenames": False,
+        }
+
+    def _generate_spotless_hook(self, project: Project) -> dict:
+        """Generate a spotless hook for Java projects."""
+        return {
+            "id": f"{project.project_id}-spotless",
+            "name": f"{project.path} Spotless Apply",
+            "entry": f"./gradlew {project.gradle_path}:spotlessApply",
+            "language": "system",
+            "files": f"^{project.path}/.*\\.java$",
+            "pass_filenames": False,
+        }
+
+
+class PrecommitDumper(yaml.Dumper):
+    """Custom YAML dumper that maintains proper indentation."""
+
+    def increase_indent(self, flow=False, *args, **kwargs):
+        return super().increase_indent(flow=flow, indentless=False)
+
+
+def write_yaml_with_spaces(file_path: str, data: dict):
+    """Write YAML file with extra spacing between hooks."""
+    with open(file_path, "w") as f:
+        yaml_str = yaml.dump(
+            data, Dumper=PrecommitDumper, sort_keys=False, default_flow_style=False
+        )
+
+        # Add extra newline between hooks
+        lines = yaml_str.split("\n")
+        result = []
+        in_hook = False
+
+        for line in lines:
+            if line.strip().startswith("- id:"):
+                if in_hook:  # If we were already in a hook, add extra newline
+                    result.append("")
+                in_hook = True
+            elif not line.strip() and in_hook:
+                in_hook = False
+
+            result.append(line)
+
+        f.write("\n".join(result))
+
+
+def main():
+    root_dir = os.path.abspath(os.curdir)
+
+    # Find projects
+    finder = ProjectFinder(root_dir)
+    projects = finder.find_all_projects()
+
+    # Print summary
+    print("Found projects:")
+    print("\nJava projects:")
+    for project in projects:
+        if project.type == ProjectType.JAVA:
+            print(f"  - {project.path}")
+
+    print("\nPython projects:")
+    for project in projects:
+        if project.type == ProjectType.PYTHON:
+            print(f"  - {project.path}")
+
+    # Generate and write config
+    generator = HookGenerator(projects)
+    config = generator.generate_config()
+    write_yaml_with_spaces(".pre-commit-config.yaml", config)
+
+    print("\nGenerated .pre-commit-config.yaml")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 898e3d262b3941..16b56d44c70bf3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,26 +1,443 @@
-exclude: ^$
-files: ^(docs/|docs-website/|metadata-ingestion/)
 repos:
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1
+  - repo: local
     hooks:
-      - id: isort
-  - repo: https://github.com/ambv/black
-    rev: 23.1.0
-    hooks:
-      - id: black
-  - repo: https://github.com/myint/autoflake
-    rev: v1.4
-    hooks:
-      - id: autoflake
-        args:
-          - --in-place
-          - --remove-unused-variables
-          - --remove-all-unused-imports
-          - --expand-star-imports
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.0.0-alpha.6" # Use the sha or tag you want to point at
-    hooks:
-      - id: prettier
-        args:
-          - --write
\ No newline at end of file
+      - id: --spotless
+        name: . Spotless Apply
+        entry: ./gradlew :.:spotlessApply
+        language: system
+        files: ^./.*\.java$
+        pass_filenames: false
+
+      - id: datahub-graphql-core-spotless
+        name: datahub-graphql-core Spotless Apply
+        entry: ./gradlew :datahub-graphql-core:spotlessApply
+        language: system
+        files: ^datahub-graphql-core/.*\.java$
+        pass_filenames: false
+
+      - id: datahub-upgrade-spotless
+        name: datahub-upgrade Spotless Apply
+        entry: ./gradlew :datahub-upgrade:spotlessApply
+        language: system
+        files: ^datahub-upgrade/.*\.java$
+        pass_filenames: false
+
+      - id: entity-registry-spotless
+        name: entity-registry Spotless Apply
+        entry: ./gradlew :entity-registry:spotlessApply
+        language: system
+        files: ^entity-registry/.*\.java$
+        pass_filenames: false
+
+      - id: ingestion-scheduler-spotless
+        name: ingestion-scheduler Spotless Apply
+        entry: ./gradlew :ingestion-scheduler:spotlessApply
+        language: system
+        files: ^ingestion-scheduler/.*\.java$
+        pass_filenames: false
+
+      - id: li-utils-spotless
+        name: li-utils Spotless Apply
+        entry: ./gradlew :li-utils:spotlessApply
+        language: system
+        files: ^li-utils/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-auth-auth-api-spotless
+        name: metadata-auth/auth-api Spotless Apply
+        entry: ./gradlew :metadata-auth:auth-api:spotlessApply
+        language: system
+        files: ^metadata-auth/auth-api/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-dao-impl-kafka-producer-spotless
+        name: metadata-dao-impl/kafka-producer Spotless Apply
+        entry: ./gradlew :metadata-dao-impl:kafka-producer:spotlessApply
+        language: system
+        files: ^metadata-dao-impl/kafka-producer/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-events-mxe-avro-spotless
+        name: metadata-events/mxe-avro Spotless Apply
+        entry: ./gradlew :metadata-events:mxe-avro:spotlessApply
+        language: system
+        files: ^metadata-events/mxe-avro/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-events-mxe-registration-spotless
+        name: metadata-events/mxe-registration Spotless Apply
+        entry: ./gradlew :metadata-events:mxe-registration:spotlessApply
+        language: system
+        files: ^metadata-events/mxe-registration/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-events-mxe-schemas-spotless
+        name: metadata-events/mxe-schemas Spotless Apply
+        entry: ./gradlew :metadata-events:mxe-schemas:spotlessApply
+        language: system
+        files: ^metadata-events/mxe-schemas/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-events-mxe-utils-avro-spotless
+        name: metadata-events/mxe-utils-avro Spotless Apply
+        entry: ./gradlew :metadata-events:mxe-utils-avro:spotlessApply
+        language: system
+        files: ^metadata-events/mxe-utils-avro/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-ingestion-lint-fix
+        name: metadata-ingestion Lint Fix
+        entry: ./gradlew :metadata-ingestion:lintFix
+        language: system
+        files: ^metadata-ingestion/.*\.py$
+        pass_filenames: false
+
+      - id: metadata-ingestion-modules-airflow-plugin-lint-fix
+        name: metadata-ingestion-modules/airflow-plugin Lint Fix
+        entry: ./gradlew :metadata-ingestion-modules:airflow-plugin:lintFix
+        language: system
+        files: ^metadata-ingestion-modules/airflow-plugin/.*\.py$
+        pass_filenames: false
+
+      - id: metadata-ingestion-modules-dagster-plugin-lint-fix
+        name: metadata-ingestion-modules/dagster-plugin Lint Fix
+        entry: ./gradlew :metadata-ingestion-modules:dagster-plugin:lintFix
+        language: system
+        files: ^metadata-ingestion-modules/dagster-plugin/.*\.py$
+        pass_filenames: false
+
+      - id: metadata-ingestion-modules-gx-plugin-lint-fix
+        name: metadata-ingestion-modules/gx-plugin Lint Fix
+        entry: ./gradlew :metadata-ingestion-modules:gx-plugin:lintFix
+        language: system
+        files: ^metadata-ingestion-modules/gx-plugin/.*\.py$
+        pass_filenames: false
+
+      - id: metadata-ingestion-modules-prefect-plugin-lint-fix
+        name: metadata-ingestion-modules/prefect-plugin Lint Fix
+        entry: ./gradlew :metadata-ingestion-modules:prefect-plugin:lintFix
+        language: system
+        files: ^metadata-ingestion-modules/prefect-plugin/.*\.py$
+        pass_filenames: false
+
+      - id: metadata-integration-java-acryl-spark-lineage-spotless
+        name: metadata-integration/java/acryl-spark-lineage Spotless Apply
+        entry: ./gradlew :metadata-integration:java:acryl-spark-lineage:spotlessApply
+        language: system
+        files: ^metadata-integration/java/acryl-spark-lineage/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-datahub-client-spotless
+        name: metadata-integration/java/datahub-client Spotless Apply
+        entry: ./gradlew :metadata-integration:java:datahub-client:spotlessApply
+        language: system
+        files: ^metadata-integration/java/datahub-client/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-datahub-event-spotless
+        name: metadata-integration/java/datahub-event Spotless Apply
+        entry: ./gradlew :metadata-integration:java:datahub-event:spotlessApply
+        language: system
+        files: ^metadata-integration/java/datahub-event/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-datahub-protobuf-spotless
+        name: metadata-integration/java/datahub-protobuf Spotless Apply
+        entry: ./gradlew :metadata-integration:java:datahub-protobuf:spotlessApply
+        language: system
+        files: ^metadata-integration/java/datahub-protobuf/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-datahub-schematron-cli-spotless
+        name: metadata-integration/java/datahub-schematron/cli Spotless Apply
+        entry: ./gradlew :metadata-integration:java:datahub-schematron:cli:spotlessApply
+        language: system
+        files: ^metadata-integration/java/datahub-schematron/cli/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-datahub-schematron-lib-spotless
+        name: metadata-integration/java/datahub-schematron/lib Spotless Apply
+        entry: ./gradlew :metadata-integration:java:datahub-schematron:lib:spotlessApply
+        language: system
+        files: ^metadata-integration/java/datahub-schematron/lib/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-examples-spotless
+        name: metadata-integration/java/examples Spotless Apply
+        entry: ./gradlew :metadata-integration:java:examples:spotlessApply
+        language: system
+        files: ^metadata-integration/java/examples/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-openlineage-converter-spotless
+        name: metadata-integration/java/openlineage-converter Spotless Apply
+        entry: ./gradlew :metadata-integration:java:openlineage-converter:spotlessApply
+        language: system
+        files: ^metadata-integration/java/openlineage-converter/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-integration-java-spark-lineage-legacy-spotless
+        name: metadata-integration/java/spark-lineage-legacy Spotless Apply
+        entry: ./gradlew :metadata-integration:java:spark-lineage-legacy:spotlessApply
+        language: system
+        files: ^metadata-integration/java/spark-lineage-legacy/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-io-spotless
+        name: metadata-io Spotless Apply
+        entry: ./gradlew :metadata-io:spotlessApply
+        language: system
+        files: ^metadata-io/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-io-metadata-io-api-spotless
+        name: metadata-io/metadata-io-api Spotless Apply
+        entry: ./gradlew :metadata-io:metadata-io-api:spotlessApply
+        language: system
+        files: ^metadata-io/metadata-io-api/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-jobs-common-spotless
+        name: metadata-jobs/common Spotless Apply
+        entry: ./gradlew :metadata-jobs:common:spotlessApply
+        language: system
+        files: ^metadata-jobs/common/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-jobs-mae-consumer-spotless
+        name: metadata-jobs/mae-consumer Spotless Apply
+        entry: ./gradlew :metadata-jobs:mae-consumer:spotlessApply
+        language: system
+        files: ^metadata-jobs/mae-consumer/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-jobs-mae-consumer-job-spotless
+        name: metadata-jobs/mae-consumer-job Spotless Apply
+        entry: ./gradlew :metadata-jobs:mae-consumer-job:spotlessApply
+        language: system
+        files: ^metadata-jobs/mae-consumer-job/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-jobs-mce-consumer-spotless
+        name: metadata-jobs/mce-consumer Spotless Apply
+        entry: ./gradlew :metadata-jobs:mce-consumer:spotlessApply
+        language: system
+        files: ^metadata-jobs/mce-consumer/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-jobs-mce-consumer-job-spotless
+        name: metadata-jobs/mce-consumer-job Spotless Apply
+        entry: ./gradlew :metadata-jobs:mce-consumer-job:spotlessApply
+        language: system
+        files: ^metadata-jobs/mce-consumer-job/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-jobs-pe-consumer-spotless
+        name: metadata-jobs/pe-consumer Spotless Apply
+        entry: ./gradlew :metadata-jobs:pe-consumer:spotlessApply
+        language: system
+        files: ^metadata-jobs/pe-consumer/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-models-spotless
+        name: metadata-models Spotless Apply
+        entry: ./gradlew :metadata-models:spotlessApply
+        language: system
+        files: ^metadata-models/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-models-custom-spotless
+        name: metadata-models-custom Spotless Apply
+        entry: ./gradlew :metadata-models-custom:spotlessApply
+        language: system
+        files: ^metadata-models-custom/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-models-validator-spotless
+        name: metadata-models-validator Spotless Apply
+        entry: ./gradlew :metadata-models-validator:spotlessApply
+        language: system
+        files: ^metadata-models-validator/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-operation-context-spotless
+        name: metadata-operation-context Spotless Apply
+        entry: ./gradlew :metadata-operation-context:spotlessApply
+        language: system
+        files: ^metadata-operation-context/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-auth-config-spotless
+        name: metadata-service/auth-config Spotless Apply
+        entry: ./gradlew :metadata-service:auth-config:spotlessApply
+        language: system
+        files: ^metadata-service/auth-config/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-auth-filter-spotless
+        name: metadata-service/auth-filter Spotless Apply
+        entry: ./gradlew :metadata-service:auth-filter:spotlessApply
+        language: system
+        files: ^metadata-service/auth-filter/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-auth-impl-spotless
+        name: metadata-service/auth-impl Spotless Apply
+        entry: ./gradlew :metadata-service:auth-impl:spotlessApply
+        language: system
+        files: ^metadata-service/auth-impl/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-auth-servlet-impl-spotless
+        name: metadata-service/auth-servlet-impl Spotless Apply
+        entry: ./gradlew :metadata-service:auth-servlet-impl:spotlessApply
+        language: system
+        files: ^metadata-service/auth-servlet-impl/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-configuration-spotless
+        name: metadata-service/configuration Spotless Apply
+        entry: ./gradlew :metadata-service:configuration:spotlessApply
+        language: system
+        files: ^metadata-service/configuration/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-factories-spotless
+        name: metadata-service/factories Spotless Apply
+        entry: ./gradlew :metadata-service:factories:spotlessApply
+        language: system
+        files: ^metadata-service/factories/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-graphql-servlet-impl-spotless
+        name: metadata-service/graphql-servlet-impl Spotless Apply
+        entry: ./gradlew :metadata-service:graphql-servlet-impl:spotlessApply
+        language: system
+        files: ^metadata-service/graphql-servlet-impl/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-openapi-analytics-servlet-spotless
+        name: metadata-service/openapi-analytics-servlet Spotless Apply
+        entry: ./gradlew :metadata-service:openapi-analytics-servlet:spotlessApply
+        language: system
+        files: ^metadata-service/openapi-analytics-servlet/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-openapi-entity-servlet-spotless
+        name: metadata-service/openapi-entity-servlet Spotless Apply
+        entry: ./gradlew :metadata-service:openapi-entity-servlet:spotlessApply
+        language: system
+        files: ^metadata-service/openapi-entity-servlet/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-openapi-entity-servlet-generators-spotless
+        name: metadata-service/openapi-entity-servlet/generators Spotless Apply
+        entry: ./gradlew :metadata-service:openapi-entity-servlet:generators:spotlessApply
+        language: system
+        files: ^metadata-service/openapi-entity-servlet/generators/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-openapi-servlet-spotless
+        name: metadata-service/openapi-servlet Spotless Apply
+        entry: ./gradlew :metadata-service:openapi-servlet:spotlessApply
+        language: system
+        files: ^metadata-service/openapi-servlet/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-openapi-servlet-models-spotless
+        name: metadata-service/openapi-servlet/models Spotless Apply
+        entry: ./gradlew :metadata-service:openapi-servlet:models:spotlessApply
+        language: system
+        files: ^metadata-service/openapi-servlet/models/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-plugin-spotless
+        name: metadata-service/plugin Spotless Apply
+        entry: ./gradlew :metadata-service:plugin:spotlessApply
+        language: system
+        files: ^metadata-service/plugin/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-plugin-src-test-sample-test-plugins-spotless
+        name: metadata-service/plugin/src/test/sample-test-plugins Spotless Apply
+        entry: ./gradlew :metadata-service:plugin:src:test:sample-test-plugins:spotlessApply
+        language: system
+        files: ^metadata-service/plugin/src/test/sample-test-plugins/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-restli-client-spotless
+        name: metadata-service/restli-client Spotless Apply
+        entry: ./gradlew :metadata-service:restli-client:spotlessApply
+        language: system
+        files: ^metadata-service/restli-client/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-restli-client-api-spotless
+        name: metadata-service/restli-client-api Spotless Apply
+        entry: ./gradlew :metadata-service:restli-client-api:spotlessApply
+        language: system
+        files: ^metadata-service/restli-client-api/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-restli-servlet-impl-spotless
+        name: metadata-service/restli-servlet-impl Spotless Apply
+        entry: ./gradlew :metadata-service:restli-servlet-impl:spotlessApply
+        language: system
+        files: ^metadata-service/restli-servlet-impl/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-schema-registry-api-spotless
+        name: metadata-service/schema-registry-api Spotless Apply
+        entry: ./gradlew :metadata-service:schema-registry-api:spotlessApply
+        language: system
+        files: ^metadata-service/schema-registry-api/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-schema-registry-servlet-spotless
+        name: metadata-service/schema-registry-servlet Spotless Apply
+        entry: ./gradlew :metadata-service:schema-registry-servlet:spotlessApply
+        language: system
+        files: ^metadata-service/schema-registry-servlet/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-services-spotless
+        name: metadata-service/services Spotless Apply
+        entry: ./gradlew :metadata-service:services:spotlessApply
+        language: system
+        files: ^metadata-service/services/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-servlet-spotless
+        name: metadata-service/servlet Spotless Apply
+        entry: ./gradlew :metadata-service:servlet:spotlessApply
+        language: system
+        files: ^metadata-service/servlet/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-utils-spotless
+        name: metadata-utils Spotless Apply
+        entry: ./gradlew :metadata-utils:spotlessApply
+        language: system
+        files: ^metadata-utils/.*\.java$
+        pass_filenames: false
+
+      - id: mock-entity-registry-spotless
+        name: mock-entity-registry Spotless Apply
+        entry: ./gradlew :mock-entity-registry:spotlessApply
+        language: system
+        files: ^mock-entity-registry/.*\.java$
+        pass_filenames: false
+
+      - id: smoke-test-lint-fix
+        name: smoke-test Lint Fix
+        entry: ./gradlew :smoke-test:lintFix
+        language: system
+        files: ^smoke-test/.*\.py$
+        pass_filenames: false
+
+      - id: test-models-spotless
+        name: test-models Spotless Apply
+        entry: ./gradlew :test-models:spotlessApply
+        language: system
+        files: ^test-models/.*\.java$
+        pass_filenames: false
diff --git a/build.gradle b/build.gradle
index 8929b4e644972c..3c36feadc5f4bb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -474,10 +474,6 @@ subprojects {
       if (compileJavaTask != null) {
         spotlessJavaTask.dependsOn compileJavaTask
       }
-      // TODO - Do not run this in CI. How?
-      // tasks.withType(JavaCompile) {
-      //   finalizedBy(tasks.findByName('spotlessApply'))
-      // }
     }
   }
 
diff --git a/settings.gradle b/settings.gradle
index b0c2c707d566c0..cb234a50360d4b 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -78,3 +78,39 @@ include ':metadata-operation-context'
 include ':metadata-service:openapi-servlet:models'
 include ':metadata-integration:java:datahub-schematron:lib'
 include ':metadata-integration:java:datahub-schematron:cli'
+
+
+// Add this to your build.gradle file
+
+def installPreCommitHooks() {
+    // Check if pre-commit is installed
+    def preCommitInstalled = false
+    try {
+        def process = "pre-commit --version".execute()
+        process.waitFor()
+        preCommitInstalled = (process.exitValue() == 0)
+    } catch (Exception e) {
+        println "pre-commit is not installed"
+    }
+
+    if (!preCommitInstalled) {
+        println "Installing pre-commit..."
+        def installProcess = "pip install pre-commit".execute()
+        installProcess.waitFor()
+        if (installProcess.exitValue() != 0) {
+            throw new GradleException("Failed to install pre-commit. Error: ${installProcess.err.text}")
+        }
+        println "pre-commit installed successfully"
+    }
+
+    // Install the pre-commit hooks
+    println "Installing pre-commit hooks..."
+    def installHooksProcess = "pre-commit install".execute()
+    installHooksProcess.waitFor()
+    if (installHooksProcess.exitValue() != 0) {
+        throw new GradleException("Failed to install pre-commit hooks. Error: ${installHooksProcess.err.text}")
+    }
+    println "pre-commit hooks installed successfully"
+}
+
+installPreCommitHooks()
\ No newline at end of file

From 435edb4d501cd692b0a486609084ffab2e959824 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 8 Jan 2025 17:07:27 +0530
Subject: [PATCH 2/6] cleanup

---
 settings.gradle | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/settings.gradle b/settings.gradle
index cb234a50360d4b..780a88eb0f37dc 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -79,11 +79,7 @@ include ':metadata-service:openapi-servlet:models'
 include ':metadata-integration:java:datahub-schematron:lib'
 include ':metadata-integration:java:datahub-schematron:cli'
 
-
-// Add this to your build.gradle file
-
 def installPreCommitHooks() {
-    // Check if pre-commit is installed
     def preCommitInstalled = false
     try {
         def process = "pre-commit --version".execute()
@@ -103,7 +99,6 @@ def installPreCommitHooks() {
         println "pre-commit installed successfully"
     }
 
-    // Install the pre-commit hooks
     println "Installing pre-commit hooks..."
     def installHooksProcess = "pre-commit install".execute()
     installHooksProcess.waitFor()

From f52d55e66aab8c29bb66d86a32caf04a8b2abd54 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 8 Jan 2025 17:10:53 +0530
Subject: [PATCH 3/6] remove redundant

---
 .github/scripts/generate_pre_commit.py |  2 -
 .pre-commit-config.yaml                | 63 --------------------------
 2 files changed, 65 deletions(-)

diff --git a/.github/scripts/generate_pre_commit.py b/.github/scripts/generate_pre_commit.py
index 054893c2822750..e95c680e904e46 100755
--- a/.github/scripts/generate_pre_commit.py
+++ b/.github/scripts/generate_pre_commit.py
@@ -161,7 +161,6 @@ def _generate_lint_fix_hook(self, project: Project) -> dict:
             "entry": f"./gradlew {project.gradle_path}:lintFix",
             "language": "system",
             "files": f"^{project.path}/.*\\.py$",
-            "pass_filenames": False,
         }
 
     def _generate_spotless_hook(self, project: Project) -> dict:
@@ -172,7 +171,6 @@ def _generate_spotless_hook(self, project: Project) -> dict:
             "entry": f"./gradlew {project.gradle_path}:spotlessApply",
             "language": "system",
             "files": f"^{project.path}/.*\\.java$",
-            "pass_filenames": False,
         }
 
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 16b56d44c70bf3..754eb0e936b3e9 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,438 +6,375 @@ repos:
         entry: ./gradlew :.:spotlessApply
         language: system
         files: ^./.*\.java$
-        pass_filenames: false
 
       - id: datahub-graphql-core-spotless
         name: datahub-graphql-core Spotless Apply
         entry: ./gradlew :datahub-graphql-core:spotlessApply
         language: system
         files: ^datahub-graphql-core/.*\.java$
-        pass_filenames: false
 
       - id: datahub-upgrade-spotless
         name: datahub-upgrade Spotless Apply
         entry: ./gradlew :datahub-upgrade:spotlessApply
         language: system
         files: ^datahub-upgrade/.*\.java$
-        pass_filenames: false
 
       - id: entity-registry-spotless
         name: entity-registry Spotless Apply
         entry: ./gradlew :entity-registry:spotlessApply
         language: system
         files: ^entity-registry/.*\.java$
-        pass_filenames: false
 
       - id: ingestion-scheduler-spotless
         name: ingestion-scheduler Spotless Apply
         entry: ./gradlew :ingestion-scheduler:spotlessApply
         language: system
         files: ^ingestion-scheduler/.*\.java$
-        pass_filenames: false
 
       - id: li-utils-spotless
         name: li-utils Spotless Apply
         entry: ./gradlew :li-utils:spotlessApply
         language: system
         files: ^li-utils/.*\.java$
-        pass_filenames: false
 
       - id: metadata-auth-auth-api-spotless
         name: metadata-auth/auth-api Spotless Apply
         entry: ./gradlew :metadata-auth:auth-api:spotlessApply
         language: system
         files: ^metadata-auth/auth-api/.*\.java$
-        pass_filenames: false
 
       - id: metadata-dao-impl-kafka-producer-spotless
         name: metadata-dao-impl/kafka-producer Spotless Apply
         entry: ./gradlew :metadata-dao-impl:kafka-producer:spotlessApply
         language: system
         files: ^metadata-dao-impl/kafka-producer/.*\.java$
-        pass_filenames: false
 
       - id: metadata-events-mxe-avro-spotless
         name: metadata-events/mxe-avro Spotless Apply
         entry: ./gradlew :metadata-events:mxe-avro:spotlessApply
         language: system
         files: ^metadata-events/mxe-avro/.*\.java$
-        pass_filenames: false
 
       - id: metadata-events-mxe-registration-spotless
         name: metadata-events/mxe-registration Spotless Apply
         entry: ./gradlew :metadata-events:mxe-registration:spotlessApply
         language: system
         files: ^metadata-events/mxe-registration/.*\.java$
-        pass_filenames: false
 
       - id: metadata-events-mxe-schemas-spotless
         name: metadata-events/mxe-schemas Spotless Apply
         entry: ./gradlew :metadata-events:mxe-schemas:spotlessApply
         language: system
         files: ^metadata-events/mxe-schemas/.*\.java$
-        pass_filenames: false
 
       - id: metadata-events-mxe-utils-avro-spotless
         name: metadata-events/mxe-utils-avro Spotless Apply
         entry: ./gradlew :metadata-events:mxe-utils-avro:spotlessApply
         language: system
         files: ^metadata-events/mxe-utils-avro/.*\.java$
-        pass_filenames: false
 
       - id: metadata-ingestion-lint-fix
         name: metadata-ingestion Lint Fix
         entry: ./gradlew :metadata-ingestion:lintFix
         language: system
         files: ^metadata-ingestion/.*\.py$
-        pass_filenames: false
 
       - id: metadata-ingestion-modules-airflow-plugin-lint-fix
         name: metadata-ingestion-modules/airflow-plugin Lint Fix
         entry: ./gradlew :metadata-ingestion-modules:airflow-plugin:lintFix
         language: system
         files: ^metadata-ingestion-modules/airflow-plugin/.*\.py$
-        pass_filenames: false
 
       - id: metadata-ingestion-modules-dagster-plugin-lint-fix
         name: metadata-ingestion-modules/dagster-plugin Lint Fix
         entry: ./gradlew :metadata-ingestion-modules:dagster-plugin:lintFix
         language: system
         files: ^metadata-ingestion-modules/dagster-plugin/.*\.py$
-        pass_filenames: false
 
       - id: metadata-ingestion-modules-gx-plugin-lint-fix
         name: metadata-ingestion-modules/gx-plugin Lint Fix
         entry: ./gradlew :metadata-ingestion-modules:gx-plugin:lintFix
         language: system
         files: ^metadata-ingestion-modules/gx-plugin/.*\.py$
-        pass_filenames: false
 
       - id: metadata-ingestion-modules-prefect-plugin-lint-fix
         name: metadata-ingestion-modules/prefect-plugin Lint Fix
         entry: ./gradlew :metadata-ingestion-modules:prefect-plugin:lintFix
         language: system
         files: ^metadata-ingestion-modules/prefect-plugin/.*\.py$
-        pass_filenames: false
 
       - id: metadata-integration-java-acryl-spark-lineage-spotless
         name: metadata-integration/java/acryl-spark-lineage Spotless Apply
         entry: ./gradlew :metadata-integration:java:acryl-spark-lineage:spotlessApply
         language: system
         files: ^metadata-integration/java/acryl-spark-lineage/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-datahub-client-spotless
         name: metadata-integration/java/datahub-client Spotless Apply
         entry: ./gradlew :metadata-integration:java:datahub-client:spotlessApply
         language: system
         files: ^metadata-integration/java/datahub-client/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-datahub-event-spotless
         name: metadata-integration/java/datahub-event Spotless Apply
         entry: ./gradlew :metadata-integration:java:datahub-event:spotlessApply
         language: system
         files: ^metadata-integration/java/datahub-event/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-datahub-protobuf-spotless
         name: metadata-integration/java/datahub-protobuf Spotless Apply
         entry: ./gradlew :metadata-integration:java:datahub-protobuf:spotlessApply
         language: system
         files: ^metadata-integration/java/datahub-protobuf/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-datahub-schematron-cli-spotless
         name: metadata-integration/java/datahub-schematron/cli Spotless Apply
         entry: ./gradlew :metadata-integration:java:datahub-schematron:cli:spotlessApply
         language: system
         files: ^metadata-integration/java/datahub-schematron/cli/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-datahub-schematron-lib-spotless
         name: metadata-integration/java/datahub-schematron/lib Spotless Apply
         entry: ./gradlew :metadata-integration:java:datahub-schematron:lib:spotlessApply
         language: system
         files: ^metadata-integration/java/datahub-schematron/lib/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-examples-spotless
         name: metadata-integration/java/examples Spotless Apply
         entry: ./gradlew :metadata-integration:java:examples:spotlessApply
         language: system
         files: ^metadata-integration/java/examples/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-openlineage-converter-spotless
         name: metadata-integration/java/openlineage-converter Spotless Apply
         entry: ./gradlew :metadata-integration:java:openlineage-converter:spotlessApply
         language: system
         files: ^metadata-integration/java/openlineage-converter/.*\.java$
-        pass_filenames: false
 
       - id: metadata-integration-java-spark-lineage-legacy-spotless
         name: metadata-integration/java/spark-lineage-legacy Spotless Apply
         entry: ./gradlew :metadata-integration:java:spark-lineage-legacy:spotlessApply
         language: system
         files: ^metadata-integration/java/spark-lineage-legacy/.*\.java$
-        pass_filenames: false
 
       - id: metadata-io-spotless
         name: metadata-io Spotless Apply
         entry: ./gradlew :metadata-io:spotlessApply
         language: system
         files: ^metadata-io/.*\.java$
-        pass_filenames: false
 
       - id: metadata-io-metadata-io-api-spotless
         name: metadata-io/metadata-io-api Spotless Apply
         entry: ./gradlew :metadata-io:metadata-io-api:spotlessApply
         language: system
         files: ^metadata-io/metadata-io-api/.*\.java$
-        pass_filenames: false
 
       - id: metadata-jobs-common-spotless
         name: metadata-jobs/common Spotless Apply
         entry: ./gradlew :metadata-jobs:common:spotlessApply
         language: system
         files: ^metadata-jobs/common/.*\.java$
-        pass_filenames: false
 
       - id: metadata-jobs-mae-consumer-spotless
         name: metadata-jobs/mae-consumer Spotless Apply
         entry: ./gradlew :metadata-jobs:mae-consumer:spotlessApply
         language: system
         files: ^metadata-jobs/mae-consumer/.*\.java$
-        pass_filenames: false
 
       - id: metadata-jobs-mae-consumer-job-spotless
         name: metadata-jobs/mae-consumer-job Spotless Apply
         entry: ./gradlew :metadata-jobs:mae-consumer-job:spotlessApply
         language: system
         files: ^metadata-jobs/mae-consumer-job/.*\.java$
-        pass_filenames: false
 
       - id: metadata-jobs-mce-consumer-spotless
         name: metadata-jobs/mce-consumer Spotless Apply
         entry: ./gradlew :metadata-jobs:mce-consumer:spotlessApply
         language: system
         files: ^metadata-jobs/mce-consumer/.*\.java$
-        pass_filenames: false
 
       - id: metadata-jobs-mce-consumer-job-spotless
         name: metadata-jobs/mce-consumer-job Spotless Apply
         entry: ./gradlew :metadata-jobs:mce-consumer-job:spotlessApply
         language: system
         files: ^metadata-jobs/mce-consumer-job/.*\.java$
-        pass_filenames: false
 
       - id: metadata-jobs-pe-consumer-spotless
         name: metadata-jobs/pe-consumer Spotless Apply
         entry: ./gradlew :metadata-jobs:pe-consumer:spotlessApply
         language: system
         files: ^metadata-jobs/pe-consumer/.*\.java$
-        pass_filenames: false
 
       - id: metadata-models-spotless
         name: metadata-models Spotless Apply
         entry: ./gradlew :metadata-models:spotlessApply
         language: system
         files: ^metadata-models/.*\.java$
-        pass_filenames: false
 
       - id: metadata-models-custom-spotless
         name: metadata-models-custom Spotless Apply
         entry: ./gradlew :metadata-models-custom:spotlessApply
         language: system
         files: ^metadata-models-custom/.*\.java$
-        pass_filenames: false
 
       - id: metadata-models-validator-spotless
         name: metadata-models-validator Spotless Apply
         entry: ./gradlew :metadata-models-validator:spotlessApply
         language: system
         files: ^metadata-models-validator/.*\.java$
-        pass_filenames: false
 
       - id: metadata-operation-context-spotless
         name: metadata-operation-context Spotless Apply
         entry: ./gradlew :metadata-operation-context:spotlessApply
         language: system
         files: ^metadata-operation-context/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-auth-config-spotless
         name: metadata-service/auth-config Spotless Apply
         entry: ./gradlew :metadata-service:auth-config:spotlessApply
         language: system
         files: ^metadata-service/auth-config/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-auth-filter-spotless
         name: metadata-service/auth-filter Spotless Apply
         entry: ./gradlew :metadata-service:auth-filter:spotlessApply
         language: system
         files: ^metadata-service/auth-filter/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-auth-impl-spotless
         name: metadata-service/auth-impl Spotless Apply
         entry: ./gradlew :metadata-service:auth-impl:spotlessApply
         language: system
         files: ^metadata-service/auth-impl/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-auth-servlet-impl-spotless
         name: metadata-service/auth-servlet-impl Spotless Apply
         entry: ./gradlew :metadata-service:auth-servlet-impl:spotlessApply
         language: system
         files: ^metadata-service/auth-servlet-impl/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-configuration-spotless
         name: metadata-service/configuration Spotless Apply
         entry: ./gradlew :metadata-service:configuration:spotlessApply
         language: system
         files: ^metadata-service/configuration/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-factories-spotless
         name: metadata-service/factories Spotless Apply
         entry: ./gradlew :metadata-service:factories:spotlessApply
         language: system
         files: ^metadata-service/factories/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-graphql-servlet-impl-spotless
         name: metadata-service/graphql-servlet-impl Spotless Apply
         entry: ./gradlew :metadata-service:graphql-servlet-impl:spotlessApply
         language: system
         files: ^metadata-service/graphql-servlet-impl/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-openapi-analytics-servlet-spotless
         name: metadata-service/openapi-analytics-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-analytics-servlet:spotlessApply
         language: system
         files: ^metadata-service/openapi-analytics-servlet/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-openapi-entity-servlet-spotless
         name: metadata-service/openapi-entity-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-entity-servlet:spotlessApply
         language: system
         files: ^metadata-service/openapi-entity-servlet/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-openapi-entity-servlet-generators-spotless
         name: metadata-service/openapi-entity-servlet/generators Spotless Apply
         entry: ./gradlew :metadata-service:openapi-entity-servlet:generators:spotlessApply
         language: system
         files: ^metadata-service/openapi-entity-servlet/generators/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-openapi-servlet-spotless
         name: metadata-service/openapi-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-servlet:spotlessApply
         language: system
         files: ^metadata-service/openapi-servlet/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-openapi-servlet-models-spotless
         name: metadata-service/openapi-servlet/models Spotless Apply
         entry: ./gradlew :metadata-service:openapi-servlet:models:spotlessApply
         language: system
         files: ^metadata-service/openapi-servlet/models/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-plugin-spotless
         name: metadata-service/plugin Spotless Apply
         entry: ./gradlew :metadata-service:plugin:spotlessApply
         language: system
         files: ^metadata-service/plugin/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-plugin-src-test-sample-test-plugins-spotless
         name: metadata-service/plugin/src/test/sample-test-plugins Spotless Apply
         entry: ./gradlew :metadata-service:plugin:src:test:sample-test-plugins:spotlessApply
         language: system
         files: ^metadata-service/plugin/src/test/sample-test-plugins/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-restli-client-spotless
         name: metadata-service/restli-client Spotless Apply
         entry: ./gradlew :metadata-service:restli-client:spotlessApply
         language: system
         files: ^metadata-service/restli-client/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-restli-client-api-spotless
         name: metadata-service/restli-client-api Spotless Apply
         entry: ./gradlew :metadata-service:restli-client-api:spotlessApply
         language: system
         files: ^metadata-service/restli-client-api/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-restli-servlet-impl-spotless
         name: metadata-service/restli-servlet-impl Spotless Apply
         entry: ./gradlew :metadata-service:restli-servlet-impl:spotlessApply
         language: system
         files: ^metadata-service/restli-servlet-impl/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-schema-registry-api-spotless
         name: metadata-service/schema-registry-api Spotless Apply
         entry: ./gradlew :metadata-service:schema-registry-api:spotlessApply
         language: system
         files: ^metadata-service/schema-registry-api/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-schema-registry-servlet-spotless
         name: metadata-service/schema-registry-servlet Spotless Apply
         entry: ./gradlew :metadata-service:schema-registry-servlet:spotlessApply
         language: system
         files: ^metadata-service/schema-registry-servlet/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-services-spotless
         name: metadata-service/services Spotless Apply
         entry: ./gradlew :metadata-service:services:spotlessApply
         language: system
         files: ^metadata-service/services/.*\.java$
-        pass_filenames: false
 
       - id: metadata-service-servlet-spotless
         name: metadata-service/servlet Spotless Apply
         entry: ./gradlew :metadata-service:servlet:spotlessApply
         language: system
         files: ^metadata-service/servlet/.*\.java$
-        pass_filenames: false
 
       - id: metadata-utils-spotless
         name: metadata-utils Spotless Apply
         entry: ./gradlew :metadata-utils:spotlessApply
         language: system
         files: ^metadata-utils/.*\.java$
-        pass_filenames: false
 
       - id: mock-entity-registry-spotless
         name: mock-entity-registry Spotless Apply
         entry: ./gradlew :mock-entity-registry:spotlessApply
         language: system
         files: ^mock-entity-registry/.*\.java$
-        pass_filenames: false
 
       - id: smoke-test-lint-fix
         name: smoke-test Lint Fix
         entry: ./gradlew :smoke-test:lintFix
         language: system
         files: ^smoke-test/.*\.py$
-        pass_filenames: false
 
       - id: test-models-spotless
         name: test-models Spotless Apply
         entry: ./gradlew :test-models:spotlessApply
         language: system
         files: ^test-models/.*\.java$
-        pass_filenames: false

From 185afb6ca55f665c3f16cba854a97ae51b4b14f7 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 8 Jan 2025 17:12:56 +0530
Subject: [PATCH 4/6] remove extra spotless

---
 .github/scripts/generate_pre_commit.py | 2 +-
 .pre-commit-config.yaml                | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/scripts/generate_pre_commit.py b/.github/scripts/generate_pre_commit.py
index e95c680e904e46..dd2954e775d3b2 100755
--- a/.github/scripts/generate_pre_commit.py
+++ b/.github/scripts/generate_pre_commit.py
@@ -88,7 +88,7 @@ def _find_java_projects(self) -> set[str]:
         return {
             p
             for p in java_projects
-            if "buildSrc" not in p and "spark-smoke-test" not in p
+            if "buildSrc" not in p and "spark-smoke-test" not in p and p != "."
         }
 
     def _find_python_projects(self) -> set[str]:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 754eb0e936b3e9..e155f91f107394 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,12 +1,6 @@
 repos:
   - repo: local
     hooks:
-      - id: --spotless
-        name: . Spotless Apply
-        entry: ./gradlew :.:spotlessApply
-        language: system
-        files: ^./.*\.java$
-
       - id: datahub-graphql-core-spotless
         name: datahub-graphql-core Spotless Apply
         entry: ./gradlew :datahub-graphql-core:spotlessApply

From ce1d1e2b8cfede9c76715e12871033e554dffe9f Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 8 Jan 2025 17:25:03 +0530
Subject: [PATCH 5/6] add override hooks file

---
 .github/scripts/generate_pre_commit.py   | 38 ++++++++++++++++++++----
 .github/scripts/pre-commit-override.yaml |  8 +++++
 .pre-commit-config.yaml                  |  6 ++++
 3 files changed, 47 insertions(+), 5 deletions(-)
 create mode 100644 .github/scripts/pre-commit-override.yaml

diff --git a/.github/scripts/generate_pre_commit.py b/.github/scripts/generate_pre_commit.py
index dd2954e775d3b2..740d3c20d263b0 100755
--- a/.github/scripts/generate_pre_commit.py
+++ b/.github/scripts/generate_pre_commit.py
@@ -1,7 +1,8 @@
 """Generate pre-commit hooks for Java and Python projects.
 
 This script scans a repository for Java and Python projects and generates appropriate
-pre-commit hooks for linting and formatting.
+pre-commit hooks for linting and formatting. It also merges in additional hooks from
+an override file.
 """
 
 import os
@@ -138,8 +139,9 @@ def _get_relative_path(self, path: Path) -> str:
 class HookGenerator:
     """Generate pre-commit hooks for projects."""
 
-    def __init__(self, projects: list[Project]):
+    def __init__(self, projects: list[Project], override_file: str = None):
         self.projects = projects
+        self.override_file = override_file
 
     def generate_config(self) -> dict:
         """Generate the complete pre-commit config."""
@@ -151,7 +153,32 @@ def generate_config(self) -> dict:
             else:  # ProjectType.JAVA
                 hooks.append(self._generate_spotless_hook(project))
 
-        return {"repos": [{"repo": "local", "hooks": hooks}]}
+        config = {"repos": [{"repo": "local", "hooks": hooks}]}
+        
+        # Merge override hooks if they exist
+        if self.override_file and os.path.exists(self.override_file):
+            try:
+                with open(self.override_file, 'r') as f:
+                    override_config = yaml.safe_load(f)
+                
+                if override_config and 'repos' in override_config:
+                    for override_repo in override_config['repos']:
+                        matching_repo = next(
+                            (repo for repo in config['repos'] 
+                             if repo['repo'] == override_repo['repo']),
+                            None
+                        )
+                        
+                        if matching_repo:
+                            matching_repo['hooks'].extend(override_repo.get('hooks', []))
+                        else:
+                            config['repos'].append(override_repo)
+                
+                print(f"Merged additional hooks from {self.override_file}")
+            except Exception as e:
+                print(f"Warning: Error reading override file {self.override_file}: {e}")
+
+        return config
 
     def _generate_lint_fix_hook(self, project: Project) -> dict:
         """Generate a lint-fix hook for Python projects."""
@@ -208,6 +235,7 @@ def write_yaml_with_spaces(file_path: str, data: dict):
 
 def main():
     root_dir = os.path.abspath(os.curdir)
+    override_file = ".github/scripts/pre-commit-override.yaml"
 
     # Find projects
     finder = ProjectFinder(root_dir)
@@ -226,7 +254,7 @@ def main():
             print(f"  - {project.path}")
 
     # Generate and write config
-    generator = HookGenerator(projects)
+    generator = HookGenerator(projects, override_file)
     config = generator.generate_config()
     write_yaml_with_spaces(".pre-commit-config.yaml", config)
 
@@ -234,4 +262,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()
\ No newline at end of file
diff --git a/.github/scripts/pre-commit-override.yaml b/.github/scripts/pre-commit-override.yaml
new file mode 100644
index 00000000000000..a085d9ea3ee93b
--- /dev/null
+++ b/.github/scripts/pre-commit-override.yaml
@@ -0,0 +1,8 @@
+repos:
+  - repo: local
+    hooks:
+      - id: smoke-test-cypress-lint-fix
+        name: smoke-test cypress Lint Fix
+        entry: ./gradlew :smoke-test:cypressLintFix
+        language: system
+        files: ^smoke-test/tests/cypress/.*$
\ No newline at end of file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e155f91f107394..c4edc2cc176355 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -372,3 +372,9 @@ repos:
         entry: ./gradlew :test-models:spotlessApply
         language: system
         files: ^test-models/.*\.java$
+
+      - id: smoke-test-cypress-lint-fix
+        name: smoke-test cypress Lint Fix
+        entry: ./gradlew :smoke-test:cypressLintFix
+        language: system
+        files: ^smoke-test/tests/cypress/.*$

From ad3f9f7092e2d77b3c35353cf2e3426e23ecd920 Mon Sep 17 00:00:00 2001
From: Aseem Bansal <asmbansal2@gmail.com>
Date: Wed, 8 Jan 2025 21:09:40 +0530
Subject: [PATCH 6/6] do not fail in case python is not installed

---
 settings.gradle | 48 +++++++++++++++++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 15 deletions(-)

diff --git a/settings.gradle b/settings.gradle
index 780a88eb0f37dc..77d0706549a439 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -82,30 +82,48 @@ include ':metadata-integration:java:datahub-schematron:cli'
 def installPreCommitHooks() {
     def preCommitInstalled = false
     try {
-        def process = "pre-commit --version".execute()
-        process.waitFor()
+        def process = ["which", "pre-commit"].execute()
+        def stdout = new StringBuilder()
+        def stderr = new StringBuilder()
+        process.waitForProcessOutput(stdout, stderr)
         preCommitInstalled = (process.exitValue() == 0)
+        println "Pre-commit check: ${stdout}"
     } catch (Exception e) {
-        println "pre-commit is not installed"
+        println "Error checking pre-commit: ${e.message}"
+        return
     }
 
     if (!preCommitInstalled) {
-        println "Installing pre-commit..."
-        def installProcess = "pip install pre-commit".execute()
-        installProcess.waitFor()
-        if (installProcess.exitValue() != 0) {
-            throw new GradleException("Failed to install pre-commit. Error: ${installProcess.err.text}")
+        try {
+            def installProcess = ["python", "-m", "pip", "install", "pre-commit"].execute()
+            def stdout = new StringBuilder()
+            def stderr = new StringBuilder()
+            installProcess.waitForProcessOutput(stdout, stderr)
+            if (installProcess.exitValue() != 0) {
+                println "Failed to install pre-commit: ${stderr}"
+                return
+            }
+            println "Install output: ${stdout}"
+        } catch (Exception e) {
+            println "Error installing pre-commit: ${e.message}"
+            return 
         }
-        println "pre-commit installed successfully"
     }
 
-    println "Installing pre-commit hooks..."
-    def installHooksProcess = "pre-commit install".execute()
-    installHooksProcess.waitFor()
-    if (installHooksProcess.exitValue() != 0) {
-        throw new GradleException("Failed to install pre-commit hooks. Error: ${installHooksProcess.err.text}")
+    try {
+        def installHooksProcess = ["python", "-m", "pre_commit", "install"].execute()
+        def stdout = new StringBuilder()
+        def stderr = new StringBuilder()
+        installHooksProcess.waitForProcessOutput(stdout, stderr)
+        if (installHooksProcess.exitValue() != 0) {
+            println "Failed to install hooks: ${stderr}"
+            return
+        }
+        println "Hooks output: ${stdout}"
+    } catch (Exception e) {
+        println "Error installing hooks: ${e.message}"
+        return
     }
-    println "pre-commit hooks installed successfully"
 }
 
 installPreCommitHooks()
\ No newline at end of file