Skip to content

Commit 8e9fc20

Browse files
david-leifkerdavid-leifker
authored
feat(gradle): add quickstartPgDebug option (#12195)
1 parent f6c0cf3 commit 8e9fc20

File tree

1 file changed

+131
-131
lines changed

1 file changed

+131
-131
lines changed

‎docker/build.gradle

+131-131
Original file line numberDiff line numberDiff line change
@@ -18,43 +18,143 @@ ext {
1818
':datahub-upgrade',
1919
':metadata-service:war',
2020
]
21-
quickstart_modules = backend_profile_modules + [
22-
':metadata-jobs:mce-consumer-job',
23-
':metadata-jobs:mae-consumer-job',
24-
':datahub-frontend'
21+
22+
python_services_modules = []
23+
24+
// Common configuration for all tasks
25+
common_config = [
26+
captureContainersOutput: true,
27+
captureContainersOutputToFiles: project.file('build/container-logs')
2528
]
2629

27-
debug_modules = quickstart_modules - [':metadata-jobs:mce-consumer-job',
28-
':metadata-jobs:mae-consumer-job']
29-
compose_args = ['-f', compose_base]
30-
debug_reloadable = [
31-
'datahub-gms-debug',
32-
'system-update-debug',
33-
'frontend-debug'
30+
// declarative task configuration
31+
quickstart_configs = [
32+
'quickstart': [
33+
profile: 'quickstart-consumers',
34+
modules: python_services_modules + backend_profile_modules + [
35+
':datahub-frontend',
36+
':metadata-jobs:mce-consumer-job',
37+
':metadata-jobs:mae-consumer-job',
38+
]
39+
],
40+
'quickstartDebug': [
41+
profile: 'debug',
42+
modules: python_services_modules + backend_profile_modules + [':datahub-frontend'],
43+
isDebug: true
44+
],
45+
'quickstartPg': [
46+
profile: 'quickstart-postgres',
47+
modules: (backend_profile_modules - [':docker:mysql-setup']) + [
48+
':docker:postgres-setup',
49+
':datahub-frontend'
50+
]
51+
],
52+
'quickstartPgDebug': [
53+
profile: 'debug-postgres',
54+
modules: python_services_modules + (backend_profile_modules - [':docker:mysql-setup']) + [
55+
':docker:postgres-setup',
56+
':datahub-frontend'
57+
],
58+
isDebug: true
59+
],
60+
'quickstartSlim': [
61+
profile: 'quickstart-backend',
62+
modules: backend_profile_modules + [':docker:datahub-ingestion'],
63+
additionalEnv: [
64+
'DATAHUB_ACTIONS_IMAGE': 'acryldata/datahub-ingestion',
65+
'ACTIONS_VERSION': "v${version}-slim",
66+
'ACTIONS_EXTRA_PACKAGES': 'acryl-datahub-actions[executor] acryl-datahub-actions',
67+
'ACTIONS_CONFIG': 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml',
68+
'DATAHUB_LOCAL_COMMON_ENV': "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
69+
]
70+
],
71+
'quickstartStorage': [
72+
profile: 'quickstart-storage',
73+
preserveVolumes: true
74+
]
3475
]
35-
// Postgres
36-
pg_quickstart_modules = quickstart_modules - [':docker:mysql-setup'] + [':docker:postgres-setup']
76+
}
77+
78+
// Register all quickstart tasks
79+
quickstart_configs.each { taskName, config ->
80+
tasks.register(taskName)
81+
}
82+
83+
// Dynamically create all quickstart tasks and configurations
84+
dockerCompose {
85+
// Configure default settings that apply to all configurations
86+
useComposeFiles = [compose_base]
87+
projectName = project_name
88+
projectNamePrefix = ''
89+
buildBeforeUp = false
90+
buildBeforePull = false
91+
stopContainers = false
92+
removeVolumes = false
93+
94+
quickstart_configs.each { taskName, config ->
95+
"${taskName}" {
96+
isRequiredBy(tasks.named(taskName))
97+
if (config.profile) {
98+
composeAdditionalArgs = ['--profile', config.profile]
99+
}
100+
101+
// Common environment variables
102+
environment.put 'DATAHUB_VERSION', config.isDebug ?
103+
System.getenv("DATAHUB_VERSION") ?: "v${version}" :
104+
"v${version}"
105+
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false'
106+
environment.put "METADATA_TESTS_ENABLED", "true"
107+
environment.put "DATAHUB_REPO", "${docker_registry}"
108+
109+
// Additional environment variables if specified
110+
if (config.additionalEnv) {
111+
config.additionalEnv.each { key, value ->
112+
environment.put key, value
113+
}
114+
}
115+
116+
useComposeFiles = [compose_base]
117+
projectName = project_name
118+
projectNamePrefix = ''
119+
buildBeforeUp = false
120+
buildBeforePull = false
121+
stopContainers = false
122+
removeVolumes = false
123+
124+
// Apply common configuration
125+
common_config.each { key, value ->
126+
delegate."${key}" = value
127+
}
128+
129+
// Apply additional task-specific configuration if specified
130+
if (config.additionalConfig) {
131+
config.additionalConfig.each { key, value ->
132+
delegate."${key}" = value
133+
}
134+
}
135+
}
136+
}
137+
}
37138

38-
revision = 1 // increment to trigger rebuild
139+
// Configure dependencies for ComposeUp tasks
140+
quickstart_configs.each { taskName, config ->
141+
if (config.modules) {
142+
tasks.getByName("${taskName}ComposeUp").dependsOn(
143+
config.modules.collect { it + ":${config.isDebug ? 'dockerTagDebug' : 'dockerTag'}" }
144+
)
145+
}
39146
}
40147

41148
tasks.register('minDockerCompose2.20', Exec) {
42149
executable 'bash'
43150
args '-c', 'echo -e "$(docker compose version --short)\n2.20"|sort --version-sort --check=quiet --reverse'
44151
}
45152

46-
tasks.register('quickstart') {}
47-
tasks.register('quickstartSlim') {}
48-
tasks.register('quickstartDebug') {}
49-
tasks.register('quickstartPg') {}
50-
tasks.register('quickstartStorage') {}
51-
52153
tasks.register('quickstartNuke') {
53154
doFirst {
54-
dockerCompose.quickstart.removeVolumes = true
55-
dockerCompose.quickstartPg.removeVolumes = true
56-
dockerCompose.quickstartSlim.removeVolumes = true
57-
dockerCompose.quickstartDebug.removeVolumes = true
155+
quickstart_configs.each { taskName, config ->
156+
dockerCompose."${taskName}".removeVolumes = !config.preserveVolumes
157+
}
58158
}
59159
finalizedBy(tasks.withType(ComposeDownForced))
60160
}
@@ -63,117 +163,17 @@ tasks.register('quickstartDown') {
63163
finalizedBy(tasks.withType(ComposeDownForced))
64164
}
65165

66-
dockerCompose {
67-
quickstart {
68-
isRequiredBy(tasks.named('quickstart'))
69-
composeAdditionalArgs = ['--profile', 'quickstart-consumers']
70-
71-
environment.put 'DATAHUB_VERSION', "v${version}"
72-
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
73-
74-
useComposeFiles = [compose_base]
75-
projectName = project_name
76-
projectNamePrefix = ''
77-
buildBeforeUp = false
78-
buildBeforePull = false
79-
stopContainers = false
80-
removeVolumes = false
81-
captureContainersOutput = true
82-
captureContainersOutputToFiles = project.file('build/container-logs')
83-
}
84-
85-
quickstartPg {
86-
isRequiredBy(tasks.named('quickstartPg'))
87-
composeAdditionalArgs = ['--profile', 'quickstart-postgres']
88-
89-
environment.put 'DATAHUB_VERSION', "v${version}"
90-
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
91-
92-
useComposeFiles = [compose_base]
93-
projectName = project_name
94-
projectNamePrefix = ''
95-
buildBeforeUp = false
96-
buildBeforePull = false
97-
stopContainers = false
98-
removeVolumes = false
99-
}
100-
101-
/**
102-
* The smallest disk footprint required for Spark integration tests
103-
*
104-
* No frontend, mae, mce, or other services
105-
*/
106-
quickstartSlim {
107-
isRequiredBy(tasks.named('quickstartSlim'))
108-
composeAdditionalArgs = ['--profile', 'quickstart-backend']
109-
110-
environment.put 'DATAHUB_VERSION', "v${version}"
111-
environment.put "DATAHUB_ACTIONS_IMAGE", "acryldata/datahub-ingestion"
112-
environment.put "ACTIONS_VERSION", "v${version}-slim"
113-
environment.put "ACTIONS_EXTRA_PACKAGES", 'acryl-datahub-actions[executor] acryl-datahub-actions'
114-
environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml'
115-
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
116-
// disabled for spark-lineage smoke-test
117-
environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
118-
119-
useComposeFiles = [compose_base]
120-
projectName = project_name
121-
projectNamePrefix = ''
122-
buildBeforeUp = false
123-
buildBeforePull = false
124-
stopContainers = false
125-
removeVolumes = false
126-
captureContainersOutput = true
127-
captureContainersOutputToFiles = project.file('build/container-logs')
128-
}
129-
130-
quickstartDebug {
131-
isRequiredBy(tasks.named('quickstartDebug'))
132-
composeAdditionalArgs = ['--profile', 'debug']
133-
134-
if (System.getenv().containsKey("DATAHUB_VERSION")) {
135-
environment.put 'DATAHUB_VERSION', System.getenv("DATAHUB_VERSION")
136-
}
137-
environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
138-
139-
useComposeFiles = [compose_base]
140-
projectName = project_name
141-
projectNamePrefix = ''
142-
buildBeforeUp = false
143-
buildBeforePull = false
144-
stopContainers = false
145-
removeVolumes = false
146-
}
147-
148-
quickstartStorage {
149-
isRequiredBy(tasks.named('quickstartStorage'))
150-
composeAdditionalArgs = ['--profile', 'quickstart-storage']
151-
152-
useComposeFiles = [compose_base]
153-
projectName = project_name
154-
projectNamePrefix = ''
155-
buildBeforeUp = false
156-
buildBeforePull = false
157-
stopContainers = false
158-
removeVolumes = false
159-
}
160-
}
161-
tasks.getByName('quickstartComposeUp').dependsOn(
162-
quickstart_modules.collect { it + ':dockerTag' })
163-
tasks.getByName('quickstartPgComposeUp').dependsOn(
164-
pg_quickstart_modules.collect { it + ':dockerTag' })
165-
tasks.getByName('quickstartSlimComposeUp').dependsOn(
166-
([':docker:datahub-ingestion'] + backend_profile_modules)
167-
.collect { it + ':dockerTag' })
168-
tasks.getByName('quickstartDebugComposeUp').dependsOn(
169-
debug_modules.collect { it + ':dockerTagDebug' }
170-
)
171166
tasks.withType(ComposeUp).configureEach {
172167
shouldRunAfter('quickstartNuke')
173168
dependsOn tasks.named("minDockerCompose2.20")
174169
}
175170

176171
task debugReload(type: Exec) {
177-
def cmd = ['docker compose -p datahub --profile debug'] + compose_args + ['restart'] + debug_reloadable
172+
def cmd = ['docker compose -p datahub --profile debug'] + ['-f', compose_base] + [
173+
'restart',
174+
'datahub-gms-debug',
175+
'system-update-debug',
176+
'frontend-debug'
177+
]
178178
commandLine 'bash', '-c', cmd.join(" ")
179-
}
179+
}

0 commit comments

Comments
 (0)