Skip to content

Commit e884040

Browse files
david-leifkerdavid-leifker
committed
readOnly to createDefaultAspects
1 parent 0acbe89 commit e884040

File tree

16 files changed

+45
-40
lines changed

16 files changed

+45
-40
lines changed

‎datahub-upgrade/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ task runRestoreIndices(type: Exec) {
140140
"-jar",
141141
"-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
142142
"-Dserver.port=8083",
143-
bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "readOnly=true"
143+
bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "createDefaultAspects=true"
144144
}
145145

146146
task runRestoreIndicesUrn(type: Exec) {

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class RestoreIndices implements Upgrade {
2626
public static final String URN_ARG_NAME = "urn";
2727
public static final String URN_LIKE_ARG_NAME = "urnLike";
2828
public static final String URN_BASED_PAGINATION_ARG_NAME = "urnBasedPagination";
29-
public static final String READ_ONLY_ARG_NAME = "readOnly";
29+
public static final String CREATE_DEFAULT_ASPECTS_ARG_NAME = "createDefaultAspects";
3030

3131
public static final String STARTING_OFFSET_ARG_NAME = "startingOffset";
3232
public static final String LAST_URN_ARG_NAME = "lastUrn";

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java

+8-6
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ private RestoreIndicesArgs getArgs(UpgradeContext context) {
104104
result.batchDelayMs = getBatchDelayMs(context.parsedArgs());
105105
result.start = getStartingOffset(context.parsedArgs());
106106
result.urnBasedPagination = getUrnBasedPagination(context.parsedArgs());
107-
result.readOnly = getReadOnly(context.parsedArgs());
107+
result.createDefaultAspects = getCreateDefaultAspects(context.parsedArgs());
108108
if (containsKey(context.parsedArgs(), RestoreIndices.ASPECT_NAME_ARG_NAME)) {
109109
result.aspectName = context.parsedArgs().get(RestoreIndices.ASPECT_NAME_ARG_NAME).get();
110110
context.report().addLine(String.format("aspect is %s", result.aspectName));
@@ -324,12 +324,14 @@ private long getBatchDelayMs(final Map<String, Optional<String>> parsedArgs) {
324324
return resolvedBatchDelayMs;
325325
}
326326

327-
private boolean getReadOnly(final Map<String, Optional<String>> parsedArgs) {
328-
Boolean readOnly = null;
329-
if (containsKey(parsedArgs, RestoreIndices.READ_ONLY_ARG_NAME)) {
330-
readOnly = Boolean.parseBoolean(parsedArgs.get(RestoreIndices.READ_ONLY_ARG_NAME).get());
327+
private boolean getCreateDefaultAspects(final Map<String, Optional<String>> parsedArgs) {
328+
Boolean createDefaultAspects = null;
329+
if (containsKey(parsedArgs, RestoreIndices.CREATE_DEFAULT_ASPECTS_ARG_NAME)) {
330+
createDefaultAspects =
331+
Boolean.parseBoolean(
332+
parsedArgs.get(RestoreIndices.CREATE_DEFAULT_ASPECTS_ARG_NAME).get());
331333
}
332-
return readOnly != null ? readOnly : false;
334+
return createDefaultAspects != null ? createDefaultAspects : false;
333335
}
334336

335337
private int getThreadCount(final Map<String, Optional<String>> parsedArgs) {

‎datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStepTest.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ public void testExecutableWithDefaultArgs() {
148148
assertEquals(capturedArgs.batchDelayMs, 250);
149149
assertEquals(capturedArgs.start, 0);
150150
assertFalse(capturedArgs.urnBasedPagination);
151-
assertFalse(capturedArgs.readOnly);
151+
assertFalse(capturedArgs.createDefaultAspects);
152152
}
153153

154154
@Test
@@ -159,7 +159,7 @@ public void testExecutableWithCustomArgs() {
159159
parsedArgs.put(RestoreIndices.BATCH_DELAY_MS_ARG_NAME, Optional.of("100"));
160160
parsedArgs.put(RestoreIndices.STARTING_OFFSET_ARG_NAME, Optional.of("100"));
161161
parsedArgs.put(RestoreIndices.URN_BASED_PAGINATION_ARG_NAME, Optional.of("true"));
162-
parsedArgs.put(RestoreIndices.READ_ONLY_ARG_NAME, Optional.of("true"));
162+
parsedArgs.put(RestoreIndices.CREATE_DEFAULT_ASPECTS_ARG_NAME, Optional.of("true"));
163163
parsedArgs.put(RestoreIndices.ASPECT_NAME_ARG_NAME, Optional.of("testAspect"));
164164
parsedArgs.put(RestoreIndices.URN_ARG_NAME, Optional.of("testUrn"));
165165

@@ -187,7 +187,7 @@ public void testExecutableWithCustomArgs() {
187187
assertEquals(capturedArgs.batchDelayMs, 100);
188188
assertEquals(capturedArgs.start, 100);
189189
assertTrue(capturedArgs.urnBasedPagination);
190-
assertTrue(capturedArgs.readOnly);
190+
assertTrue(capturedArgs.createDefaultAspects);
191191
assertEquals(capturedArgs.aspectName, "testAspect");
192192
assertEquals(capturedArgs.urn, "testUrn");
193193
}

‎docs/how/restore-indices.md

+5-5
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ The following configurations are available:
4040
* `urnLike`: SQL LIKE pattern to filter URNs (e.g., "urn:li:dataset%")
4141

4242

43-
### Elasticsearch Only
43+
### Default Aspects
4444

45-
* `readOnly`: Enable SQL read-only mode
45+
* `createDefaultAspects`: Create default aspects in both SQL and ES if missing.
4646

47-
During the restore indices process, it will create missing default aspects in SQL by default. While this may be
47+
During the restore indices process, it will create default aspects in SQL. While this may be
4848
desired in some situations, disabling this feature is required when using a read-only SQL replica.
4949

5050
### Nuclear option
@@ -272,8 +272,8 @@ Remember to reset this after restoration completes!
272272

273273
#### SQL/Primary Storage
274274

275-
Consider using a read replica as the source of the job's data. If you enable configure a read-only replica
276-
you must also provide the parameter `readOnly=true`.
275+
Consider using a read replica as the source of the job's data. If you configure a read-only replica
276+
you must also provide the parameter `createDefaultAspects=true`.
277277

278278
#### Kafka & Consumers
279279

‎metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java

+6-5
Original file line numberDiff line numberDiff line change
@@ -1677,7 +1677,7 @@ public List<RestoreIndicesResult> restoreIndices(
16771677
opContext.getRetrieverContext(), batch.collect(Collectors.toList()), false);
16781678

16791679
RestoreIndicesResult result =
1680-
restoreIndices(opContext, systemAspects, logger, args.readOnly());
1680+
restoreIndices(opContext, systemAspects, logger, args.createDefaultAspects());
16811681
result.timeSqlQueryMs = timeSqlQueryMs;
16821682

16831683
logger.accept("Batch completed.");
@@ -1700,7 +1700,7 @@ public List<RestoreIndicesResult> restoreIndices(
17001700
@Nonnull Set<Urn> urns,
17011701
@Nullable Set<String> inputAspectNames,
17021702
@Nullable Integer inputBatchSize,
1703-
boolean readOnly)
1703+
boolean createDefaultAspects)
17041704
throws RemoteInvocationException, URISyntaxException {
17051705
int batchSize = inputBatchSize != null ? inputBatchSize : 100;
17061706

@@ -1725,7 +1725,8 @@ public List<RestoreIndicesResult> restoreIndices(
17251725
false);
17261726
long timeSqlQueryMs = System.currentTimeMillis() - startTime;
17271727

1728-
RestoreIndicesResult result = restoreIndices(opContext, systemAspects, s -> {}, readOnly);
1728+
RestoreIndicesResult result =
1729+
restoreIndices(opContext, systemAspects, s -> {}, createDefaultAspects);
17291730
result.timeSqlQueryMs = timeSqlQueryMs;
17301731
results.add(result);
17311732
}
@@ -1745,7 +1746,7 @@ private RestoreIndicesResult restoreIndices(
17451746
@Nonnull OperationContext opContext,
17461747
List<SystemAspect> systemAspects,
17471748
@Nonnull Consumer<String> logger,
1748-
boolean readOnly) {
1749+
boolean createDefaultAspects) {
17491750
RestoreIndicesResult result = new RestoreIndicesResult();
17501751
long startTime = System.currentTimeMillis();
17511752
int ignored = 0;
@@ -1847,7 +1848,7 @@ private RestoreIndicesResult restoreIndices(
18471848
.getFirst());
18481849

18491850
// 6. Ensure default aspects are in existence in SQL
1850-
if (!readOnly) {
1851+
if (createDefaultAspects) {
18511852
List<MCPItem> keyAspect =
18521853
List.of(
18531854
ChangeItemImpl.builder()

‎metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/operations/elastic/ElasticsearchController.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,8 @@ public ResponseEntity<List<RestoreIndicesResult>> restoreIndices(
491491
@RequestParam(required = false, name = "gePitEpochMs", defaultValue = "0") @Nullable
492492
Long gePitEpochMs,
493493
@RequestParam(required = false, name = "lePitEpochMs") @Nullable Long lePitEpochMs,
494-
@RequestParam(value = "readOnly", defaultValue = "false") Boolean readOnly) {
494+
@RequestParam(value = "createDefaultAspects", defaultValue = "false")
495+
Boolean createDefaultAspects) {
495496

496497
Authentication authentication = AuthenticationContext.getAuthentication();
497498
OperationContext opContext =
@@ -521,7 +522,7 @@ public ResponseEntity<List<RestoreIndicesResult>> restoreIndices(
521522
.limit(limit)
522523
.gePitEpochMs(gePitEpochMs)
523524
.lePitEpochMs(lePitEpochMs)
524-
.readOnly(readOnly);
525+
.createDefaultAspects(createDefaultAspects);
525526

526527
return ResponseEntity.of(Optional.of(entityService.restoreIndices(opContext, args, log::info)));
527528
}
@@ -534,7 +535,8 @@ public ResponseEntity<List<RestoreIndicesResult>> restoreIndices(
534535
@RequestParam(required = false, name = "aspectNames") @Nullable Set<String> aspectNames,
535536
@RequestParam(required = false, name = "batchSize", defaultValue = "100") @Nullable
536537
Integer batchSize,
537-
@RequestParam(value = "readOnly", defaultValue = "false") Boolean readOnly,
538+
@RequestParam(value = "createDefaultAspects", defaultValue = "false")
539+
Boolean createDefaultAspects,
538540
@RequestBody @Nonnull Set<String> urns)
539541
throws RemoteInvocationException, URISyntaxException {
540542

@@ -560,6 +562,6 @@ public ResponseEntity<List<RestoreIndicesResult>> restoreIndices(
560562
urns.stream().map(UrnUtils::getUrn).collect(Collectors.toSet()),
561563
aspectNames,
562564
batchSize,
563-
readOnly)));
565+
createDefaultAspects)));
564566
}
565567
}

‎metadata-service/restli-api/src/main/idl/com.linkedin.entity.aspects.restspec.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@
135135
"type" : "long",
136136
"optional" : true
137137
}, {
138-
"name" : "readOnly",
138+
"name" : "createDefaultAspects",
139139
"type" : "boolean",
140140
"optional" : true
141141
} ],

‎metadata-service/restli-api/src/main/idl/com.linkedin.operations.operations.restspec.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@
6868
"type" : "long",
6969
"optional" : true
7070
}, {
71-
"name" : "readOnly",
71+
"name" : "createDefaultAspects",
7272
"type" : "boolean",
7373
"optional" : true
7474
} ],

‎metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -4427,7 +4427,7 @@
44274427
"type" : "long",
44284428
"optional" : true
44294429
}, {
4430-
"name" : "readOnly",
4430+
"name" : "createDefaultAspects",
44314431
"type" : "boolean",
44324432
"optional" : true
44334433
} ],

‎metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -4044,7 +4044,7 @@
40444044
"type" : "long",
40454045
"optional" : true
40464046
}, {
4047-
"name" : "readOnly",
4047+
"name" : "createDefaultAspects",
40484048
"type" : "boolean",
40494049
"optional" : true
40504050
} ],

‎metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ public Task<String> restoreIndices(
374374
@ActionParam("limit") @Optional @Nullable Integer limit,
375375
@ActionParam("gePitEpochMs") @Optional @Nullable Long gePitEpochMs,
376376
@ActionParam("lePitEpochMs") @Optional @Nullable Long lePitEpochMs,
377-
@ActionParam("readOnly") @Optional @Nullable Boolean readOnly) {
377+
@ActionParam("createDefaultAspects") @Optional @Nullable Boolean createDefaultAspects) {
378378
return RestliUtils.toTask(systemOperationContext,
379379
() -> {
380380

@@ -391,7 +391,7 @@ public Task<String> restoreIndices(
391391
}
392392

393393
return Utils.restoreIndices(systemOperationContext, getContext(),
394-
aspectName, urn, urnLike, start, batchSize, limit, gePitEpochMs, lePitEpochMs, _authorizer, _entityService, readOnly != null ? readOnly : false);
394+
aspectName, urn, urnLike, start, batchSize, limit, gePitEpochMs, lePitEpochMs, _authorizer, _entityService, createDefaultAspects != null ? createDefaultAspects : false);
395395
},
396396
MetricRegistry.name(this.getClass(), "restoreIndices"));
397397
}

‎metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/OperationsResource.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,10 @@ public Task<String> restoreIndices(
104104
@ActionParam("limit") @Optional @Nullable Integer limit,
105105
@ActionParam("gePitEpochMs") @Optional @Nullable Long gePitEpochMs,
106106
@ActionParam("lePitEpochMs") @Optional @Nullable Long lePitEpochMs,
107-
@ActionParam("readOnly") @Optional @Nullable Boolean readOnly) {
107+
@ActionParam("createDefaultAspects") @Optional @Nullable Boolean createDefaultAspects) {
108108
return RestliUtils.toTask(systemOperationContext,
109109
() -> Utils.restoreIndices(systemOperationContext, getContext(),
110-
aspectName, urn, urnLike, start, batchSize, limit, gePitEpochMs, lePitEpochMs, _authorizer, _entityService, readOnly != null ? readOnly : false),
110+
aspectName, urn, urnLike, start, batchSize, limit, gePitEpochMs, lePitEpochMs, _authorizer, _entityService, createDefaultAspects != null ? createDefaultAspects : false),
111111
MetricRegistry.name(this.getClass(), "restoreIndices"));
112112
}
113113

‎metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/operations/Utils.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public static String restoreIndices(
4545
@Nullable Long lePitEpochMs,
4646
@Nonnull Authorizer authorizer,
4747
@Nonnull EntityService<?> entityService,
48-
boolean readOnly) {
48+
boolean createDefaultAspects) {
4949

5050
EntitySpec resourceSpec = null;
5151
if (StringUtils.isNotBlank(urn)) {
@@ -75,7 +75,7 @@ public static String restoreIndices(
7575
.limit(limit)
7676
.gePitEpochMs(gePitEpochMs)
7777
.lePitEpochMs(lePitEpochMs)
78-
.readOnly(readOnly);
78+
.createDefaultAspects(createDefaultAspects);
7979
Map<String, Object> result = new HashMap<>();
8080
result.put("args", args);
8181
result.put("result", entityService

‎metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ List<RestoreIndicesResult> restoreIndices(
433433
@Nonnull Set<Urn> urns,
434434
@Nullable Set<String> inputAspectNames,
435435
@Nullable Integer inputBatchSize,
436-
boolean readOnly)
436+
boolean createDefaultAspects)
437437
throws RemoteInvocationException, URISyntaxException;
438438

439439
ListUrnsResult listUrns(

‎metadata-service/services/src/main/java/com/linkedin/metadata/entity/restoreindices/RestoreIndicesArgs.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public class RestoreIndicesArgs implements Cloneable {
1414
public static final int DEFAULT_NUM_THREADS = 1;
1515
public static final int DEFAULT_BATCH_DELAY_MS = 1000;
1616
public static final long DEFAULT_GE_PIT_EPOCH_MS = 0;
17-
public static final boolean DEFAULT_READ_ONLY = false;
17+
public static final boolean DEFAULT_CREATE_DEFAULT_ASPECTS = false;
1818

1919
public int start = 0;
2020
public int batchSize = DEFAULT_BATCH_SIZE;
@@ -23,7 +23,7 @@ public class RestoreIndicesArgs implements Cloneable {
2323
public long batchDelayMs = DEFAULT_BATCH_DELAY_MS;
2424
public long gePitEpochMs = DEFAULT_GE_PIT_EPOCH_MS;
2525
public long lePitEpochMs;
26-
public boolean readOnly = DEFAULT_READ_ONLY;
26+
public boolean createDefaultAspects = DEFAULT_CREATE_DEFAULT_ASPECTS;
2727
public String aspectName;
2828
public List<String> aspectNames = Collections.emptyList();
2929
public String urn;
@@ -78,8 +78,8 @@ public RestoreIndicesArgs lePitEpochMs(Long lePitEpochMs) {
7878
return this;
7979
}
8080

81-
public RestoreIndicesArgs readOnly(@Nullable Boolean readOnly) {
82-
this.readOnly = readOnly != null ? readOnly : false;
81+
public RestoreIndicesArgs createDefaultAspects(@Nullable Boolean readOnly) {
82+
this.createDefaultAspects = readOnly != null ? readOnly : false;
8383
return this;
8484
}
8585
}

0 commit comments

Comments
 (0)