@@ -207,6 +207,9 @@ def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]:
207
207
assert self .ctx .graph
208
208
dpis = []
209
209
start = 0
210
+ # This graphql endpoint doesn't support scrolling and therefore after 10k DPIs it causes performance issues on ES
211
+ # Therefore, we are limiting the max DPIs to 9000
212
+ max_item = 9000
210
213
while True :
211
214
try :
212
215
job_query_result = self .ctx .graph .execute_graphql (
@@ -226,10 +229,12 @@ def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]:
226
229
runs = runs_data .get ("runs" )
227
230
dpis .extend (runs )
228
231
start += batch_size
229
- if len (runs ) < batch_size :
232
+ if len (runs ) < batch_size or start >= max_item :
230
233
break
231
234
except Exception as e :
232
- logger .error (f"Exception while fetching DPIs for job { job_urn } : { e } " )
235
+ self .report .failure (
236
+ f"Exception while fetching DPIs for job { job_urn } :" , exc = e
237
+ )
233
238
break
234
239
return dpis
235
240
@@ -254,8 +259,9 @@ def keep_last_n_dpi(
254
259
deleted_count_last_n += 1
255
260
futures [future ]["deleted" ] = True
256
261
except Exception as e :
257
- logger .error (f"Exception while deleting DPI: { e } " )
258
-
262
+ self .report .report_failure (
263
+ f"Exception while deleting DPI: { e } " , exc = e
264
+ )
259
265
if deleted_count_last_n % self .config .batch_size == 0 :
260
266
logger .info (f"Deleted { deleted_count_last_n } DPIs from { job .urn } " )
261
267
if self .config .delay :
@@ -289,7 +295,7 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None:
289
295
dpis = self .fetch_dpis (job .urn , self .config .batch_size )
290
296
dpis .sort (
291
297
key = lambda x : x ["created" ]["time" ]
292
- if "created" in x and "time" in x ["created" ]
298
+ if x . get ( "created" ) and x ["created" ]. get ( "time" )
293
299
else 0 ,
294
300
reverse = True ,
295
301
)
@@ -325,8 +331,8 @@ def remove_old_dpis(
325
331
continue
326
332
327
333
if (
328
- "created" not in dpi
329
- or "time" not in dpi ["created" ]
334
+ not dpi . get ( "created" )
335
+ or not dpi ["created" ]. get ( "time" )
330
336
or dpi ["created" ]["time" ] < retention_time * 1000
331
337
):
332
338
future = executor .submit (
@@ -340,7 +346,7 @@ def remove_old_dpis(
340
346
deleted_count_retention += 1
341
347
futures [future ]["deleted" ] = True
342
348
except Exception as e :
343
- logger . error (f"Exception while deleting DPI: { e } " )
349
+ self . report . report_failure (f"Exception while deleting DPI: { e } " , exc = e )
344
350
345
351
if deleted_count_retention % self .config .batch_size == 0 :
346
352
logger .info (
@@ -351,9 +357,12 @@ def remove_old_dpis(
351
357
logger .info (f"Sleeping for { self .config .delay } seconds" )
352
358
time .sleep (self .config .delay )
353
359
354
- logger .info (
355
- f"Deleted { deleted_count_retention } DPIs from { job .urn } due to retention"
356
- )
360
+ if deleted_count_retention > 0 :
361
+ logger .info (
362
+ f"Deleted { deleted_count_retention } DPIs from { job .urn } due to retention"
363
+ )
364
+ else :
365
+ logger .debug (f"No DPIs to delete from { job .urn } due to retention" )
357
366
358
367
def get_data_flows (self ) -> Iterable [DataFlowEntity ]:
359
368
assert self .ctx .graph
0 commit comments