Skip to content

Commit 67f51f5

Browse files
Merge pull request #153 from unicef/hotfix/report-long-execution-in-encode_chunk
Hotfix: Report long execution in encode_chunk
2 parents 17cd875 + 2d9e3db commit 67f51f5

File tree

4 files changed

+39
-10
lines changed

4 files changed

+39
-10
lines changed

docker/bin/docker-entrypoint.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ case "$1" in
3131
;;
3232
worker)
3333
set -- tini -- "$@"
34-
set -- gosu hope:unicef celery -A hope_dedup_engine.config.celery worker -E --loglevel=ERROR --concurrency=4
34+
set -- gosu hope:unicef celery -A hope_dedup_engine.config.celery worker -E --loglevel=ERROR --concurrency=2
3535
;;
3636
beat)
3737
set -- tini -- "$@"

src/hope_dedup_engine/apps/faces/celery_tasks.py

+7-3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from hope_dedup_engine.apps.api.utils.notification import send_notification
1616
from hope_dedup_engine.apps.faces.managers import FileSyncManager
1717
from hope_dedup_engine.apps.faces.services.facial import dedupe_images, encode_faces
18+
from hope_dedup_engine.apps.faces.utils import report_long_execution
1819
from hope_dedup_engine.config.celery import DedupeTask, app
1920
from hope_dedup_engine.types import EncodingType, FindingType
2021

@@ -72,11 +73,14 @@ def encode_chunk(
7273
config: dict[str, Any],
7374
) -> tuple[EncodingType, int, int]:
7475
"""Encode faces in a chunk of files."""
75-
ds = DeduplicationSet.objects.get(pk=config.get("deduplication_set_id"))
76+
with report_long_execution('DeduplicationSet.objects.get(pk=config.get("deduplication_set_id"))'):
77+
ds = DeduplicationSet.objects.get(pk=config.get("deduplication_set_id"))
7678
try:
7779
callback = partial(notify_status, task=self, dedup_job_id=ds.dedupjob.pk)
78-
pre_encodings = ds.get_encodings()
79-
return encode_faces(files, config.get("encoding"), pre_encodings, progress=callback)
80+
with report_long_execution("ds.get_encodings()"):
81+
pre_encodings = ds.get_encodings()
82+
with report_long_execution('encode_faces(files, config.get("encoding"), pre_encodings, progress=callback)'):
83+
return encode_faces(files, config.get("encoding"), pre_encodings, progress=callback)
8084
except Exception as e:
8185
sentry_sdk.capture_exception(e)
8286
handle_error(ds)

src/hope_dedup_engine/apps/faces/services/facial.py

+12-6
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from hope_dedup_engine.apps.api.models import Image
88
from hope_dedup_engine.apps.faces.managers import ImagesStorageManager
9-
from hope_dedup_engine.apps.faces.utils import is_facial_error
9+
from hope_dedup_engine.apps.faces.utils import is_facial_error, report_long_execution
1010
from hope_dedup_engine.types import EncodingType, FindingType, IgnoredPairType
1111

1212
logger = logging.getLogger(__name__)
@@ -25,24 +25,30 @@ def encode_faces(
2525
if not callable(progress):
2626
progress = default_progress
2727

28-
storage = ImagesStorageManager()
29-
images = storage.get_files()
28+
with report_long_execution("ImagesStorageManager()"):
29+
storage = ImagesStorageManager()
30+
31+
with report_long_execution("storage.get_files()"):
32+
images = storage.get_files()
3033

3134
encoded = {}
3235
if pre_encodings:
33-
encoded.update(pre_encodings)
36+
with report_long_execution("encoded.update(pre_encodings)"):
37+
encoded.update(pre_encodings)
3438
added_cnt = existing_cnt = 0
3539
existing_cnt = 1000
3640
for file in files:
37-
progress()
41+
with report_long_execution("progress()"):
42+
progress()
3843
if file not in images:
3944
encoded[file] = Image.StatusCode.NO_FILE_FOUND.name
4045
continue
4146
if file in encoded:
4247
existing_cnt += 1
4348
continue
4449
try:
45-
result = DeepFace.represent(storage.load_image(file), **(options or {}))
50+
with report_long_execution("DeepFace.represent(storage.load_image(file), **(options or {}))"):
51+
result = DeepFace.represent(storage.load_image(file), **(options or {}))
4652
if len(result) > 1:
4753
encoded[file] = Image.StatusCode.MULTIPLE_FACES_DETECTED.name
4854
else:

src/hope_dedup_engine/apps/faces/utils.py

+19
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
import time
2+
from contextlib import contextmanager
3+
from typing import Generator
4+
5+
import sentry_sdk
6+
17
from hope_dedup_engine.apps.api.models import Image
28

39

@@ -11,3 +17,16 @@ def is_facial_error(value):
1117
Image.StatusCode.values + Image.StatusCode.names + [choice.label for choice in Image.StatusCode]
1218
)
1319
return False
20+
21+
22+
DEFAULT_THRESHOLD_SECONDS = 60
23+
24+
25+
@contextmanager
26+
def report_long_execution(
27+
message: str, threshold_seconds: int = DEFAULT_THRESHOLD_SECONDS
28+
) -> Generator[None, None, None]:
29+
start = time.time()
30+
yield
31+
if (total := time.time() - start) > threshold_seconds:
32+
sentry_sdk.capture_message(f"Execution took {total} seconds: {message}")

0 commit comments

Comments
 (0)