1
1
import traceback
2
- from collections import ChainMap
3
2
from functools import partial
4
3
from typing import Any , Final
5
4
17
16
from hope_dedup_engine .apps .faces .services .facial import dedupe_images , encode_faces
18
17
from hope_dedup_engine .apps .faces .utils import report_long_execution
19
18
from hope_dedup_engine .config .celery import DedupeTask , app
20
- from hope_dedup_engine .types import EncodingType , FindingType
19
+ from hope_dedup_engine .types import FindingType
21
20
22
21
CHUNK_SIZE : Final [int ] = 25
23
22
@@ -71,7 +70,7 @@ def encode_chunk(
71
70
self : DedupeTask ,
72
71
files : list [str ],
73
72
config : dict [str , Any ],
74
- ) -> tuple [ EncodingType , int , int ] :
73
+ ) -> None :
75
74
"""Encode faces in a chunk of files."""
76
75
with report_long_execution ('DeduplicationSet.objects.get(pk=config.get("deduplication_set_id"))' ):
77
76
ds = DeduplicationSet .objects .get (pk = config .get ("deduplication_set_id" ))
@@ -80,7 +79,9 @@ def encode_chunk(
80
79
with report_long_execution ("ds.get_encodings()" ):
81
80
pre_encodings = ds .get_encodings ()
82
81
with report_long_execution ('encode_faces(files, config.get("encoding"), pre_encodings, progress=callback)' ):
83
- return encode_faces (files , config .get ("encoding" ), pre_encodings , progress = callback )
82
+ results = encode_faces (files , config .get ("encoding" ), pre_encodings , progress = callback )
83
+ with report_long_execution ("ds.update_encodings(results[0])" ):
84
+ ds .update_encodings (results [0 ])
84
85
except Exception as e :
85
86
sentry_sdk .capture_exception (e )
86
87
handle_error (ds )
@@ -149,17 +150,15 @@ def callback_findings(
149
150
@app .task (bind = True , base = DedupeTask )
150
151
def callback_encodings (
151
152
self : Task ,
152
- results : tuple [ EncodingType , int , int ],
153
+ results : list [ None ],
153
154
config : dict [str , Any ],
154
155
) -> dict [str , Any ]:
155
156
"""Aggregate and save encodings."""
156
157
ds = DeduplicationSet .objects .get (pk = config .get ("deduplication_set_id" ))
157
158
try :
158
- encodings = dict (ChainMap (* [result [0 ] for result in results ]))
159
- ds .update_encodings (encodings )
160
159
deduplicate_dataset .delay (config )
161
160
return {
162
- "Encoded" : len ( encodings ) ,
161
+ "Encoded" : True ,
163
162
}
164
163
except Exception as e :
165
164
sentry_sdk .capture_exception (e )
0 commit comments