Skip to content

Commit 380c69b

Browse files
Merge pull request #154 from unicef/hotfix/do_not_list_all_files_in_storage
Hotfix: Do not list all files in storage
2 parents 03458fb + d3ec5ab commit 380c69b

File tree

2 files changed

+4
-15
lines changed

2 files changed

+4
-15
lines changed

src/hope_dedup_engine/apps/faces/managers/storage.py

-9
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,14 @@
1-
from fnmatch import fnmatch
2-
from typing import Final
3-
41
from django.conf import settings
52

63
import cv2
74
import numpy as np
85
from storages.backends.azure_storage import AzureStorage
96

10-
FILES_PATTERN: Final[tuple[str]] = ("*.png", "*.jpg", "*.jpeg")
11-
127

138
class ImagesStorageManager:
149
def __init__(self) -> None:
1510
self.storage: AzureStorage = AzureStorage(**settings.STORAGES.get("hope").get("OPTIONS"))
1611

17-
def get_files(self, pattern: tuple = FILES_PATTERN) -> list[str]:
18-
_, images = self.storage.listdir("")
19-
return [f for f in images if any(fnmatch(f, p) for p in pattern)]
20-
2112
def load_image(self, file: str) -> np.ndarray:
2213
with self.storage.open(file, "rb") as img_file:
2314
img_array = np.frombuffer(img_file.read(), dtype=np.uint8)

src/hope_dedup_engine/apps/faces/services/facial.py

+4-6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from collections import defaultdict
33
from typing import Any
44

5+
from azure.core.exceptions import ResourceNotFoundError
56
from deepface import DeepFace
67

78
from hope_dedup_engine.apps.api.models import Image
@@ -28,9 +29,6 @@ def encode_faces(
2829
with report_long_execution("ImagesStorageManager()"):
2930
storage = ImagesStorageManager()
3031

31-
with report_long_execution("storage.get_files()"):
32-
images = storage.get_files()
33-
3432
encoded = {}
3533
if pre_encodings:
3634
with report_long_execution("encoded.update(pre_encodings)"):
@@ -40,9 +38,6 @@ def encode_faces(
4038
for file in files:
4139
with report_long_execution("progress()"):
4240
progress()
43-
if file not in images:
44-
encoded[file] = Image.StatusCode.NO_FILE_FOUND.name
45-
continue
4641
if file in encoded:
4742
existing_cnt += 1
4843
continue
@@ -59,6 +54,9 @@ def encode_faces(
5954
encoded[file] = Image.StatusCode.GENERIC_ERROR.name
6055
except ValueError:
6156
encoded[file] = Image.StatusCode.NO_FACE_DETECTED.name
57+
except ResourceNotFoundError:
58+
encoded[file] = Image.StatusCode.NO_FILE_FOUND.name
59+
6260
return encoded, added_cnt, existing_cnt
6361

6462

0 commit comments

Comments
 (0)