Merge pull request #154 from unicef/hotfix/do_not_list_all_files_in_storage

Hotfix: Do not list all files in storage

Author: sergey-misuk-valor

src/hope_dedup_engine/apps/faces/managers/storage.py

@@ -1,23 +1,14 @@
-from fnmatch import fnmatch
-from typing import Final
-
 from django.conf import settings
 
 import cv2
 import numpy as np
 from storages.backends.azure_storage import AzureStorage
 
-FILES_PATTERN: Final[tuple[str]] = ("*.png", "*.jpg", "*.jpeg")
-
 
 class ImagesStorageManager:
     def __init__(self) -> None:
         self.storage: AzureStorage = AzureStorage(**settings.STORAGES.get("hope").get("OPTIONS"))
 
-    def get_files(self, pattern: tuple = FILES_PATTERN) -> list[str]:
-        _, images = self.storage.listdir("")
-        return [f for f in images if any(fnmatch(f, p) for p in pattern)]
-
     def load_image(self, file: str) -> np.ndarray:
         with self.storage.open(file, "rb") as img_file:
             img_array = np.frombuffer(img_file.read(), dtype=np.uint8)

src/hope_dedup_engine/apps/faces/services/facial.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 from typing import Any
 
+from azure.core.exceptions import ResourceNotFoundError
 from deepface import DeepFace
 
 from hope_dedup_engine.apps.api.models import Image
@@ -28,9 +29,6 @@ def encode_faces(
     with report_long_execution("ImagesStorageManager()"):
         storage = ImagesStorageManager()
 
-    with report_long_execution("storage.get_files()"):
-        images = storage.get_files()
-
     encoded = {}
     if pre_encodings:
         with report_long_execution("encoded.update(pre_encodings)"):
@@ -40,9 +38,6 @@ def encode_faces(
     for file in files:
         with report_long_execution("progress()"):
             progress()
-        if file not in images:
-            encoded[file] = Image.StatusCode.NO_FILE_FOUND.name
-            continue
         if file in encoded:
             existing_cnt += 1
             continue
@@ -59,6 +54,9 @@ def encode_faces(
             encoded[file] = Image.StatusCode.GENERIC_ERROR.name
         except ValueError:
             encoded[file] = Image.StatusCode.NO_FACE_DETECTED.name
+        except ResourceNotFoundError:
+            encoded[file] = Image.StatusCode.NO_FILE_FOUND.name
+
     return encoded, added_cnt, existing_cnt