fix: batch eSTS writes

Author: pl4nty

ests-errors.py

@@ -26,27 +26,23 @@ async def fetch_error_message(code, session):
 async def main():
     ERROR_CODE_RANGE = range(0, 10000000)
     OUTPUT_FILENAME = 'ests-errors.csv'
-    BATCH_SIZE = 10000  # Adjust this value based on your available memory
-
-    results = []
+    BATCH_SIZE = 1000  # Adjust this value based on your available memory
 
     async with aiohttp.TCPConnector(limit_per_host=50) as connector, aiohttp.ClientSession(connector=connector) as session:
-        for i in range(0, len(ERROR_CODE_RANGE), BATCH_SIZE):
-            batch = ERROR_CODE_RANGE[i:i + BATCH_SIZE]
-            tasks = [fetch_error_message(code, session) for code in batch]
-            for result in await asyncio.gather(*tasks):
-                if result:
-                    error_code, message, remediation = result
-                    results.append((error_code, message, remediation))
-
-    # Sort the results by error code
-    results.sort(key=lambda x: x[0])
-
-    # Write the sorted results to the CSV file
-    with open(OUTPUT_FILENAME, mode='w', newline='') as file:
-        writer = csv.writer(file)
-        writer.writerow(['Code', 'Message', 'Remediation'])
-        writer.writerows(results)
+        with open(OUTPUT_FILENAME, mode='w', newline='') as file:
+          writer = csv.writer(file)
+          writer.writerow(['Code', 'Message', 'Remediation'])
+          for i in range(0, len(ERROR_CODE_RANGE), BATCH_SIZE):
+              results = []
+              batch = ERROR_CODE_RANGE[i:i + BATCH_SIZE]
+              tasks = [fetch_error_message(code, session) for code in batch]
+              for result in await asyncio.gather(*tasks):
+                  if result:
+                      error_code, message, remediation = result
+                      results.append((error_code, message, remediation))
+              # Sort the results by error code
+              results.sort(key=lambda x: x[0])
+              writer.writerows(results)
 
 if __name__ == '__main__':
     # 7GB RAM limit to avoid runner OOM