Added a new config option to control whether or not results aggregation is aborted if there are more than 8000 results. This is useful when using a debug configuration to generate a search results cache offline.

Author: Joe Kohlmann

flask/config.py

@@ -22,7 +22,9 @@ class Config(object):
         CACHE_ROOT,
         "queries.json"
     )
+    LIMIT_QUERIES = True
 
 
 class DevelopmentConfig(Config):
     DEBUG = True
+    LIMIT_QUERIES = False

flask/nyt_api/__init__.py

@@ -107,6 +107,13 @@ def __init__(self, **kwargs):
         else:
             self.debug = False
 
+        # Limit Queries
+        if "limit_queries" in kwargs:
+            self.limit_queries = kwargs["limit_queries"]
+        else:
+            # Default to True since retrieving article data for more than 8000 results is prohibitive
+            self.limit_queries = True
+
         # Replace existing cache files
         if "replace_existing_cache_files" in kwargs:
             self.replace = kwargs["replace_existing_cache_files"]
@@ -143,7 +150,7 @@ def search(self, **query_params):
             if self.debug:
                 print "No hits for query '%s'" % query_params
             return None
-        elif ir_stats["hits"] > self.query_max_hits:
+        elif self.limit_queries and ir_stats["hits"] > self.query_max_hits:
             if self.debug:
                 print "More than %d hits (%d hits, to be exact) for query '%s'" % (self.query_max_hits, ir_stats["hits"], query_params)
             return None

flask/util/search.py

@@ -40,6 +40,7 @@
     ),
     search_api_key=Config.SEARCH_API_KEY,
     debug=Config.DEBUG,
+    limit_queries=Config.LIMIT_QUERIES,
     replace=True
 )