Simplify scraper task

Author: jpadilla

contratospr/contracts/management/commands/scrape_contracts.py

@@ -7,6 +7,7 @@ class Command(BaseCommand):
     help = "Scrape search results"
 
     def add_arguments(self, parser):
+        parser.add_argument("--max-items", nargs="?", type=int, default=None)
         parser.add_argument("--limit", nargs="?", type=int, default=None)
         parser.add_argument("--date-of-grant-start", nargs="?", type=str, default=None)
         parser.add_argument("--date-of-grant-end", nargs="?", type=str, default=None)
@@ -22,4 +23,5 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         limit = options.pop("limit", None)
-        scrape_contracts.delay(limit=limit, **options)
+        max_items = options.pop("max_items", None)
+        scrape_contracts.delay(limit=limit, max_items=max_items, **options)

contratospr/contracts/scraper.py

@@ -135,7 +135,7 @@ def get_contracts(offset, limit, **kwargs):
                     "search": {"value": "", "regex": False},
                 },
             ],
-            "order": [{"column": 3, "dir": "desc"}, {"column": 6, "dir": "desc"}],
+            "order": [{"column": 1, "dir": "desc"}],
             "start": offset,
             "length": limit,
             "EntityId": kwargs.get("entity_id"),

contratospr/contracts/tasks.py

@@ -2,6 +2,7 @@
 import re
 
 import pytz
+from structlog import get_logger
 
 from ..tasks import app
 from .models import Contract, Contractor, Document, Entity, Service, ServiceGroup
@@ -14,6 +15,8 @@
 )
 from .search import index_contract
 
+logger = get_logger(__name__)
+
 
 def parse_date(value):
     if not value:
@@ -29,6 +32,8 @@ def strip_whitespace(value):
 
 @app.task
 def expand_contract(contract):
+    logger.info("Expanding contract", contract=contract["ContractNumber"])
+
     result = {
         "entity_id": contract["EntityId"],
         "entity_name": strip_whitespace(contract["EntityName"]),
@@ -118,6 +123,10 @@ def request_contract_document(contract_id):
 
 @app.task
 def update_contract(result, parent_id=None):
+    logger.info(
+        "Updating contract", contract=result["contract_number"], parent_id=parent_id
+    )
+
     entity, _ = Entity.objects.get_or_create(
         source_id=result["entity_id"], defaults={"name": result["entity_name"]}
     )
@@ -175,20 +184,28 @@ def update_contract(result, parent_id=None):
 
 
 @app.task
-def scrape_contracts(limit=None, **kwargs):
+def scrape_contracts(limit=None, max_items=None, **kwargs):
     offset = 0
     total_records = 0
-    default_limit = 1000
+    default_limit = 10
+    real_limit = limit or default_limit
 
     while offset <= total_records:
-        real_limit = limit or default_limit
+        logger.info(
+            "Scraping contracts",
+            limit=limit,
+            real_limit=real_limit,
+            offset=offset,
+            total_records=total_records,
+        )
+
         contracts = get_contracts(offset, real_limit, **kwargs)
 
         if not total_records:
-            total_records = limit if limit else contracts["recordsFiltered"]
+            total_records = max_items if max_items else contracts["recordsFiltered"]
 
         for contract in contracts["data"]:
-            chain = expand_contract.s(contract) | update_contract.s()
-            chain()
+            expanded = expand_contract(contract)
+            update_contract(expanded)
 
         offset += real_limit

contratospr/tasks.py

@@ -1,7 +1,9 @@
 import os
 
 import configurations
+import structlog
 from celery import Celery
+from celery.signals import task_prerun
 
 configuration = os.getenv("ENVIRONMENT", "development").title()
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "contratospr.settings")
@@ -11,6 +13,13 @@
 
 from django.conf import settings  # noqa isort:skip
 
+
+@task_prerun.connect
+def configure_structlog(sender, body=None, **kwargs):
+    logger = structlog.get_logger("contratospr.tasks")
+    logger.new(task_id=kwargs["task_id"], task_name=sender.__name__)
+
+
 app = Celery("contratospr")
 app.config_from_object(settings, namespace="CELERY")
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)