Add utils and docs for downloading all contracts

Author: jpadilla

.gitignore

@@ -94,5 +94,7 @@ media
 node_modules
 
 .vscode/
-*.dump
 .idea/
+
+docker/postgres/backup.dump
+data/contracts.jsonl.gz

CONTRIBUTING.md

@@ -87,9 +87,9 @@ If you want to go the usual route and run the project locally, though:
 - Install pre-commit hooks: `pipenv run pre-commit install`
 - Setup environment file: `cp example.env .env`
 - Install docker-compose: https://docs.docker.com/compose/install/
+- [Download latest database dump](docs/database-dump.md)
 - Build and start containers, including test container: `docker-compose up --build -d`
 - Create super user: `docker-compose exec web python manage.py createsuperuser`
-- Scrape some contracts: `docker-compose exec web python manage.py scrape_contracts --limit 100`
 - open http://localhost:8000
 
 And you should be ready to go!

bin/docker-entrypoint

@@ -6,9 +6,16 @@ postgres_ready(){
 python manage.py shell << END
 import sys
 import psycopg2
-from django.db import connections
+import dj_database_url
 try:
-    connections['default'].cursor()
+    parsed = dj_database_url.parse("${DATABASE_URL}")
+    psycopg2.connect(
+        dbname=parsed["NAME"],
+        user=parsed["USER"],
+        password=parsed["PASSWORD"],
+        host=parsed["HOST"],
+        port=parsed["PORT"],
+    )
 except psycopg2.OperationalError:
     sys.exit(-1)
 sys.exit(0)
@@ -35,6 +42,11 @@ case "$1" in
     ;;
 
   "run-tests")
+    until postgres_ready; do
+      >&2 echo "==> Waiting for Postgres..."
+      sleep 1
+    done
+
     python runtests.py
     ;;
 

contratospr/contracts/management/commands/download_contracts.py

@@ -0,0 +1,94 @@
+import json
+
+from django.core.management.base import BaseCommand
+from structlog import get_logger
+
+from ...scraper import get_amendments, get_contractors, get_contracts, get_entities
+
+logger = get_logger("contratospr.commands.download_contracts")
+
+
+def get_contracts_by_entity(entity):
+    offset = 0
+    total_records = 0
+    limit = 1000
+
+    entity_id = entity["Code"]
+    entity_name = entity["Name"].strip()
+
+    while offset <= total_records:
+        logger.info(
+            "Scraping contracts",
+            limit=limit,
+            entity_id=entity_id,
+            entity_name=entity_name,
+            offset=offset,
+            total_records=total_records,
+        )
+
+        contracts_json = get_contracts(offset, limit, entity_id=entity_id)
+
+        with open(f"data/contracts-{entity_id}-{offset}.json", "w+") as f:
+            json.dump(contracts_json, f)
+
+        expanded_contracts = []
+
+        for contract_data in contracts_json.get("data", []):
+            try:
+                logger.info(
+                    "Getting contractors", contract_id=contract_data["ContractId"]
+                )
+                contract_data["_Contractors"] = get_contractors(
+                    contract_data["ContractId"]
+                )
+                contract_data["_Amendments"] = None
+
+                if contract_data["HasAmendments"]:
+                    logger.info(
+                        "Getting amendments",
+                        contract_number=contract_data["ContractNumber"],
+                        entity_id=contract_data["EntityId"],
+                    )
+                    contract_data["_Amendments"] = get_amendments(
+                        contract_data["ContractNumber"], contract_data["EntityId"]
+                    )
+            except Exception as exc:
+                logger.info(
+                    "Error extending contract",
+                    contract_id=contract_data["ContractId"],
+                    exception=exc,
+                )
+
+            expanded_contracts.append(contract_data)
+
+        contracts_json["data"] = expanded_contracts
+
+        with open(f"data/contracts-{entity_id}-{offset}.json", "w+") as f:
+            json.dump(contracts_json, f)
+
+        if not total_records:
+            total_records = contracts_json["recordsFiltered"]
+
+        offset += limit
+
+
+def get_contracts_by_entities(entities):
+    for entity in entities:
+        get_contracts_by_entity(entity)
+
+
+class Command(BaseCommand):
+    help = "Download contracts for entities from consultacontratos.ocpr.gov.pr"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--file", nargs="?", type=str, default=None)
+
+    def handle(self, *args, **options):
+        entities_file_path = options.get("file")
+
+        if entities_file_path:
+            entities = json.load(open(entities_file_path)).get("Results", [])
+        else:
+            entities = get_entities()
+
+        get_contracts_by_entities(entities)

contratospr/contracts/management/commands/import_contracts.py

@@ -0,0 +1,49 @@
+import gzip
+import json
+import os
+
+from django.core.management.base import BaseCommand
+from structlog import get_logger
+
+from ...tasks import normalize_contract, normalize_contractors, update_contract
+
+logger = get_logger("contratospr.commands.import_contracts")
+
+
+def _normalize_contract(contract):
+    normalized_contract = normalize_contract(contract)
+    normalized_contract["contractors"] = normalize_contractors(
+        contract.get("_Contractors", [])
+    )
+
+    if contract.get("_Amendments"):
+        for amendment in contract.get("_Amendments", []):
+            normalized_contract["amendments"].append(_normalize_contract(amendment))
+
+    return normalized_contract
+
+
+def import_contracts(contracts):
+    for contract in contracts:
+        normalized = _normalize_contract(contract)
+
+        logger.info(
+            "Importing contract",
+            contract_id=normalized["contract_id"],
+            entity_id=normalized["entity_id"],
+        )
+
+        update_contract(normalized)
+
+
+class Command(BaseCommand):
+    help = "Import contracts"
+
+    def handle(self, *args, **options):
+        gzipped_merged_file_name = "contracts.jsonl.gz"
+        gzipped_merged_file_path = os.path.join("data", gzipped_merged_file_name)
+
+        with gzip.open(gzipped_merged_file_path, "r") as f:
+            for jsonline in f:
+                contracts_data = json.loads(jsonline)
+                import_contracts(contracts_data)

contratospr/contracts/management/commands/merge_contracts.py

@@ -0,0 +1,44 @@
+import gzip
+import json
+import os
+import shutil
+
+from django.core.management.base import BaseCommand
+from structlog import get_logger
+
+logger = get_logger("contratospr.commands.merge_contracts")
+
+
+class Command(BaseCommand):
+    help = "Merge contracts"
+
+    def handle(self, *args, **options):
+        merged_file_name = "contracts.json"
+        merged_file_path = os.path.join("data", merged_file_name)
+        gzipped_merged_file_name = "contracts.jsonl.gz"
+        gzipped_merged_file_path = os.path.join("data", gzipped_merged_file_name)
+
+        for filename in os.listdir("data"):
+            file_path = os.path.join("data", filename)
+
+            if (
+                os.path.isfile(file_path)
+                and file_path.endswith(".json")
+                and file_path != merged_file_name
+                and file_path != gzipped_merged_file_name
+            ):
+                logger.info("Merging contracts", filename=filename)
+
+                with open(file_path) as f:
+                    contracts_json = json.load(f)
+
+                contracts_data = contracts_json.get("data", [])
+
+                json_str = f"{json.dumps(contracts_data)}\n"
+
+                with open(merged_file_path, "a+") as f:
+                    f.write(json_str)
+
+        with open(merged_file_path, "rb") as f_in:
+            with gzip.open(gzipped_merged_file_path, "wb") as f_out:
+                shutil.copyfileobj(f_in, f_out)

contratospr/contracts/scraper.py

@@ -2,6 +2,8 @@
 
 import requests
 
+from ..utils.requests_retry import requests_retry_session
+
 BASE_URL = "https://consultacontratos.ocpr.gov.pr"
 BASE_CONTRACT_URL = f"{BASE_URL}/contract"
 BASE_CONTRACTOR_URL = f"{BASE_URL}/contractor"
@@ -21,6 +23,9 @@
 ]
 
 
+session = requests_retry_session(retries=3, backoff_factor=0.3)
+
+
 def send_document_request(contract_id):
     response = requests.post(
         f"{BASE_CONTRACT_URL}/senddocumentrequest",
@@ -32,7 +37,7 @@ def send_document_request(contract_id):
 
 
 def get_contractors(contract_id):
-    response = requests.post(
+    response = session.post(
         f"{BASE_CONTRACTOR_URL}/findbycontractid",
         json={"contractId": contract_id},
         headers={"user-agent": random.choice(USER_AGENTS)},
@@ -42,7 +47,7 @@ def get_contractors(contract_id):
 
 
 def get_amendments(contract_number, entity_id):
-    response = requests.post(
+    response = session.post(
         f"{BASE_CONTRACT_URL}/getamendments",
         json={"contractNumber": contract_number, "entityId": entity_id},
         headers={"user-agent": random.choice(USER_AGENTS)},
@@ -52,7 +57,7 @@ def get_amendments(contract_number, entity_id):
 
 
 def get_contracts(offset, limit, **kwargs):
-    response = requests.post(
+    response = session.post(
         f"{BASE_CONTRACT_URL}/search",
         json={
             "draw": 1,
@@ -154,3 +159,12 @@ def get_contracts(offset, limit, **kwargs):
     )
 
     return response.json()
+
+
+def get_entities():
+    response = session.get(
+        f"{BASE_URL}/entity/findby?name=&pageIndex=1&pageSize=1000",
+        headers={"user-agent": random.choice(USER_AGENTS)},
+    )
+
+    return response.json().get("Results", [])

contratospr/contracts/tasks.py

@@ -30,10 +30,7 @@ def strip_whitespace(value):
     return value.strip() if value else None
 
 
-@app.task
-def expand_contract(contract):
-    logger.info("Expanding contract", contract=contract["ContractNumber"])
-
+def normalize_contract(contract):
     result = {
         "entity_id": contract["EntityId"],
         "entity_name": strip_whitespace(contract["EntityName"]),
@@ -60,17 +57,34 @@ def expand_contract(contract):
             "document_url"
         ] = f"{BASE_CONTRACT_URL}/downloaddocument?documentid={document_id}"
 
-    contractors = get_contractors(result["contract_id"])
+    return result
+
+
+def normalize_contractors(contractors):
+    results = []
 
     for contractor in contractors:
-        result["contractors"].append(
+        results.append(
             {
                 "contractor_id": contractor["ContractorId"],
                 "entity_id": contractor["EntityId"],
                 "name": contractor["Name"],
             }
         )
 
+    return results
+
+
+@app.task
+def expand_contract(contract):
+    logger.info("Expanding contract", contract=contract["ContractNumber"])
+
+    result = normalize_contract(contract)
+
+    contractors = get_contractors(result["contract_id"])
+
+    result["contractors"] = normalize_contractors(contractors)
+
     if result["has_amendments"]:
         amendments = get_amendments(result["contract_number"], result["entity_id"])
 

contratospr/utils/requests_retry.py

@@ -0,0 +1,21 @@
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+
+# From https://www.peterbe.com/plog/best-practice-with-retries-with-requests
+def requests_retry_session(
+    retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504), session=None
+):
+    session = session or requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session

docker-compose.yml

@@ -11,9 +11,12 @@ services:
   database:
     image: postgres:10.6
     environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=postgres
       - POSTGRES_PASSWORD=password
     volumes:
       - database:/var/lib/postgresql/data
+      - ./docker/postgres:/docker-entrypoint-initdb.d
 
   redis:
     image: redis:3.2
@@ -71,6 +74,7 @@ services:
     command: run-tests
     entrypoint: /app/bin/docker-entrypoint
     environment:
+      - ENVIRONMENT=testing
       - DATABASE_URL=postgres://postgres:password@database/postgres
       - REDIS_URL=redis://redis:6379
     volumes: