feat: implement collection runs tracking and error acknowledgment features

collector/backfill_historical.py

@@ -51,7 +51,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent))
 
-from db import get_connection, init_db, upsert_collection
+from db import get_connection, init_db, upsert_collection, start_collection_run, finish_collection_run
 from providers import get_aws_cost, get_azure_cost, convert_result
 from collect import collect_tenant, collect_tenant_resources, RESOURCE_WINDOW_DAYS
 
@@ -89,8 +89,12 @@ def month_ranges(n_months: int) -> list[tuple[date, date]]:
     return list(reversed(months))
 
 
-def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, date]]) -> None:
+def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, date]]) -> tuple[int, int]:
+    """Retourne (nb_relevés_ok, nb_relevés_en_erreur) — un mois vide (aucune
+    donnée) n'est compté ni comme un succès ni comme une erreur."""
     name = entry["name"]
+    ok = 0
+    errors = 0
     for start, end in months:
         collected_at = datetime.now(timezone.utc).isoformat()
         month_label = start.isoformat()[:7]
@@ -113,15 +117,19 @@ def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, d
                 conn, provider, name, "monthly", start.isoformat(), end.isoformat(),
                 collected_at, "ok", data=data,
             )
+            ok += 1
             log.info("OK    %-6s %-30s %s  %.2f %s", provider, name, month_label, data["total"], data["currency"])
 
         except Exception as e:
+            errors += 1
             log.error("ECHEC %-6s %-30s %s : %s", provider, name, month_label, e)
 
         finally:
             if provider == "azure":
                 time.sleep(AZURE_CALL_DELAY_SECONDS)
 
+    return ok, errors
+
 
 def day_ranges(n_days: int) -> list[date]:
     """Les n derniers jours (le plus ancien en premier), jusqu'à hier inclus
@@ -131,25 +139,32 @@ def day_ranges(n_days: int) -> list[date]:
     return [yesterday - timedelta(days=i) for i in range(n_days - 1, -1, -1)]
 
 
-def backfill_tenant_daily(conn, provider: str, entry: dict, days: list[date]) -> None:
+def backfill_tenant_daily(conn, provider: str, entry: dict, days: list[date]) -> tuple[int, int]:
     """Rejoue collect_tenant (collect.py) sur chaque jour de `days`, puis
     collect_tenant_resources une fois si au moins un jour a réussi — exactement
-    ce que ferait le cron quotidien, jour après jour, mais en une passe."""
+    ce que ferait le cron quotidien, jour après jour, mais en une passe.
+    Retourne (nb_jours_ok, nb_jours_en_erreur)."""
     if not days:
-        return
+        return 0, 0
 
-    any_ok = False
+    ok_count = 0
+    error_count = 0
     for day in days:
         ok = collect_tenant(conn, provider, entry, day.isoformat(), (day + timedelta(days=1)).isoformat())
-        any_ok = any_ok or ok
+        if ok:
+            ok_count += 1
+        else:
+            error_count += 1
         if provider == "azure":
             time.sleep(AZURE_CALL_DELAY_SECONDS)
 
-    if any_ok:
+    if ok_count > 0:
         resource_end = days[-1] + timedelta(days=1)
         resource_start = resource_end - timedelta(days=RESOURCE_WINDOW_DAYS)
         collect_tenant_resources(conn, provider, entry, resource_start.isoformat(), resource_end.isoformat())
 
+    return ok_count, error_count
+
 
 def main():
     parser = argparse.ArgumentParser(description="Backfill de l'historique de coûts depuis AWS/Azure")
@@ -183,13 +198,20 @@ def main():
     if days:
         log.info("Backfill journalier de %d jours (%s -> %s)", len(days), days[0].isoformat(), days[-1].isoformat())
 
+    run_id = start_collection_run(conn, "backfill")
+    total_ok = 0
+    total_error = 0
+
     for provider, entries in (("aws", config.get("aws", [])), ("azure", config.get("azure", []))):
         for entry in entries:
             if args.tenant and entry["name"] != args.tenant:
                 continue
-            backfill_tenant(conn, provider, entry, months)
-            backfill_tenant_daily(conn, provider, entry, days)
+            ok1, err1 = backfill_tenant(conn, provider, entry, months)
+            ok2, err2 = backfill_tenant_daily(conn, provider, entry, days)
+            total_ok += ok1 + ok2
+            total_error += err1 + err2
 
+    finish_collection_run(conn, run_id, total_ok, total_error)
     conn.close()
     log.info("Terminé.")