Exécutions
+Clique une exécution pour voir le détail des erreurs.
+diff --git a/api/main.py b/api/main.py index 6fbb3bf..2f6a7cb 100644 --- a/api/main.py +++ b/api/main.py @@ -57,6 +57,11 @@ def services_page(): return _read_page("services.html") +@app.get("/collecte", response_class=HTMLResponse) +def collecte_page(): + return _read_page("collecte.html") + + @app.get("/health") def health(): try: @@ -150,3 +155,42 @@ def api_resource_detail(provider: str, name: str, service_name: str): return queries.resource_detail(conn, provider, name, service_name) finally: conn.close() + + +@app.get("/api/collection-runs") +def api_collection_runs(limit: int = Query(50, ge=1, le=200)): + conn = queries.get_connection() + try: + return {"runs": queries.list_collection_runs(conn, limit=limit)} + finally: + conn.close() + + +@app.get("/api/collection-runs/alert-status") +def api_collection_runs_alert_status(): + conn = queries.get_connection() + try: + return {"has_unacknowledged_errors": queries.has_unacknowledged_errors(conn)} + finally: + conn.close() + + +@app.get("/api/collection-runs/{run_id}/errors") +def api_collection_run_errors(run_id: int): + conn = queries.get_connection() + try: + return {"errors": queries.collection_run_errors(conn, run_id)} + finally: + conn.close() + + +@app.post("/api/collection-runs/{run_id}/acknowledge") +def api_acknowledge_collection_run(run_id: int): + conn = queries.get_connection() + try: + acknowledged = queries.acknowledge_collection_run(conn, run_id) + if not acknowledged: + raise HTTPException(status_code=404, detail="Exécution introuvable ou déjà acquittée") + return {"status": "acknowledged"} + finally: + conn.close() diff --git a/api/queries.py b/api/queries.py index 33f7708..cc4a256 100644 --- a/api/queries.py +++ b/api/queries.py @@ -10,7 +10,7 @@ from __future__ import annotations import os import sqlite3 -from datetime import date +from datetime import date, datetime, timezone from pathlib import Path DEFAULT_DB_PATH = "cost_dashboard.db" @@ -656,3 +656,61 @@ def resource_detail(conn: sqlite3.Connection, provider: str, name: str, service_ "period_end": period_end, "resources": resources, } + + +# ---------------------------------------------------------------------- +# Exécutions de collecte (page "Collecte") +# ---------------------------------------------------------------------- + +def list_collection_runs(conn: sqlite3.Connection, limit: int = 50) -> list[dict]: + rows = conn.execute( + """ + SELECT id, run_type, started_at, finished_at, tenants_ok, tenants_error, acknowledged_at + FROM collection_runs + ORDER BY started_at DESC + LIMIT ? + """, + (limit,), + ).fetchall() + return [dict(r) for r in rows] + + +def collection_run_errors(conn: sqlite3.Connection, run_id: int) -> list[dict]: + """Détail des relevés en erreur d'une exécution — relu depuis + `collections` sur la fenêtre [started_at, finished_at] du run, plutôt que + dupliqué dans collection_runs.""" + run = conn.execute( + "SELECT started_at, finished_at FROM collection_runs WHERE id = ?", (run_id,) + ).fetchone() + if run is None: + return [] + + rows = conn.execute( + """ + SELECT provider, tenant_name, granularity, period_start, error_message, collected_at + FROM collections + WHERE status = 'error' AND collected_at BETWEEN ? AND ? + ORDER BY collected_at + """, + (run["started_at"], run["finished_at"] or run["started_at"]), + ).fetchall() + return [dict(r) for r in rows] + + +def has_unacknowledged_errors(conn: sqlite3.Connection) -> bool: + row = conn.execute( + "SELECT 1 FROM collection_runs WHERE tenants_error > 0 AND acknowledged_at IS NULL LIMIT 1" + ).fetchone() + return row is not None + + +def acknowledge_collection_run(conn: sqlite3.Connection, run_id: int) -> bool: + """Seule écriture de ce module (le reste est lecture seule) : marque une + exécution en erreur comme acquittée manuellement depuis le dashboard. + Retourne False si l'exécution n'existe pas ou était déjà acquittée.""" + cur = conn.execute( + "UPDATE collection_runs SET acknowledged_at = ? WHERE id = ? AND acknowledged_at IS NULL", + (datetime.now(timezone.utc).isoformat(), run_id), + ) + conn.commit() + return cur.rowcount > 0 diff --git a/collector/backfill_historical.py b/collector/backfill_historical.py index 2de2e28..718de94 100644 --- a/collector/backfill_historical.py +++ b/collector/backfill_historical.py @@ -51,7 +51,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) -from db import get_connection, init_db, upsert_collection +from db import get_connection, init_db, upsert_collection, start_collection_run, finish_collection_run from providers import get_aws_cost, get_azure_cost, convert_result from collect import collect_tenant, collect_tenant_resources, RESOURCE_WINDOW_DAYS @@ -89,8 +89,12 @@ def month_ranges(n_months: int) -> list[tuple[date, date]]: return list(reversed(months)) -def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, date]]) -> None: +def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, date]]) -> tuple[int, int]: + """Retourne (nb_relevés_ok, nb_relevés_en_erreur) — un mois vide (aucune + donnée) n'est compté ni comme un succès ni comme une erreur.""" name = entry["name"] + ok = 0 + errors = 0 for start, end in months: collected_at = datetime.now(timezone.utc).isoformat() month_label = start.isoformat()[:7] @@ -113,15 +117,19 @@ def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, d conn, provider, name, "monthly", start.isoformat(), end.isoformat(), collected_at, "ok", data=data, ) + ok += 1 log.info("OK %-6s %-30s %s %.2f %s", provider, name, month_label, data["total"], data["currency"]) except Exception as e: + errors += 1 log.error("ECHEC %-6s %-30s %s : %s", provider, name, month_label, e) finally: if provider == "azure": time.sleep(AZURE_CALL_DELAY_SECONDS) + return ok, errors + def day_ranges(n_days: int) -> list[date]: """Les n derniers jours (le plus ancien en premier), jusqu'à hier inclus @@ -131,25 +139,32 @@ def day_ranges(n_days: int) -> list[date]: return [yesterday - timedelta(days=i) for i in range(n_days - 1, -1, -1)] -def backfill_tenant_daily(conn, provider: str, entry: dict, days: list[date]) -> None: +def backfill_tenant_daily(conn, provider: str, entry: dict, days: list[date]) -> tuple[int, int]: """Rejoue collect_tenant (collect.py) sur chaque jour de `days`, puis collect_tenant_resources une fois si au moins un jour a réussi — exactement - ce que ferait le cron quotidien, jour après jour, mais en une passe.""" + ce que ferait le cron quotidien, jour après jour, mais en une passe. + Retourne (nb_jours_ok, nb_jours_en_erreur).""" if not days: - return + return 0, 0 - any_ok = False + ok_count = 0 + error_count = 0 for day in days: ok = collect_tenant(conn, provider, entry, day.isoformat(), (day + timedelta(days=1)).isoformat()) - any_ok = any_ok or ok + if ok: + ok_count += 1 + else: + error_count += 1 if provider == "azure": time.sleep(AZURE_CALL_DELAY_SECONDS) - if any_ok: + if ok_count > 0: resource_end = days[-1] + timedelta(days=1) resource_start = resource_end - timedelta(days=RESOURCE_WINDOW_DAYS) collect_tenant_resources(conn, provider, entry, resource_start.isoformat(), resource_end.isoformat()) + return ok_count, error_count + def main(): parser = argparse.ArgumentParser(description="Backfill de l'historique de coûts depuis AWS/Azure") @@ -183,13 +198,20 @@ def main(): if days: log.info("Backfill journalier de %d jours (%s -> %s)", len(days), days[0].isoformat(), days[-1].isoformat()) + run_id = start_collection_run(conn, "backfill") + total_ok = 0 + total_error = 0 + for provider, entries in (("aws", config.get("aws", [])), ("azure", config.get("azure", []))): for entry in entries: if args.tenant and entry["name"] != args.tenant: continue - backfill_tenant(conn, provider, entry, months) - backfill_tenant_daily(conn, provider, entry, days) + ok1, err1 = backfill_tenant(conn, provider, entry, months) + ok2, err2 = backfill_tenant_daily(conn, provider, entry, days) + total_ok += ok1 + ok2 + total_error += err1 + err2 + finish_collection_run(conn, run_id, total_ok, total_error) conn.close() log.info("Terminé.") diff --git a/collector/collect.py b/collector/collect.py index f89e0d8..c250358 100644 --- a/collector/collect.py +++ b/collector/collect.py @@ -29,6 +29,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from db import ( get_connection, init_db, upsert_collection, upsert_resource_costs, set_resource_collection_status, + start_collection_run, finish_collection_run, ) from providers import ( get_aws_cost, get_azure_cost, convert_result, @@ -148,15 +149,23 @@ def main(): resource_period_end = target_day + timedelta(days=1) resource_period_start = resource_period_end - timedelta(days=RESOURCE_WINDOW_DAYS) + run_id = start_collection_run(conn, "daily") + tenants_ok = 0 + tenants_error = 0 + for provider, entries in (("aws", config.get("aws", [])), ("azure", config.get("azure", []))): for entry in entries: ok = collect_tenant(conn, provider, entry, period_start, period_end) if ok: + tenants_ok += 1 collect_tenant_resources( conn, provider, entry, resource_period_start.isoformat(), resource_period_end.isoformat(), ) + else: + tenants_error += 1 + finish_collection_run(conn, run_id, tenants_ok, tenants_error) conn.close() log.info("Terminé.") diff --git a/collector/db.py b/collector/db.py index 46d1774..5bdb7d0 100644 --- a/collector/db.py +++ b/collector/db.py @@ -10,6 +10,7 @@ from __future__ import annotations import os import sqlite3 +from datetime import datetime, timezone from pathlib import Path DEFAULT_DB_PATH = "cost_dashboard.db" @@ -134,6 +135,30 @@ def set_resource_collection_status( conn.commit() +def start_collection_run(conn: sqlite3.Connection, run_type: str) -> int: + """Enregistre le début d'une exécution de collecte (page "Collecte" du + dashboard). `run_type` : 'daily' (collect.py) ou 'backfill' + (backfill_historical.py).""" + cur = conn.execute( + "INSERT INTO collection_runs (run_type, started_at) VALUES (?, ?)", + (run_type, datetime.now(timezone.utc).isoformat()), + ) + conn.commit() + return cur.lastrowid + + +def finish_collection_run(conn: sqlite3.Connection, run_id: int, tenants_ok: int, tenants_error: int) -> None: + conn.execute( + """ + UPDATE collection_runs + SET finished_at = ?, tenants_ok = ?, tenants_error = ? + WHERE id = ? + """, + (datetime.now(timezone.utc).isoformat(), tenants_ok, tenants_error, run_id), + ) + conn.commit() + + def upsert_resource_costs( conn: sqlite3.Connection, provider: str, diff --git a/db/schema.sql b/db/schema.sql index e1fc2d7..c68407d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -74,3 +74,23 @@ CREATE TABLE IF NOT EXISTS resource_collection_status ( collected_at TIMESTAMP NOT NULL, PRIMARY KEY (provider, tenant_name) ); + +-- Historique des exécutions de tâches de collecte (page "Collecte" du +-- dashboard) : une ligne par lancement de collect.py (run_type='daily') ou +-- backfill_historical.py (run_type='backfill'). Le détail des tenants en +-- erreur n'est pas dupliqué ici : il se relit dans `collections` +-- (status='error') sur la fenêtre [started_at, finished_at] de l'exécution. +-- acknowledged_at reste NULL tant qu'une erreur n'a pas été acquittée +-- manuellement depuis le dashboard. +CREATE TABLE IF NOT EXISTS collection_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_type TEXT NOT NULL CHECK(run_type IN ('daily', 'backfill')), + started_at TIMESTAMP NOT NULL, + finished_at TIMESTAMP, + tenants_ok INTEGER NOT NULL DEFAULT 0, + tenants_error INTEGER NOT NULL DEFAULT 0, + acknowledged_at TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_collection_runs_started + ON collection_runs(started_at); diff --git a/frontend/collecte.html b/frontend/collecte.html new file mode 100644 index 0000000..75c36ec --- /dev/null +++ b/frontend/collecte.html @@ -0,0 +1,61 @@ + + +
+ + +Exécutions
+Clique une exécution pour voir le détail des erreurs.
+Azure — mois en cours
—
Tenants en échec de collecte
-—
-Évolution mensuelle (12 mois glissants)
diff --git a/frontend/services.html b/frontend/services.html index c33464c..9a25160 100644 --- a/frontend/services.html +++ b/frontend/services.html @@ -14,6 +14,7 @@collect.py ou backfill_historical.py.| Type | +Démarrée | +Durée | +OK | +Erreurs | +Statut | +
|---|
Acquittée le ${formatDateTime(run.acknowledged_at)}
` + : ``; + + bodyEl.innerHTML = `${ackBlock}${list}`; + + const btn = document.getElementById("ack-button"); + if (btn) { + btn.addEventListener("click", () => acknowledgeRun(run.id)); + } +} + +async function acknowledgeRun(runId) { + const btn = document.getElementById("ack-button"); + if (btn) btn.disabled = true; + + try { + const res = await fetch(`/api/collection-runs/${runId}/acknowledge`, { method: "POST" }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + closeRunDrawer(); + await init(); + refreshCollecteAlert(); + } catch (e) { + document.getElementById("drawer-body").innerHTML = + ``; + } +} + +function closeRunDrawer() { + document.getElementById("run-drawer").classList.remove("open"); + document.getElementById("run-drawer").setAttribute("aria-hidden", "true"); + document.getElementById("drawer-backdrop").classList.remove("open"); +} + +function setupDrawer() { + document.getElementById("drawer-close").addEventListener("click", closeRunDrawer); + document.getElementById("drawer-backdrop").addEventListener("click", closeRunDrawer); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape") closeRunDrawer(); + }); +} + +async function init() { + try { + const data = await loadRuns(); + currentRuns = data.runs; + renderRunsTable(); + } catch (e) { + document.getElementById("error-banner").innerHTML = ` + `; + document.getElementById("runs-table-wrap").innerHTML = ""; + } +} + +init(); +setupDrawer(); diff --git a/frontend/static/common.js b/frontend/static/common.js index 89924fb..6dcf4a4 100644 --- a/frontend/static/common.js +++ b/frontend/static/common.js @@ -63,6 +63,24 @@ function cssVar(name) { return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); } +// Pastille d'alerte sur le lien "Collecte" de la sidebar (présente sur +// toutes les pages) — signale une erreur de collecte pas encore acquittée. +// La page Collecte elle-même se rafraîchit après un acquittement. +async function refreshCollecteAlert() { + const dot = document.getElementById("nav-collecte-alert"); + if (!dot) return; + try { + const res = await fetch("/api/collection-runs/alert-status"); + if (!res.ok) return; + const data = await res.json(); + dot.hidden = !data.has_unacknowledged_errors; + } catch (e) { + // silencieux : une pastille ratée ne doit pas bloquer le reste de la page + } +} + +refreshCollecteAlert(); + // Config partagée des axes/grilles Chart.js (gridlines discrètes, pas de // deuxième axe, tooltip alignée sur le design system). function chartBaseOptions() { diff --git a/frontend/static/style.css b/frontend/static/style.css index 9516636..be0c097 100644 --- a/frontend/static/style.css +++ b/frontend/static/style.css @@ -180,7 +180,9 @@ a { } .sidebar-link { - display: block; + display: flex; + align-items: center; + gap: 7px; padding: 9px 12px; border-radius: var(--radius-control); color: var(--text-secondary); @@ -189,6 +191,14 @@ a { font-weight: 500; } +.nav-alert-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--status-critical); + flex: none; +} + .sidebar-link:hover { background: color-mix(in srgb, var(--text-primary) 6%, transparent); color: var(--text-primary); @@ -527,6 +537,7 @@ table.data-table { .status-dot.ok { background: var(--status-good); } .status-dot.error { background: var(--status-critical); } +.status-dot.warning { background: var(--status-warning); } .status-dot.unknown { background: var(--text-muted); } .status-label { @@ -660,6 +671,25 @@ table.data-table { padding: 8px 20px 20px; } +.btn { + display: inline-flex; + align-items: center; + gap: 6px; + border: 1px solid var(--border); + background: var(--surface-1); + color: var(--text-primary); + font-size: 13px; + font-weight: 500; + padding: 8px 14px; + border-radius: var(--radius-control); + cursor: pointer; + font-family: inherit; +} + +.btn:hover { + background: color-mix(in srgb, var(--text-primary) 6%, transparent); +} + .resource-row { display: flex; align-items: center; diff --git a/frontend/tenant.html b/frontend/tenant.html index 5da3b15..3a520ad 100644 --- a/frontend/tenant.html +++ b/frontend/tenant.html @@ -15,6 +15,7 @@