diff --git a/api/main.py b/api/main.py index 6fbb3bf..2f6a7cb 100644 --- a/api/main.py +++ b/api/main.py @@ -57,6 +57,11 @@ def services_page(): return _read_page("services.html") +@app.get("/collecte", response_class=HTMLResponse) +def collecte_page(): + return _read_page("collecte.html") + + @app.get("/health") def health(): try: @@ -150,3 +155,42 @@ def api_resource_detail(provider: str, name: str, service_name: str): return queries.resource_detail(conn, provider, name, service_name) finally: conn.close() + + +@app.get("/api/collection-runs") +def api_collection_runs(limit: int = Query(50, ge=1, le=200)): + conn = queries.get_connection() + try: + return {"runs": queries.list_collection_runs(conn, limit=limit)} + finally: + conn.close() + + +@app.get("/api/collection-runs/alert-status") +def api_collection_runs_alert_status(): + conn = queries.get_connection() + try: + return {"has_unacknowledged_errors": queries.has_unacknowledged_errors(conn)} + finally: + conn.close() + + +@app.get("/api/collection-runs/{run_id}/errors") +def api_collection_run_errors(run_id: int): + conn = queries.get_connection() + try: + return {"errors": queries.collection_run_errors(conn, run_id)} + finally: + conn.close() + + +@app.post("/api/collection-runs/{run_id}/acknowledge") +def api_acknowledge_collection_run(run_id: int): + conn = queries.get_connection() + try: + acknowledged = queries.acknowledge_collection_run(conn, run_id) + if not acknowledged: + raise HTTPException(status_code=404, detail="Exécution introuvable ou déjà acquittée") + return {"status": "acknowledged"} + finally: + conn.close() diff --git a/api/queries.py b/api/queries.py index 33f7708..cc4a256 100644 --- a/api/queries.py +++ b/api/queries.py @@ -10,7 +10,7 @@ from __future__ import annotations import os import sqlite3 -from datetime import date +from datetime import date, datetime, timezone from pathlib import Path DEFAULT_DB_PATH = "cost_dashboard.db" @@ -656,3 +656,61 @@ def resource_detail(conn: sqlite3.Connection, provider: str, name: str, service_ "period_end": period_end, "resources": resources, } + + +# ---------------------------------------------------------------------- +# Exécutions de collecte (page "Collecte") +# ---------------------------------------------------------------------- + +def list_collection_runs(conn: sqlite3.Connection, limit: int = 50) -> list[dict]: + rows = conn.execute( + """ + SELECT id, run_type, started_at, finished_at, tenants_ok, tenants_error, acknowledged_at + FROM collection_runs + ORDER BY started_at DESC + LIMIT ? + """, + (limit,), + ).fetchall() + return [dict(r) for r in rows] + + +def collection_run_errors(conn: sqlite3.Connection, run_id: int) -> list[dict]: + """Détail des relevés en erreur d'une exécution — relu depuis + `collections` sur la fenêtre [started_at, finished_at] du run, plutôt que + dupliqué dans collection_runs.""" + run = conn.execute( + "SELECT started_at, finished_at FROM collection_runs WHERE id = ?", (run_id,) + ).fetchone() + if run is None: + return [] + + rows = conn.execute( + """ + SELECT provider, tenant_name, granularity, period_start, error_message, collected_at + FROM collections + WHERE status = 'error' AND collected_at BETWEEN ? AND ? + ORDER BY collected_at + """, + (run["started_at"], run["finished_at"] or run["started_at"]), + ).fetchall() + return [dict(r) for r in rows] + + +def has_unacknowledged_errors(conn: sqlite3.Connection) -> bool: + row = conn.execute( + "SELECT 1 FROM collection_runs WHERE tenants_error > 0 AND acknowledged_at IS NULL LIMIT 1" + ).fetchone() + return row is not None + + +def acknowledge_collection_run(conn: sqlite3.Connection, run_id: int) -> bool: + """Seule écriture de ce module (le reste est lecture seule) : marque une + exécution en erreur comme acquittée manuellement depuis le dashboard. + Retourne False si l'exécution n'existe pas ou était déjà acquittée.""" + cur = conn.execute( + "UPDATE collection_runs SET acknowledged_at = ? WHERE id = ? AND acknowledged_at IS NULL", + (datetime.now(timezone.utc).isoformat(), run_id), + ) + conn.commit() + return cur.rowcount > 0 diff --git a/collector/backfill_historical.py b/collector/backfill_historical.py index 2de2e28..718de94 100644 --- a/collector/backfill_historical.py +++ b/collector/backfill_historical.py @@ -51,7 +51,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) -from db import get_connection, init_db, upsert_collection +from db import get_connection, init_db, upsert_collection, start_collection_run, finish_collection_run from providers import get_aws_cost, get_azure_cost, convert_result from collect import collect_tenant, collect_tenant_resources, RESOURCE_WINDOW_DAYS @@ -89,8 +89,12 @@ def month_ranges(n_months: int) -> list[tuple[date, date]]: return list(reversed(months)) -def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, date]]) -> None: +def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, date]]) -> tuple[int, int]: + """Retourne (nb_relevés_ok, nb_relevés_en_erreur) — un mois vide (aucune + donnée) n'est compté ni comme un succès ni comme une erreur.""" name = entry["name"] + ok = 0 + errors = 0 for start, end in months: collected_at = datetime.now(timezone.utc).isoformat() month_label = start.isoformat()[:7] @@ -113,15 +117,19 @@ def backfill_tenant(conn, provider: str, entry: dict, months: list[tuple[date, d conn, provider, name, "monthly", start.isoformat(), end.isoformat(), collected_at, "ok", data=data, ) + ok += 1 log.info("OK %-6s %-30s %s %.2f %s", provider, name, month_label, data["total"], data["currency"]) except Exception as e: + errors += 1 log.error("ECHEC %-6s %-30s %s : %s", provider, name, month_label, e) finally: if provider == "azure": time.sleep(AZURE_CALL_DELAY_SECONDS) + return ok, errors + def day_ranges(n_days: int) -> list[date]: """Les n derniers jours (le plus ancien en premier), jusqu'à hier inclus @@ -131,25 +139,32 @@ def day_ranges(n_days: int) -> list[date]: return [yesterday - timedelta(days=i) for i in range(n_days - 1, -1, -1)] -def backfill_tenant_daily(conn, provider: str, entry: dict, days: list[date]) -> None: +def backfill_tenant_daily(conn, provider: str, entry: dict, days: list[date]) -> tuple[int, int]: """Rejoue collect_tenant (collect.py) sur chaque jour de `days`, puis collect_tenant_resources une fois si au moins un jour a réussi — exactement - ce que ferait le cron quotidien, jour après jour, mais en une passe.""" + ce que ferait le cron quotidien, jour après jour, mais en une passe. + Retourne (nb_jours_ok, nb_jours_en_erreur).""" if not days: - return + return 0, 0 - any_ok = False + ok_count = 0 + error_count = 0 for day in days: ok = collect_tenant(conn, provider, entry, day.isoformat(), (day + timedelta(days=1)).isoformat()) - any_ok = any_ok or ok + if ok: + ok_count += 1 + else: + error_count += 1 if provider == "azure": time.sleep(AZURE_CALL_DELAY_SECONDS) - if any_ok: + if ok_count > 0: resource_end = days[-1] + timedelta(days=1) resource_start = resource_end - timedelta(days=RESOURCE_WINDOW_DAYS) collect_tenant_resources(conn, provider, entry, resource_start.isoformat(), resource_end.isoformat()) + return ok_count, error_count + def main(): parser = argparse.ArgumentParser(description="Backfill de l'historique de coûts depuis AWS/Azure") @@ -183,13 +198,20 @@ def main(): if days: log.info("Backfill journalier de %d jours (%s -> %s)", len(days), days[0].isoformat(), days[-1].isoformat()) + run_id = start_collection_run(conn, "backfill") + total_ok = 0 + total_error = 0 + for provider, entries in (("aws", config.get("aws", [])), ("azure", config.get("azure", []))): for entry in entries: if args.tenant and entry["name"] != args.tenant: continue - backfill_tenant(conn, provider, entry, months) - backfill_tenant_daily(conn, provider, entry, days) + ok1, err1 = backfill_tenant(conn, provider, entry, months) + ok2, err2 = backfill_tenant_daily(conn, provider, entry, days) + total_ok += ok1 + ok2 + total_error += err1 + err2 + finish_collection_run(conn, run_id, total_ok, total_error) conn.close() log.info("Terminé.") diff --git a/collector/collect.py b/collector/collect.py index f89e0d8..c250358 100644 --- a/collector/collect.py +++ b/collector/collect.py @@ -29,6 +29,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from db import ( get_connection, init_db, upsert_collection, upsert_resource_costs, set_resource_collection_status, + start_collection_run, finish_collection_run, ) from providers import ( get_aws_cost, get_azure_cost, convert_result, @@ -148,15 +149,23 @@ def main(): resource_period_end = target_day + timedelta(days=1) resource_period_start = resource_period_end - timedelta(days=RESOURCE_WINDOW_DAYS) + run_id = start_collection_run(conn, "daily") + tenants_ok = 0 + tenants_error = 0 + for provider, entries in (("aws", config.get("aws", [])), ("azure", config.get("azure", []))): for entry in entries: ok = collect_tenant(conn, provider, entry, period_start, period_end) if ok: + tenants_ok += 1 collect_tenant_resources( conn, provider, entry, resource_period_start.isoformat(), resource_period_end.isoformat(), ) + else: + tenants_error += 1 + finish_collection_run(conn, run_id, tenants_ok, tenants_error) conn.close() log.info("Terminé.") diff --git a/collector/db.py b/collector/db.py index 46d1774..5bdb7d0 100644 --- a/collector/db.py +++ b/collector/db.py @@ -10,6 +10,7 @@ from __future__ import annotations import os import sqlite3 +from datetime import datetime, timezone from pathlib import Path DEFAULT_DB_PATH = "cost_dashboard.db" @@ -134,6 +135,30 @@ def set_resource_collection_status( conn.commit() +def start_collection_run(conn: sqlite3.Connection, run_type: str) -> int: + """Enregistre le début d'une exécution de collecte (page "Collecte" du + dashboard). `run_type` : 'daily' (collect.py) ou 'backfill' + (backfill_historical.py).""" + cur = conn.execute( + "INSERT INTO collection_runs (run_type, started_at) VALUES (?, ?)", + (run_type, datetime.now(timezone.utc).isoformat()), + ) + conn.commit() + return cur.lastrowid + + +def finish_collection_run(conn: sqlite3.Connection, run_id: int, tenants_ok: int, tenants_error: int) -> None: + conn.execute( + """ + UPDATE collection_runs + SET finished_at = ?, tenants_ok = ?, tenants_error = ? + WHERE id = ? + """, + (datetime.now(timezone.utc).isoformat(), tenants_ok, tenants_error, run_id), + ) + conn.commit() + + def upsert_resource_costs( conn: sqlite3.Connection, provider: str, diff --git a/db/schema.sql b/db/schema.sql index e1fc2d7..c68407d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -74,3 +74,23 @@ CREATE TABLE IF NOT EXISTS resource_collection_status ( collected_at TIMESTAMP NOT NULL, PRIMARY KEY (provider, tenant_name) ); + +-- Historique des exécutions de tâches de collecte (page "Collecte" du +-- dashboard) : une ligne par lancement de collect.py (run_type='daily') ou +-- backfill_historical.py (run_type='backfill'). Le détail des tenants en +-- erreur n'est pas dupliqué ici : il se relit dans `collections` +-- (status='error') sur la fenêtre [started_at, finished_at] de l'exécution. +-- acknowledged_at reste NULL tant qu'une erreur n'a pas été acquittée +-- manuellement depuis le dashboard. +CREATE TABLE IF NOT EXISTS collection_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_type TEXT NOT NULL CHECK(run_type IN ('daily', 'backfill')), + started_at TIMESTAMP NOT NULL, + finished_at TIMESTAMP, + tenants_ok INTEGER NOT NULL DEFAULT 0, + tenants_error INTEGER NOT NULL DEFAULT 0, + acknowledged_at TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_collection_runs_started + ON collection_runs(started_at); diff --git a/frontend/collecte.html b/frontend/collecte.html new file mode 100644 index 0000000..75c36ec --- /dev/null +++ b/frontend/collecte.html @@ -0,0 +1,61 @@ + + + + + +Collecte — simple-cost-dashboard + + + + +
+ +
+ +
+
+

Collecte

+ historique des exécutions (collecte journalière et backfill historique) +
+
+ +
+ +
+ +

Exécutions

+
+

Clique une exécution pour voir le détail des erreurs.

+
+
Chargement…
+
+
+ +
+ +
+
+ +
+ + + + + + diff --git a/frontend/index.html b/frontend/index.html index 3f6612b..deef28c 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -15,6 +15,7 @@
@@ -43,10 +44,6 @@

Azure — mois en cours

—

-
-

Tenants en échec de collecte

-

—

-

Évolution mensuelle (12 mois glissants)

diff --git a/frontend/services.html b/frontend/services.html index c33464c..9a25160 100644 --- a/frontend/services.html +++ b/frontend/services.html @@ -14,6 +14,7 @@
diff --git a/frontend/static/app.js b/frontend/static/app.js index 04611bb..2ff9c1b 100644 --- a/frontend/static/app.js +++ b/frontend/static/app.js @@ -20,10 +20,6 @@ function renderKpis(data) { document.getElementById("kpi-azure").textContent = formatEUR(data.kpis.total_azure_current_month); const errorCount = data.kpis.tenants_error.length; - const errEl = document.getElementById("kpi-errors"); - errEl.textContent = String(errorCount); - errEl.style.color = errorCount > 0 ? "var(--status-critical)" : ""; - const banner = document.getElementById("error-banner"); if (errorCount > 0) { const names = data.kpis.tenants_error @@ -33,7 +29,7 @@ function renderKpis(data) { `; } else { diff --git a/frontend/static/collecte.js b/frontend/static/collecte.js new file mode 100644 index 0000000..4e66a7f --- /dev/null +++ b/frontend/static/collecte.js @@ -0,0 +1,181 @@ +// collecte.js — page "Collecte" : historique des exécutions de tâches de +// collecte (cron quotidien + backfill historique), détail des erreurs par +// exécution, et acquittement manuel. + +let currentRuns = []; + +async function loadRuns() { + const res = await fetch("/api/collection-runs"); + if (!res.ok) throw new Error(`API collection-runs: HTTP ${res.status}`); + return res.json(); +} + +function runTypeLabel(type) { + return type === "backfill" ? "Historique (backfill)" : "Journalière"; +} + +function formatDateTime(iso) { + if (!iso) return "—"; + return new Intl.DateTimeFormat("fr-FR", { dateStyle: "medium", timeStyle: "short" }).format(new Date(iso)); +} + +function formatDuration(startIso, endIso) { + if (!startIso || !endIso) return "—"; + const seconds = Math.max(0, Math.round((new Date(endIso) - new Date(startIso)) / 1000)); + if (seconds < 60) return `${seconds} s`; + return `${Math.floor(seconds / 60)} min ${seconds % 60} s`; +} + +function statusBadge(run) { + if (run.tenants_error === 0) { + return `OK`; + } + if (run.acknowledged_at) { + return `Erreurs (acquittées)`; + } + return `Erreurs`; +} + +function renderRunsTable() { + const wrap = document.getElementById("runs-table-wrap"); + + if (currentRuns.length === 0) { + wrap.innerHTML = `
Aucune exécution enregistrée pour l'instant. Lance collect.py ou backfill_historical.py.
`; + return; + } + + const rows = currentRuns + .map( + (r) => ` + + ${runTypeLabel(r.run_type)} + ${formatDateTime(r.started_at)} + ${formatDuration(r.started_at, r.finished_at)} + ${r.tenants_ok} + ${r.tenants_error} + ${statusBadge(r)} + ` + ) + .join(""); + + wrap.innerHTML = ` + + + + + + + + + + + + ${rows} +
TypeDémarréeDuréeOKErreursStatut
`; + + wrap.querySelectorAll("tr.clickable").forEach((tr) => { + tr.addEventListener("click", () => openRunDrawer(Number(tr.dataset.id))); + }); +} + +function openRunDrawer(runId) { + const run = currentRuns.find((r) => r.id === runId); + if (!run) return; + + document.getElementById("drawer-title").textContent = `${runTypeLabel(run.run_type)} — ${formatDateTime(run.started_at)}`; + document.getElementById("drawer-sub").textContent = `${run.tenants_ok} OK, ${run.tenants_error} en erreur`; + document.getElementById("run-drawer").classList.add("open"); + document.getElementById("run-drawer").setAttribute("aria-hidden", "false"); + document.getElementById("drawer-backdrop").classList.add("open"); + + if (run.tenants_error === 0) { + document.getElementById("drawer-body").innerHTML = `
Aucune erreur sur cette exécution.
`; + return; + } + + document.getElementById("drawer-body").innerHTML = `
Chargement…
`; + + fetch(`/api/collection-runs/${runId}/errors`) + .then((res) => { + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return res.json(); + }) + .then((data) => renderDrawerContent(run, data.errors)) + .catch((e) => { + document.getElementById("drawer-body").innerHTML = + ``; + }); +} + +function renderDrawerContent(run, errors) { + const bodyEl = document.getElementById("drawer-body"); + + const list = errors + .map( + (e) => ` +
+
+ ${providerLabel(e.provider)} + ${escapeHtml(e.tenant_name)} — ${escapeHtml(e.period_start)} +
+
${escapeHtml(e.error_message || "erreur inconnue")}
+
` + ) + .join(""); + + const ackBlock = run.acknowledged_at + ? `

Acquittée le ${formatDateTime(run.acknowledged_at)}

` + : ``; + + bodyEl.innerHTML = `${ackBlock}${list}`; + + const btn = document.getElementById("ack-button"); + if (btn) { + btn.addEventListener("click", () => acknowledgeRun(run.id)); + } +} + +async function acknowledgeRun(runId) { + const btn = document.getElementById("ack-button"); + if (btn) btn.disabled = true; + + try { + const res = await fetch(`/api/collection-runs/${runId}/acknowledge`, { method: "POST" }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + closeRunDrawer(); + await init(); + refreshCollecteAlert(); + } catch (e) { + document.getElementById("drawer-body").innerHTML = + ``; + } +} + +function closeRunDrawer() { + document.getElementById("run-drawer").classList.remove("open"); + document.getElementById("run-drawer").setAttribute("aria-hidden", "true"); + document.getElementById("drawer-backdrop").classList.remove("open"); +} + +function setupDrawer() { + document.getElementById("drawer-close").addEventListener("click", closeRunDrawer); + document.getElementById("drawer-backdrop").addEventListener("click", closeRunDrawer); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape") closeRunDrawer(); + }); +} + +async function init() { + try { + const data = await loadRuns(); + currentRuns = data.runs; + renderRunsTable(); + } catch (e) { + document.getElementById("error-banner").innerHTML = ` + `; + document.getElementById("runs-table-wrap").innerHTML = ""; + } +} + +init(); +setupDrawer(); diff --git a/frontend/static/common.js b/frontend/static/common.js index 89924fb..6dcf4a4 100644 --- a/frontend/static/common.js +++ b/frontend/static/common.js @@ -63,6 +63,24 @@ function cssVar(name) { return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); } +// Pastille d'alerte sur le lien "Collecte" de la sidebar (présente sur +// toutes les pages) — signale une erreur de collecte pas encore acquittée. +// La page Collecte elle-même se rafraîchit après un acquittement. +async function refreshCollecteAlert() { + const dot = document.getElementById("nav-collecte-alert"); + if (!dot) return; + try { + const res = await fetch("/api/collection-runs/alert-status"); + if (!res.ok) return; + const data = await res.json(); + dot.hidden = !data.has_unacknowledged_errors; + } catch (e) { + // silencieux : une pastille ratée ne doit pas bloquer le reste de la page + } +} + +refreshCollecteAlert(); + // Config partagée des axes/grilles Chart.js (gridlines discrètes, pas de // deuxième axe, tooltip alignée sur le design system). function chartBaseOptions() { diff --git a/frontend/static/style.css b/frontend/static/style.css index 9516636..be0c097 100644 --- a/frontend/static/style.css +++ b/frontend/static/style.css @@ -180,7 +180,9 @@ a { } .sidebar-link { - display: block; + display: flex; + align-items: center; + gap: 7px; padding: 9px 12px; border-radius: var(--radius-control); color: var(--text-secondary); @@ -189,6 +191,14 @@ a { font-weight: 500; } +.nav-alert-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--status-critical); + flex: none; +} + .sidebar-link:hover { background: color-mix(in srgb, var(--text-primary) 6%, transparent); color: var(--text-primary); @@ -527,6 +537,7 @@ table.data-table { .status-dot.ok { background: var(--status-good); } .status-dot.error { background: var(--status-critical); } +.status-dot.warning { background: var(--status-warning); } .status-dot.unknown { background: var(--text-muted); } .status-label { @@ -660,6 +671,25 @@ table.data-table { padding: 8px 20px 20px; } +.btn { + display: inline-flex; + align-items: center; + gap: 6px; + border: 1px solid var(--border); + background: var(--surface-1); + color: var(--text-primary); + font-size: 13px; + font-weight: 500; + padding: 8px 14px; + border-radius: var(--radius-control); + cursor: pointer; + font-family: inherit; +} + +.btn:hover { + background: color-mix(in srgb, var(--text-primary) 6%, transparent); +} + .resource-row { display: flex; align-items: center; diff --git a/frontend/tenant.html b/frontend/tenant.html index 5da3b15..3a520ad 100644 --- a/frontend/tenant.html +++ b/frontend/tenant.html @@ -15,6 +15,7 @@