check_etcd_health - fix for new etcd version output

This commit is contained in:
Ludovic Cartier
2026-06-16 18:33:45 +02:00
parent 190fb9bc16
commit 55d834040e
+77 -45
View File
@@ -8,14 +8,12 @@
# --endpoints "https://192.168.1.41:2379,https://192.168.1.42:2379" \ # --endpoints "https://192.168.1.41:2379,https://192.168.1.42:2379" \
# --cacert /etc/ssl/etcd/ssl/ca.pem --cert /etc/ssl/etcd/ssl/admin.pem --key /etc/ssl/etcd/ssl/admin-key.pem \ # --cacert /etc/ssl/etcd/ssl/ca.pem --cert /etc/ssl/etcd/ssl/admin.pem --key /etc/ssl/etcd/ssl/admin-key.pem \
# --test-snapshot --snapshot-dir /var/backups/etcd --snapshot-max-age 24 # --test-snapshot --snapshot-dir /var/backups/etcd --snapshot-max-age 24
#
# Notes: # Forcer la locale C pour garantir la cohérence du format des nombres (points vs virgules) et des dates
# - For security, run this script on a master (or via NRPE/SSH) with a user having access to the keys. export LC_ALL=C
# - --snapshot-max-age in hours (default 24). Set to 0 to disable age verification.
# - --test-snapshot will create a temporary snapshot to validate creation + verification via `etcdctl snapshot status`.
# - If --keep-snapshot-on-failure is enabled, the temporary snapshot will be kept on error for debugging.
ETCDCTL=${ETCDCTL:-/usr/local/bin/etcdctl} ETCDCTL=${ETCDCTL:-/usr/local/bin/etcdctl}
SNAPFILE=""
print_usage() { print_usage() {
cat <<EOF cat <<EOF
@@ -32,6 +30,23 @@ Options:
EOF EOF
} }
# Fonction globale de nettoyage (Trap)
cleanup() {
rc=$?
if [[ -n "$SNAPFILE" && -f "$SNAPFILE" ]]; then
if [[ $rc -eq 0 ]]; then
rm -f "$SNAPFILE" 2>/dev/null || true
else
if [[ ${KEEP_SNAPSHOT_ON_FAILURE:-0} -eq 0 ]]; then
rm -f "$SNAPFILE" 2>/dev/null || true
else
echo "NOTICE - snapshot kept at $SNAPFILE for debugging"
fi
fi
fi
}
trap 'cleanup' EXIT
# Defaults # Defaults
WARN_DB_MB=${WARN_DB_MB:-1024} WARN_DB_MB=${WARN_DB_MB:-1024}
CRIT_DB_MB=${CRIT_DB_MB:-1800} CRIT_DB_MB=${CRIT_DB_MB:-1800}
@@ -56,11 +71,11 @@ while [[ $# -gt 0 ]]; do
--keep-snapshot-on-failure) KEEP_SNAPSHOT_ON_FAILURE=1; shift 1;; --keep-snapshot-on-failure) KEEP_SNAPSHOT_ON_FAILURE=1; shift 1;;
--snapshot-max-age) SNAPSHOT_MAX_AGE_HOURS="$2"; shift 2;; --snapshot-max-age) SNAPSHOT_MAX_AGE_HOURS="$2"; shift 2;;
-h|--help) print_usage; exit 3;; -h|--help) print_usage; exit 3;;
*) echo "Unknown arg: $1"; print_usage; exit 3;; *) echo "UNKNOWN - Unknown arg: $1"; print_usage; exit 3;;
esac esac
done done
# Allow env fallback (if ETCDCTL_* env vars set) # Allow env fallback
ENDPOINTS=${ENDPOINTS:-${ETCDCTL_ENDPOINTS:-}} ENDPOINTS=${ENDPOINTS:-${ETCDCTL_ENDPOINTS:-}}
CACERT=${CACERT:-${ETCDCTL_CACERT:-}} CACERT=${CACERT:-${ETCDCTL_CACERT:-}}
CERT=${CERT:-${ETCDCTL_CERT:-}} CERT=${CERT:-${ETCDCTL_CERT:-}}
@@ -85,7 +100,7 @@ fi
export ETCDCTL_API=3 export ETCDCTL_API=3
# 1) endpoint status check # 1) Endpoint status check
OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" endpoint status 2>&1) || { OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" endpoint status 2>&1) || {
echo "CRITICAL - etcdctl endpoint status failed: $OUT" echo "CRITICAL - etcdctl endpoint status failed: $OUT"
exit 2 exit 2
@@ -94,29 +109,62 @@ OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --ca
leaders=0 leaders=0
total=0 total=0
max_db_mb=0 max_db_mb=0
while IFS= read -r line; do while IFS= read -r line; do
line=${line//$'\r'/} line=${line//$'\r'/}
[[ -z "$line" ]] && continue # Ignorer les lignes vides ou les warnings d'etcdctl
[[ -z "$line" || "$line" =~ "level\":" || "$line" =~ "unrecognized environment variable" ]] && continue
total=$((total+1)) total=$((total+1))
IFS=',' read -r endpoint id version dbsize isLeader isLearner memberCount rest <<<"$line"
isLeader=$(echo "${isLeader:-}" | tr -d ' ' | tr '[:upper:]' '[:lower:]') # Extraction intelligente via awk (indépendante du nombre de colonnes)
if [[ "$isLeader" == "true" ]]; then leaders=$((leaders+1)); fi # Le leader est le champ qui contient strictement 'true' ou 'false' juste après le pourcentage et la taille en GB
isLeader=$(echo "$line" | awk -F',' '{
for(i=1; i<=NF; i++) {
gsub(/[ \t]+/, "", $i);
if ($i == "true" || $i == "false") {
print $i;
exit;
}
}
}')
# Extraction de la taille (on cherche le premier champ qui contient 'MB' ou 'GB' ou 'KB')
dbsize=$(echo "$line" | awk -F',' '{
for(i=1; i<=NF; i++) {
if ($i ~ /[0-9]+.*[kKmMgG][bB]/) {
print $i;
exit;
}
}
}')
if [[ "$isLeader" == "true" ]]; then
leaders=$((leaders+1))
fi
db_mb=0 db_mb=0
if [[ -n "${dbsize:-}" ]]; then if [[ -n "${dbsize:-}" ]]; then
dbsize=$(echo "$dbsize" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') db_mb=$(awk -v size="$dbsize" '
num=$(echo "$dbsize" | awk '{print $1}' 2>/dev/null || echo "") BEGIN {
unit=$(echo "$dbsize" | awk '{print $2}' 2>/dev/null || echo "") gsub(/^[ \t]+|[ \t]+$/, "", size);
if [[ "$num" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then split(size, arr, /[ \t]+/);
case "${unit^^}" in val = arr[1];
B) db_mb=$(( num / 1024 / 1024 )) ;; unit = toupper(arr[2]);
KB) db_mb=$(( num / 1024 )) ;;
MB) db_mb=$(printf "%.0f" "$num") ;; if (val !~ /^[0-9]+(\.[0-9]+)?$/) { print 0; exit }
GB) db_mb=$(( num * 1024 )) ;;
*) db_mb=$(printf "%.0f" "$num") ;; if (unit == "B") { print int(val / 1024 / 1024) }
esac else if (unit == "KB" || unit == "K") { print int(val / 1024) }
fi else if (unit == "GB" || unit == "G") { print int(val * 1024) }
else { print int(val) }
}
')
fi
if (( db_mb > max_db_mb )); then
max_db_mb=$db_mb
fi fi
if (( db_mb > max_db_mb )); then max_db_mb=$db_mb; fi
done <<< "$OUT" done <<< "$OUT"
if (( total == 0 )); then if (( total == 0 )); then
@@ -143,7 +191,6 @@ fi
# 2) Verification of recent snapshot files (optional, default 24h) # 2) Verification of recent snapshot files (optional, default 24h)
SNAP_CHECK_MSG="" SNAP_CHECK_MSG=""
if [[ -n "$SNAPSHOT_MAX_AGE_HOURS" ]]; then if [[ -n "$SNAPSHOT_MAX_AGE_HOURS" ]]; then
# SNAPSHOT_MAX_AGE_HOURS == 0 -> disabled
if (( SNAPSHOT_MAX_AGE_HOURS > 0 )); then if (( SNAPSHOT_MAX_AGE_HOURS > 0 )); then
mkdir -p "$SNAPSHOT_DIR" 2>/dev/null || { mkdir -p "$SNAPSHOT_DIR" 2>/dev/null || {
echo "CRITICAL - cannot create/access snapshot dir $SNAPSHOT_DIR" echo "CRITICAL - cannot create/access snapshot dir $SNAPSHOT_DIR"
@@ -187,21 +234,6 @@ if (( TEST_SNAPSHOT == 1 )); then
exit 2 exit 2
} }
cleanup() {
rc=$?
if [[ $rc -eq 0 ]]; then
rm -f "$SNAPFILE" 2>/dev/null || true
else
if [[ $KEEP_SNAPSHOT_ON_FAILURE -eq 0 ]]; then
rm -f "$SNAPFILE" 2>/dev/null || true
else
echo "NOTICE - snapshot kept at $SNAPFILE for debugging"
fi
fi
return $rc
}
trap 'cleanup' EXIT
SAVE_OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" snapshot save "$SNAPFILE" 2>&1) || { SAVE_OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" snapshot save "$SNAPFILE" 2>&1) || {
echo "CRITICAL - snapshot save failed: $SAVE_OUT" echo "CRITICAL - snapshot save failed: $SAVE_OUT"
exit 2 exit 2
@@ -212,9 +244,9 @@ if (( TEST_SNAPSHOT == 1 )); then
exit 2 exit 2
} }
# If we reach here, creation+status ok # Formatage propre de la sortie sur une seule ligne
SNAP_TEST_MSG="snapshot test ok: $SNAPFILE ; status: $(echo "$STATUS_OUT" | tr '\n' ' ' | sed 's/ */ /g')" CLEAN_STATUS=$(echo "$STATUS_OUT" | tr '\n' ' ' | sed 's/ */ /g')
# cleanup will remove the snapshot (unless KEEP_SNAPSHOT_ON_FAILURE and rc != 0) SNAP_TEST_MSG="snapshot test ok: $SNAPFILE ; status: $CLEAN_STATUS"
fi fi
# Compose final message # Compose final message