check_etcd_health - fix for new etcd version output
This commit is contained in:
Ludovic Cartier
@@ -8,14 +8,12 @@
|
||||
# --endpoints "https://192.168.1.41:2379,https://192.168.1.42:2379" \
|
||||
# --cacert /etc/ssl/etcd/ssl/ca.pem --cert /etc/ssl/etcd/ssl/admin.pem --key /etc/ssl/etcd/ssl/admin-key.pem \
|
||||
# --test-snapshot --snapshot-dir /var/backups/etcd --snapshot-max-age 24
|
||||
#
|
||||
# Notes:
|
||||
# - For security, run this script on a master (or via NRPE/SSH) with a user having access to the keys.
|
||||
# - --snapshot-max-age in hours (default 24). Set to 0 to disable age verification.
|
||||
# - --test-snapshot will create a temporary snapshot to validate creation + verification via `etcdctl snapshot status`.
|
||||
# - If --keep-snapshot-on-failure is enabled, the temporary snapshot will be kept on error for debugging.
|
||||
|
||||
# Forcer la locale C pour garantir la cohérence du format des nombres (points vs virgules) et des dates
|
||||
export LC_ALL=C
|
||||
|
||||
ETCDCTL=${ETCDCTL:-/usr/local/bin/etcdctl}
|
||||
SNAPFILE=""
|
||||
|
||||
print_usage() {
|
||||
cat <<EOF
|
||||
@@ -32,6 +30,23 @@ Options:
|
||||
EOF
|
||||
}
|
||||
|
||||
# Fonction globale de nettoyage (Trap)
|
||||
cleanup() {
|
||||
rc=$?
|
||||
if [[ -n "$SNAPFILE" && -f "$SNAPFILE" ]]; then
|
||||
if [[ $rc -eq 0 ]]; then
|
||||
rm -f "$SNAPFILE" 2>/dev/null || true
|
||||
else
|
||||
if [[ ${KEEP_SNAPSHOT_ON_FAILURE:-0} -eq 0 ]]; then
|
||||
rm -f "$SNAPFILE" 2>/dev/null || true
|
||||
else
|
||||
echo "NOTICE - snapshot kept at $SNAPFILE for debugging"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
trap 'cleanup' EXIT
|
||||
|
||||
# Defaults
|
||||
WARN_DB_MB=${WARN_DB_MB:-1024}
|
||||
CRIT_DB_MB=${CRIT_DB_MB:-1800}
|
||||
@@ -56,11 +71,11 @@ while [[ $# -gt 0 ]]; do
|
||||
--keep-snapshot-on-failure) KEEP_SNAPSHOT_ON_FAILURE=1; shift 1;;
|
||||
--snapshot-max-age) SNAPSHOT_MAX_AGE_HOURS="$2"; shift 2;;
|
||||
-h|--help) print_usage; exit 3;;
|
||||
*) echo "Unknown arg: $1"; print_usage; exit 3;;
|
||||
*) echo "UNKNOWN - Unknown arg: $1"; print_usage; exit 3;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Allow env fallback (if ETCDCTL_* env vars set)
|
||||
# Allow env fallback
|
||||
ENDPOINTS=${ENDPOINTS:-${ETCDCTL_ENDPOINTS:-}}
|
||||
CACERT=${CACERT:-${ETCDCTL_CACERT:-}}
|
||||
CERT=${CERT:-${ETCDCTL_CERT:-}}
|
||||
@@ -85,7 +100,7 @@ fi
|
||||
|
||||
export ETCDCTL_API=3
|
||||
|
||||
# 1) endpoint status check
|
||||
# 1) Endpoint status check
|
||||
OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" endpoint status 2>&1) || {
|
||||
echo "CRITICAL - etcdctl endpoint status failed: $OUT"
|
||||
exit 2
|
||||
@@ -94,29 +109,62 @@ OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --ca
|
||||
leaders=0
|
||||
total=0
|
||||
max_db_mb=0
|
||||
|
||||
while IFS= read -r line; do
|
||||
line=${line//$'\r'/}
|
||||
[[ -z "$line" ]] && continue
|
||||
# Ignorer les lignes vides ou les warnings d'etcdctl
|
||||
[[ -z "$line" || "$line" =~ "level\":" || "$line" =~ "unrecognized environment variable" ]] && continue
|
||||
|
||||
total=$((total+1))
|
||||
IFS=',' read -r endpoint id version dbsize isLeader isLearner memberCount rest <<<"$line"
|
||||
isLeader=$(echo "${isLeader:-}" | tr -d ' ' | tr '[:upper:]' '[:lower:]')
|
||||
if [[ "$isLeader" == "true" ]]; then leaders=$((leaders+1)); fi
|
||||
|
||||
# Extraction intelligente via awk (indépendante du nombre de colonnes)
|
||||
# Le leader est le champ qui contient strictement 'true' ou 'false' juste après le pourcentage et la taille en GB
|
||||
isLeader=$(echo "$line" | awk -F',' '{
|
||||
for(i=1; i<=NF; i++) {
|
||||
gsub(/[ \t]+/, "", $i);
|
||||
if ($i == "true" || $i == "false") {
|
||||
print $i;
|
||||
exit;
|
||||
}
|
||||
}
|
||||
}')
|
||||
|
||||
# Extraction de la taille (on cherche le premier champ qui contient 'MB' ou 'GB' ou 'KB')
|
||||
dbsize=$(echo "$line" | awk -F',' '{
|
||||
for(i=1; i<=NF; i++) {
|
||||
if ($i ~ /[0-9]+.*[kKmMgG][bB]/) {
|
||||
print $i;
|
||||
exit;
|
||||
}
|
||||
}
|
||||
}')
|
||||
|
||||
if [[ "$isLeader" == "true" ]]; then
|
||||
leaders=$((leaders+1))
|
||||
fi
|
||||
|
||||
db_mb=0
|
||||
if [[ -n "${dbsize:-}" ]]; then
|
||||
dbsize=$(echo "$dbsize" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
num=$(echo "$dbsize" | awk '{print $1}' 2>/dev/null || echo "")
|
||||
unit=$(echo "$dbsize" | awk '{print $2}' 2>/dev/null || echo "")
|
||||
if [[ "$num" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
|
||||
case "${unit^^}" in
|
||||
B) db_mb=$(( num / 1024 / 1024 )) ;;
|
||||
KB) db_mb=$(( num / 1024 )) ;;
|
||||
MB) db_mb=$(printf "%.0f" "$num") ;;
|
||||
GB) db_mb=$(( num * 1024 )) ;;
|
||||
*) db_mb=$(printf "%.0f" "$num") ;;
|
||||
esac
|
||||
fi
|
||||
db_mb=$(awk -v size="$dbsize" '
|
||||
BEGIN {
|
||||
gsub(/^[ \t]+|[ \t]+$/, "", size);
|
||||
split(size, arr, /[ \t]+/);
|
||||
val = arr[1];
|
||||
unit = toupper(arr[2]);
|
||||
|
||||
if (val !~ /^[0-9]+(\.[0-9]+)?$/) { print 0; exit }
|
||||
|
||||
if (unit == "B") { print int(val / 1024 / 1024) }
|
||||
else if (unit == "KB" || unit == "K") { print int(val / 1024) }
|
||||
else if (unit == "GB" || unit == "G") { print int(val * 1024) }
|
||||
else { print int(val) }
|
||||
}
|
||||
')
|
||||
fi
|
||||
|
||||
if (( db_mb > max_db_mb )); then
|
||||
max_db_mb=$db_mb
|
||||
fi
|
||||
if (( db_mb > max_db_mb )); then max_db_mb=$db_mb; fi
|
||||
done <<< "$OUT"
|
||||
|
||||
if (( total == 0 )); then
|
||||
@@ -143,7 +191,6 @@ fi
|
||||
# 2) Verification of recent snapshot files (optional, default 24h)
|
||||
SNAP_CHECK_MSG=""
|
||||
if [[ -n "$SNAPSHOT_MAX_AGE_HOURS" ]]; then
|
||||
# SNAPSHOT_MAX_AGE_HOURS == 0 -> disabled
|
||||
if (( SNAPSHOT_MAX_AGE_HOURS > 0 )); then
|
||||
mkdir -p "$SNAPSHOT_DIR" 2>/dev/null || {
|
||||
echo "CRITICAL - cannot create/access snapshot dir $SNAPSHOT_DIR"
|
||||
@@ -187,21 +234,6 @@ if (( TEST_SNAPSHOT == 1 )); then
|
||||
exit 2
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
rc=$?
|
||||
if [[ $rc -eq 0 ]]; then
|
||||
rm -f "$SNAPFILE" 2>/dev/null || true
|
||||
else
|
||||
if [[ $KEEP_SNAPSHOT_ON_FAILURE -eq 0 ]]; then
|
||||
rm -f "$SNAPFILE" 2>/dev/null || true
|
||||
else
|
||||
echo "NOTICE - snapshot kept at $SNAPFILE for debugging"
|
||||
fi
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
trap 'cleanup' EXIT
|
||||
|
||||
SAVE_OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" snapshot save "$SNAPFILE" 2>&1) || {
|
||||
echo "CRITICAL - snapshot save failed: $SAVE_OUT"
|
||||
exit 2
|
||||
@@ -212,9 +244,9 @@ if (( TEST_SNAPSHOT == 1 )); then
|
||||
exit 2
|
||||
}
|
||||
|
||||
# If we reach here, creation+status ok
|
||||
SNAP_TEST_MSG="snapshot test ok: $SNAPFILE ; status: $(echo "$STATUS_OUT" | tr '\n' ' ' | sed 's/ */ /g')"
|
||||
# cleanup will remove the snapshot (unless KEEP_SNAPSHOT_ON_FAILURE and rc != 0)
|
||||
# Formatage propre de la sortie sur une seule ligne
|
||||
CLEAN_STATUS=$(echo "$STATUS_OUT" | tr '\n' ' ' | sed 's/ */ /g')
|
||||
SNAP_TEST_MSG="snapshot test ok: $SNAPFILE ; status: $CLEAN_STATUS"
|
||||
fi
|
||||
|
||||
# Compose final message
|
||||
|
||||
Reference in New Issue
Block a user