check_etcd_health - fix for new etcd version output
This commit is contained in:
@@ -8,14 +8,12 @@
|
|||||||
# --endpoints "https://192.168.1.41:2379,https://192.168.1.42:2379" \
|
# --endpoints "https://192.168.1.41:2379,https://192.168.1.42:2379" \
|
||||||
# --cacert /etc/ssl/etcd/ssl/ca.pem --cert /etc/ssl/etcd/ssl/admin.pem --key /etc/ssl/etcd/ssl/admin-key.pem \
|
# --cacert /etc/ssl/etcd/ssl/ca.pem --cert /etc/ssl/etcd/ssl/admin.pem --key /etc/ssl/etcd/ssl/admin-key.pem \
|
||||||
# --test-snapshot --snapshot-dir /var/backups/etcd --snapshot-max-age 24
|
# --test-snapshot --snapshot-dir /var/backups/etcd --snapshot-max-age 24
|
||||||
#
|
|
||||||
# Notes:
|
# Forcer la locale C pour garantir la cohérence du format des nombres (points vs virgules) et des dates
|
||||||
# - For security, run this script on a master (or via NRPE/SSH) with a user having access to the keys.
|
export LC_ALL=C
|
||||||
# - --snapshot-max-age in hours (default 24). Set to 0 to disable age verification.
|
|
||||||
# - --test-snapshot will create a temporary snapshot to validate creation + verification via `etcdctl snapshot status`.
|
|
||||||
# - If --keep-snapshot-on-failure is enabled, the temporary snapshot will be kept on error for debugging.
|
|
||||||
|
|
||||||
ETCDCTL=${ETCDCTL:-/usr/local/bin/etcdctl}
|
ETCDCTL=${ETCDCTL:-/usr/local/bin/etcdctl}
|
||||||
|
SNAPFILE=""
|
||||||
|
|
||||||
print_usage() {
|
print_usage() {
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
@@ -32,6 +30,23 @@ Options:
|
|||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Fonction globale de nettoyage (Trap)
|
||||||
|
cleanup() {
|
||||||
|
rc=$?
|
||||||
|
if [[ -n "$SNAPFILE" && -f "$SNAPFILE" ]]; then
|
||||||
|
if [[ $rc -eq 0 ]]; then
|
||||||
|
rm -f "$SNAPFILE" 2>/dev/null || true
|
||||||
|
else
|
||||||
|
if [[ ${KEEP_SNAPSHOT_ON_FAILURE:-0} -eq 0 ]]; then
|
||||||
|
rm -f "$SNAPFILE" 2>/dev/null || true
|
||||||
|
else
|
||||||
|
echo "NOTICE - snapshot kept at $SNAPFILE for debugging"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap 'cleanup' EXIT
|
||||||
|
|
||||||
# Defaults
|
# Defaults
|
||||||
WARN_DB_MB=${WARN_DB_MB:-1024}
|
WARN_DB_MB=${WARN_DB_MB:-1024}
|
||||||
CRIT_DB_MB=${CRIT_DB_MB:-1800}
|
CRIT_DB_MB=${CRIT_DB_MB:-1800}
|
||||||
@@ -56,11 +71,11 @@ while [[ $# -gt 0 ]]; do
|
|||||||
--keep-snapshot-on-failure) KEEP_SNAPSHOT_ON_FAILURE=1; shift 1;;
|
--keep-snapshot-on-failure) KEEP_SNAPSHOT_ON_FAILURE=1; shift 1;;
|
||||||
--snapshot-max-age) SNAPSHOT_MAX_AGE_HOURS="$2"; shift 2;;
|
--snapshot-max-age) SNAPSHOT_MAX_AGE_HOURS="$2"; shift 2;;
|
||||||
-h|--help) print_usage; exit 3;;
|
-h|--help) print_usage; exit 3;;
|
||||||
*) echo "Unknown arg: $1"; print_usage; exit 3;;
|
*) echo "UNKNOWN - Unknown arg: $1"; print_usage; exit 3;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
# Allow env fallback (if ETCDCTL_* env vars set)
|
# Allow env fallback
|
||||||
ENDPOINTS=${ENDPOINTS:-${ETCDCTL_ENDPOINTS:-}}
|
ENDPOINTS=${ENDPOINTS:-${ETCDCTL_ENDPOINTS:-}}
|
||||||
CACERT=${CACERT:-${ETCDCTL_CACERT:-}}
|
CACERT=${CACERT:-${ETCDCTL_CACERT:-}}
|
||||||
CERT=${CERT:-${ETCDCTL_CERT:-}}
|
CERT=${CERT:-${ETCDCTL_CERT:-}}
|
||||||
@@ -85,7 +100,7 @@ fi
|
|||||||
|
|
||||||
export ETCDCTL_API=3
|
export ETCDCTL_API=3
|
||||||
|
|
||||||
# 1) endpoint status check
|
# 1) Endpoint status check
|
||||||
OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" endpoint status 2>&1) || {
|
OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" endpoint status 2>&1) || {
|
||||||
echo "CRITICAL - etcdctl endpoint status failed: $OUT"
|
echo "CRITICAL - etcdctl endpoint status failed: $OUT"
|
||||||
exit 2
|
exit 2
|
||||||
@@ -94,29 +109,62 @@ OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --ca
|
|||||||
leaders=0
|
leaders=0
|
||||||
total=0
|
total=0
|
||||||
max_db_mb=0
|
max_db_mb=0
|
||||||
|
|
||||||
while IFS= read -r line; do
|
while IFS= read -r line; do
|
||||||
line=${line//$'\r'/}
|
line=${line//$'\r'/}
|
||||||
[[ -z "$line" ]] && continue
|
# Ignorer les lignes vides ou les warnings d'etcdctl
|
||||||
|
[[ -z "$line" || "$line" =~ "level\":" || "$line" =~ "unrecognized environment variable" ]] && continue
|
||||||
|
|
||||||
total=$((total+1))
|
total=$((total+1))
|
||||||
IFS=',' read -r endpoint id version dbsize isLeader isLearner memberCount rest <<<"$line"
|
|
||||||
isLeader=$(echo "${isLeader:-}" | tr -d ' ' | tr '[:upper:]' '[:lower:]')
|
# Extraction intelligente via awk (indépendante du nombre de colonnes)
|
||||||
if [[ "$isLeader" == "true" ]]; then leaders=$((leaders+1)); fi
|
# Le leader est le champ qui contient strictement 'true' ou 'false' juste après le pourcentage et la taille en GB
|
||||||
|
isLeader=$(echo "$line" | awk -F',' '{
|
||||||
|
for(i=1; i<=NF; i++) {
|
||||||
|
gsub(/[ \t]+/, "", $i);
|
||||||
|
if ($i == "true" || $i == "false") {
|
||||||
|
print $i;
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}')
|
||||||
|
|
||||||
|
# Extraction de la taille (on cherche le premier champ qui contient 'MB' ou 'GB' ou 'KB')
|
||||||
|
dbsize=$(echo "$line" | awk -F',' '{
|
||||||
|
for(i=1; i<=NF; i++) {
|
||||||
|
if ($i ~ /[0-9]+.*[kKmMgG][bB]/) {
|
||||||
|
print $i;
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}')
|
||||||
|
|
||||||
|
if [[ "$isLeader" == "true" ]]; then
|
||||||
|
leaders=$((leaders+1))
|
||||||
|
fi
|
||||||
|
|
||||||
db_mb=0
|
db_mb=0
|
||||||
if [[ -n "${dbsize:-}" ]]; then
|
if [[ -n "${dbsize:-}" ]]; then
|
||||||
dbsize=$(echo "$dbsize" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
db_mb=$(awk -v size="$dbsize" '
|
||||||
num=$(echo "$dbsize" | awk '{print $1}' 2>/dev/null || echo "")
|
BEGIN {
|
||||||
unit=$(echo "$dbsize" | awk '{print $2}' 2>/dev/null || echo "")
|
gsub(/^[ \t]+|[ \t]+$/, "", size);
|
||||||
if [[ "$num" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
|
split(size, arr, /[ \t]+/);
|
||||||
case "${unit^^}" in
|
val = arr[1];
|
||||||
B) db_mb=$(( num / 1024 / 1024 )) ;;
|
unit = toupper(arr[2]);
|
||||||
KB) db_mb=$(( num / 1024 )) ;;
|
|
||||||
MB) db_mb=$(printf "%.0f" "$num") ;;
|
if (val !~ /^[0-9]+(\.[0-9]+)?$/) { print 0; exit }
|
||||||
GB) db_mb=$(( num * 1024 )) ;;
|
|
||||||
*) db_mb=$(printf "%.0f" "$num") ;;
|
if (unit == "B") { print int(val / 1024 / 1024) }
|
||||||
esac
|
else if (unit == "KB" || unit == "K") { print int(val / 1024) }
|
||||||
fi
|
else if (unit == "GB" || unit == "G") { print int(val * 1024) }
|
||||||
|
else { print int(val) }
|
||||||
|
}
|
||||||
|
')
|
||||||
|
fi
|
||||||
|
|
||||||
|
if (( db_mb > max_db_mb )); then
|
||||||
|
max_db_mb=$db_mb
|
||||||
fi
|
fi
|
||||||
if (( db_mb > max_db_mb )); then max_db_mb=$db_mb; fi
|
|
||||||
done <<< "$OUT"
|
done <<< "$OUT"
|
||||||
|
|
||||||
if (( total == 0 )); then
|
if (( total == 0 )); then
|
||||||
@@ -143,7 +191,6 @@ fi
|
|||||||
# 2) Verification of recent snapshot files (optional, default 24h)
|
# 2) Verification of recent snapshot files (optional, default 24h)
|
||||||
SNAP_CHECK_MSG=""
|
SNAP_CHECK_MSG=""
|
||||||
if [[ -n "$SNAPSHOT_MAX_AGE_HOURS" ]]; then
|
if [[ -n "$SNAPSHOT_MAX_AGE_HOURS" ]]; then
|
||||||
# SNAPSHOT_MAX_AGE_HOURS == 0 -> disabled
|
|
||||||
if (( SNAPSHOT_MAX_AGE_HOURS > 0 )); then
|
if (( SNAPSHOT_MAX_AGE_HOURS > 0 )); then
|
||||||
mkdir -p "$SNAPSHOT_DIR" 2>/dev/null || {
|
mkdir -p "$SNAPSHOT_DIR" 2>/dev/null || {
|
||||||
echo "CRITICAL - cannot create/access snapshot dir $SNAPSHOT_DIR"
|
echo "CRITICAL - cannot create/access snapshot dir $SNAPSHOT_DIR"
|
||||||
@@ -187,21 +234,6 @@ if (( TEST_SNAPSHOT == 1 )); then
|
|||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
rc=$?
|
|
||||||
if [[ $rc -eq 0 ]]; then
|
|
||||||
rm -f "$SNAPFILE" 2>/dev/null || true
|
|
||||||
else
|
|
||||||
if [[ $KEEP_SNAPSHOT_ON_FAILURE -eq 0 ]]; then
|
|
||||||
rm -f "$SNAPFILE" 2>/dev/null || true
|
|
||||||
else
|
|
||||||
echo "NOTICE - snapshot kept at $SNAPFILE for debugging"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
return $rc
|
|
||||||
}
|
|
||||||
trap 'cleanup' EXIT
|
|
||||||
|
|
||||||
SAVE_OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" snapshot save "$SNAPFILE" 2>&1) || {
|
SAVE_OUT=$("$ETCDCTL" --command-timeout="${TIMEOUT}s" --endpoints="${ENDPOINTS}" --cacert="$CACERT" --cert="$CERT" --key="$KEY" snapshot save "$SNAPFILE" 2>&1) || {
|
||||||
echo "CRITICAL - snapshot save failed: $SAVE_OUT"
|
echo "CRITICAL - snapshot save failed: $SAVE_OUT"
|
||||||
exit 2
|
exit 2
|
||||||
@@ -212,9 +244,9 @@ if (( TEST_SNAPSHOT == 1 )); then
|
|||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
# If we reach here, creation+status ok
|
# Formatage propre de la sortie sur une seule ligne
|
||||||
SNAP_TEST_MSG="snapshot test ok: $SNAPFILE ; status: $(echo "$STATUS_OUT" | tr '\n' ' ' | sed 's/ */ /g')"
|
CLEAN_STATUS=$(echo "$STATUS_OUT" | tr '\n' ' ' | sed 's/ */ /g')
|
||||||
# cleanup will remove the snapshot (unless KEEP_SNAPSHOT_ON_FAILURE and rc != 0)
|
SNAP_TEST_MSG="snapshot test ok: $SNAPFILE ; status: $CLEAN_STATUS"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Compose final message
|
# Compose final message
|
||||||
|
|||||||
Reference in New Issue
Block a user