README | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++ clear.sh | 3 +++ helper/dns | 12 ++++++++++++ helper/http | 9 +++++++++ helper/https-cert | 9 +++++++++ helper/iface | 5 +++++ helper/netstat | 3 +++ helper/pid-alive | 5 +++++ helper/ping | 11 +++++++++++ helper/smtp | 5 +++++ notify-bad.sh | 12 ++++++++++++ notify-ok.sh | 4 ++++ probelist.sh | 6 ++++++ run.sh | 33 +++++++++++++++++++++++++++++++++ start.sh | 35 +++++++++++++++++++++++++++++++++++ status.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ diff --git a/README b/README new file mode 100644 index 0000000000000000000000000000000000000000..8454efe691d3b0f32fe5950f4d310f2c549679ee --- /dev/null +++ b/README @@ -0,0 +1,72 @@ +sgmon -- Simple stargrave's monitoring system +Intended for my own personal use, as a replacement for Monit. +All of that is just an ordinary plain POSIX shell scripts. + +Create some directory hierarchy for convenience: + + $ mkdir -p probes/example.com/{ping4,ping6,http} + $ mkdir -p probes/jails/foobar + +Make probes from them: + + $ cat > probes/example.com/ping4/run < probes/example.com/http/max-attempts + + $ cat > probes/example.com/ping4/run < probes/example.com/ping6/run < probes/example.com/ping4/period # every 5 seconds + + $ cat > probes/jails/foobar/run < rc < /path/to/index.html + +If some probe is failed, then email message with its stdout/stderr will +be sent. If it succeeds after the fail, then another message is sent. +Each startup and shutdown are also notified by email. diff --git a/clear.sh b/clear.sh new file mode 100755 index 0000000000000000000000000000000000000000..38ad620673c479013bb83d0472fef3ef887d9cbe --- /dev/null +++ b/clear.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +find . \( -name state -or -name -pid \) -delete -print diff --git a/helper/dns b/helper/dns new file mode 100755 index 0000000000000000000000000000000000000000..af23f10120fcf1488417466d2a42749426877851 --- /dev/null +++ b/helper/dns @@ -0,0 +1,12 @@ +#!/bin/sh -e + +proto=$1 +srv=$2 +shift +shift + +drill -${proto} @$srv $@ >state/drill-out +grep -q "rcode: NOERROR" state/drill-out || { + cat state/drill-out + exit 1 +} diff --git a/helper/http b/helper/http new file mode 100755 index 0000000000000000000000000000000000000000..a917d218538e59f498eea59703fd3a66c4b96789 --- /dev/null +++ b/helper/http @@ -0,0 +1,9 @@ +#!/bin/sh -e + +bad=0 +curl --max-time 60 --verbose "$1" >state/curl-out 2>&1 || bad=1 +grep -q "$togrep" state/curl-out || bad=1 +if [ "$bad" -eq 1 ] ; then + cat state/curl-out + exit 1 +fi diff --git a/helper/https-cert b/helper/https-cert new file mode 100755 index 0000000000000000000000000000000000000000..3053837481dfcde07c9a56106cc241dbcfe580d2 --- /dev/null +++ b/helper/https-cert @@ -0,0 +1,9 @@ +#!/bin/sh -e + +bad=0 +curl --max-time 60 --verbose "$1" >/dev/null 2>state/curl-out || bad=1 +grep -q "subject: CN=$2" state/curl-out || bad=1 +if [ "$bad" -eq 1 ] ; then + cat state/curl-out + exit 1 +fi diff --git a/helper/iface b/helper/iface new file mode 100755 index 0000000000000000000000000000000000000000..e22758126de8cc7f32b8b334de43900dd67c9c11 --- /dev/null +++ b/helper/iface @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ifconfig $1 > state/ifconfig-out +cat state/ifconfig-out +grep -q "flags=.*UP" state/ifconfig-out diff --git a/helper/netstat b/helper/netstat new file mode 100755 index 0000000000000000000000000000000000000000..54cb36bdef769dbc1aeba2383c6fed488c222696 --- /dev/null +++ b/helper/netstat @@ -0,0 +1,3 @@ +#!/bin/sh -e + +netstat -I $1 -w 1 -q 1 -h diff --git a/helper/pid-alive b/helper/pid-alive new file mode 100755 index 0000000000000000000000000000000000000000..2c8447b4757b22d7544b9d8b1c7f7a1cf229699c --- /dev/null +++ b/helper/pid-alive @@ -0,0 +1,5 @@ +#!/bin/sh -e + +read pid < "$1" +ps $pid > state/ps-out +tail -1 state/ps-out diff --git a/helper/ping b/helper/ping new file mode 100755 index 0000000000000000000000000000000000000000..180f4876db91cdc252a601417a2c7488827fc79a --- /dev/null +++ b/helper/ping @@ -0,0 +1,11 @@ +#!/bin/sh -e + +proto=$1 +shift +[ "$proto" = "6" ] || proto="" +if ping${proto} -c 1 $@ > state/ping-out ; then + tail -1 < state/ping-out +else + cat state/ping-out + exit 1 +fi diff --git a/helper/smtp b/helper/smtp new file mode 100755 index 0000000000000000000000000000000000000000..6d7002f0dd49d15434222f9362f961d2d7d6f8ee --- /dev/null +++ b/helper/smtp @@ -0,0 +1,5 @@ +#!/bin/sh -e + +proto=$1 +host=$2 +printf "QUIT\r\n" | nc -${proto} -w 30 $host 25 | grep -q "220 $host ESMTP" diff --git a/notify-bad.sh b/notify-bad.sh new file mode 100755 index 0000000000000000000000000000000000000000..ca3769a010e310040887eb0ef73b81a50a9efa5c --- /dev/null +++ b/notify-bad.sh @@ -0,0 +1,12 @@ +#!/bin/sh -e + +. "$SGMONDIR"/rc +( + echo stdout: + cat state/stdout + if [ -s state/stderr ] ; then + echo stderr: + cat state/stderr + fi +) | +mailx -s "Failed $SGMONSRV" "$NOTIFY_EMAIL" diff --git a/notify-ok.sh b/notify-ok.sh new file mode 100755 index 0000000000000000000000000000000000000000..b265cb6ad7ff5cec13654099594b3e6e2a760b81 --- /dev/null +++ b/notify-ok.sh @@ -0,0 +1,4 @@ +#!/bin/sh -e + +. "$SGMONDIR"/rc +echo OK | mailx -s "Succeeded $SGMONSRV" "$NOTIFY_EMAIL" diff --git a/probelist.sh b/probelist.sh new file mode 100755 index 0000000000000000000000000000000000000000..6cafcb2d3a837b600d07761cb342b2ce1a4f696f --- /dev/null +++ b/probelist.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +find . -mindepth 1 -type d -not -name ".*" | while read probe ; do + [ -x $probe/run ] || continue + echo ${probe#./} +done | sort diff --git a/run.sh b/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..13ff28e8a32547c0165a149fc29532a18b11ce15 --- /dev/null +++ b/run.sh @@ -0,0 +1,33 @@ +#!/bin/sh -e + +export SGMONDIR="$(dirname "$(realpath -- "$0")")" +. "$SGMONDIR"/rc +[ -z "$1" ] && SGMONSRV="$(basename "$PWD")" || SGMONSRV="$1" +export SGMONSRV +export SGMONTMP=`mktemp -d -t sgmon` +trap "rm -fr $SGMONTMP" HUP PIPE INT QUIT TERM EXIT +rm -f state +ln -s $SGMONTMP state + +while : ; do + [ -s max-attempts ] && read max_attempts < max-attempts || max_attempts=1 + [ -s state/attempts ] && read attempts < state/attempts || attempts=0 + if ./run >state/stdout 2>state/stderr ; then + if [ "$attempts" -ge "$max_attempts" ] ; then + "$SGMONDIR"/notify-ok.sh + fi + rm -f state/attempts + touch state/ok + echo OK + else + rm -f state/ok + attempts=$(( $attempts + 1 )) + echo $attempts > state/attempts + echo BAD ${attempts}/${max_attempts} + if [ $attempts -eq $max_attempts ] ; then + "$SGMONDIR"/notify-bad.sh + fi + fi + [ -s period ] && read period < period || period=$PERIOD_DEFAULT + sleep $period +done diff --git a/start.sh b/start.sh new file mode 100755 index 0000000000000000000000000000000000000000..30315ddd28040e6816eabff11ad5a1a48787bcb0 --- /dev/null +++ b/start.sh @@ -0,0 +1,35 @@ +#!/bin/sh -e + +stop() { + trap - HUP PIPE INT QUIT TERM EXIT + echo ... | mailx -s "Stopping" "$NOTIFY_EMAIL" + for probe in $probes ; do + [ -s $probe/pid ] || continue + echo killing ${probe}... + pkill -F $probe/pid 2>/dev/null || : + done + for probe in $probes ; do + read pid < $probe/pid || : + [ -n "$pid" ] || continue + echo waiting ${probe}... + wait $pid || : + done + echo finished + exit +} + +root="`pwd`" +SGMONDIR="$(dirname "$(realpath -- "$0")")" +. "$SGMONDIR"/rc +probes="$("$SGMONDIR"/probelist.sh)" +echo $(echo "$probes" | wc -l) probes | mailx -s "Starting" "$NOTIFY_EMAIL" +for probe in $probes ; do + cd $probe + echo starting ${probe}... + "$SGMONDIR"/run.sh $probe >/dev/null 2>&1 & + echo $! > pid || : + cd "$root" +done +echo OK +trap stop HUP PIPE INT QUIT TERM EXIT +while : ; do sleep 1 ; done diff --git a/status.sh b/status.sh new file mode 100755 index 0000000000000000000000000000000000000000..d3eec5b61b1f87e0f08202b02b8128e1666bc787 --- /dev/null +++ b/status.sh @@ -0,0 +1,45 @@ +#!/bin/sh -e + +catfiles() { + for f in $@ ; do + [ -s $f ] || continue + sed 's/&/\&/g ; s//\>/g' < $f + done +} + +SGMONDIR="$(dirname "$(realpath -- "$0")")" +. "$SGMONDIR"/rc +probes="$("$SGMONDIR"/probelist.sh)" +cat < +sgmon status $(date -Iseconds) + + +EOF +for probe in $probes ; do + if [ -s $probe/state/attempts ] ; then + read attempts < $probe/state/attempts || attempts=1 + mtime=$(stat -t "%F %T" -f %Sm $probe/state/attempts) + [ -s max-attempts ] && read max_attempts < max-attempts || max_attempts=1 + [ "$attempts" -lt "$max_attempts" ] && + status="Prefail" || status="FAIL" + status="$status (${attempts}/${max_attempts})" + else + if [ -r $probe/state/ok ] ; then + mtime=$(stat -t "%F %T" -f %Sm $probe/state/ok) + status=OK + else + mtime= + status="???" + fi + fi + cat < + + + + + +EOF +done +echo "
StatusprobeLastMessage
$status$probe$mtime
$(catfiles $probe/state/stdout $probe/state/stderr)
"