New syslog script

	Yet another script to analyze syslogs. This time a bit more
	generic in its processing capabilities by using SNOBOL4.

ChangeLog:

	* .gitignore: Created.
	* syslog-all.sh: Created.
	* syslog-all.sno: Created.

.gitignore

@@ -0,0 +1,9 @@
+# Editor backups
+*~
+# Generated PDF
+*.pdf
+# Binaries
+rcmd-dump
+# data files
+*.dat
+*.csv

ChangeLog

@@ -1,3 +1,12 @@
+2017-06-06  Jeremy Bennett  <[email protected]>
+
+	Yet another script to analyze syslogs. This time a bit more
+	generic in its processing capabilities by using SNOBOL4.
+
+	* .gitignore: Created.
+	* syslog-all.sh: Created.
+	* syslog-all.sno: Created.
+
 2017-06-06  Jeremy Bennett  <[email protected]>
 
 	* updown-graph.sh: Created.

syslog-all.sh

@@ -0,0 +1,196 @@
+#!/bin/sh
+
+# Copyright (C) 2017 Embecosm Limited <www.embecosm.com>
+
+# Contributor Jeremy Bennett <[email protected]>
+
+# General script to handle syslog data
+
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along with
+# this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Usage:
+
+#     updown-graph.sh
+
+# Prerequisites:
+
+#     sudo dnf install fuse-sshfs
+#     SNOBOL4 installation (see http://www.snobol4.org/csnobol4/curr/)
+#     SSH public key on loglady
+
+###############################################################################
+#
+# Mount the remote directory
+#
+###############################################################################
+
+remdir=/tmp/syslog-all-dir-$$
+#remdir=testing
+remhost=loglady
+
+# Unmount first if already mounted
+
+fusermount -q -u ${remdir} || true
+rm -rf ${remdir}
+mkdir ${remdir}
+
+if ! sshfs embadmin@${remhost}:/var/log ${remdir}
+then
+    echo "ERROR: Could not mount remote file system"
+    exit 1
+fi
+
+
+###############################################################################
+#
+# Strip out irrelevant content from all the syslogs and make one unified file.
+#
+###############################################################################
+
+tmpf1=/tmp/syslog-all-tmp-1-$$
+tmpf2=/tmp/syslog-all-tmp-2-$$
+updowndat=/tmp/syslog-all-tmp-3-$$
+speeddat=/tmp/syslog-all-tmp-4-$$
+#tmpf1=tmpf1
+#tmpf2=tmpf2
+#updowndat=updown.dat
+#speeddat=speed.dat
+rm -f ${tmpf1}
+rm -f ${tmpf2}
+
+# We are not interested in messages at the TCP/UDP level
+
+echo "Gathering data from current logs"
+
+for f in ${remdir}/syslog ${remdir}/syslog.1
+do
+    grep -v 'TCP\|UDP\|ICMP\|DNS' ${f} | tac >> ${tmpf1}
+done
+
+echo "Gathering data from historic logs"
+
+for f in ${remdir}/syslog.?.gz ${remdir}/syslog.??.gz
+do
+    zcat ${f} | grep -v 'TCP\|UDP\|ICMP\|DNS' | tac >> ${tmpf1}
+done
+
+tac < ${tmpf1} | sed -e 's/DrayTek2/DrayTek/' > ${tmpf2}
+
+###############################################################################
+#
+# Now process the data
+#
+###############################################################################
+
+# Use SNOBOL4 to do the processing.
+
+# Processing the data
+
+snobol4 syslog-all.sno ${tmpf2} ${updowndat} ${speeddat}
+
+firstday=$(head -n 1 ${updowndat} | cut -f 1 -d /)
+lastday=$(tail -n 1 ${updowndat} | cut -f 1 -d /)
+
+
+###############################################################################
+#
+# Plot the results
+#
+###############################################################################
+
+echo "Plotting a graph"
+
+cat > ${tmpf2} <<EOF
+set xlabel "date"
+set xdata time
+set timefmt "%Y-%b-%d/%H:%M:%S"
+set format x "%d-%b"
+set border 3
+set xrange ["firstday/00:00:00" : "lastday/23:59:59"]
+set xtics out nomirror
+set style line 1 lc rgb '#0000ff' lt 1 lw .2 pt 7 ps 1.5   # --- blue
+set style line 2 lc rgb '#00ff00' lt 1 lw .2 pt 7 ps 1.5   # --- green
+set style line 3 lc rgb '#ff0000' lt 1 lw .2 pt 7 ps 1.5   # --- red
+set style line 4 lc rgb '#ffff00' lt 1 lw .2 pt 7 ps 1.5   # --- yellow
+
+set ylabel "status"
+set yrange ["0" : "12"]
+set ytics out nomirror ("DSL down" 1, "DSL up" 4, "PPP down" 6, "PPP up" 9)
+plot 'updowndat' using 1:2 linestyle 1 with lines title "DSL status", \
+     'updowndat' using 1:3 linestyle 2 with lines title "PPP status"
+
+set terminal x11 1
+set ylabel "speed"
+set yrange ["0" : "100000000"]
+unset ytics
+set ytics out nomirror
+plot 'speeddat' using 1:2 linestyle 1 with lines title "Speed up", \
+     'speeddat' using 1:3 linestyle 2 with lines title "Speed sown"
+
+set terminal x11 2
+set ylabel "noise"
+set yrange ["-10" : "25"]
+unset ytics
+set ytics out nomirror
+plot 'speeddat' using 1:4 linestyle 1 with lines title "SNR", \
+     'speeddat' using 1:5 linestyle 2 with lines title "Attenuation"
+
+set terminal pdf
+
+set output "status.pdf"
+set ylabel "status"
+set yrange ["0" : "12"]
+set ytics out nomirror ("DSL down" 1, "DSL up" 4, "PPP down" 6, "PPP up" 9)
+plot 'updowndat' using 1:2 linestyle 1 with lines title "DSL status", \
+     'updowndat' using 1:3 linestyle 2 with lines title "PPP status"
+
+set output "speed.pdf"
+set ylabel "speed"
+set yrange ["0" : "100000000"]
+unset ytics
+set ytics out nomirror
+plot 'speeddat' using 1:2 linestyle 1 with lines title "Speed up", \
+     'speeddat' using 1:3 linestyle 2 with lines title "Speed sown"
+
+set output "noise.pdf"
+set ylabel "noise"
+set yrange ["-10" : "25"]
+unset ytics
+set ytics out nomirror
+plot 'speeddat' using 1:4 linestyle 1 with lines title "SNR", \
+     'speeddat' using 1:5 linestyle 2 with lines title "Attenuation"
+EOF
+
+sed -i ${tmpf2} -e "s|firstday|${firstday}|" -e "s|lastday|${lastday}|" \
+    -e "s|updowndat|${updowndat}|g" -e "s|speeddat|${speeddat}|g"
+gnuplot -persist ${tmpf2}
+
+
+###############################################################################
+#
+# Tidy up
+#
+###############################################################################
+
+# Remove temporary files
+
+rm -f ${tmpf1}
+rm -f ${tmpf2}
+rm -f ${updowndat}
+rm -f ${speeddat}
+
+# Lazy unmount to allow previous commands to catch up
+
+fusermount -z -u ${remdir}
+rmdir ${remdir}

syslog-all.sno

@@ -0,0 +1,141 @@
+*	Usage:
+
+*	syslog-call.sno <infile> <upfile> <speedfile>
+
+*	infile - raw syslog ADSL and PPPoE lines
+*	upfile - graph data for DSL and PPPoE uptime
+*	speedfile - graph data for up/down speed, SNR and attenuation
+
+	argv = ARRAY('0:9')
+	i = 0
+	first_arg = HOST(3)
+	argv[i] = HOST(2, first_arg - 1)
+
+get_args
+	arg = HOST(2, first_arg + i)				:F(args_done)
+	i = i + 1
+	argv[i] = arg						:(get_args)
+
+args_done
+	argc = i + 1
+
+*	Get the argument
+
+	EQ(argc,4)						:S(args_ok)
+	TERMINAL = 'Usage: syslog-call.sno <infile> <upfile> '
++		   '<speedfile>'				:(END)
+
+args_ok
+	INPUT('rawdat',9,'',argv[1])
+	OUTPUT('updownout',10,'',argv[2])
+	OUTPUT('speedout',11,'',argv[3])
+
+*	Process the data
+*
+*	DSL failure is reported with 'Modem Shut Down from ADSL Phy Layer'
+*	DSL restart is reported with 'States=SHOWTIME'
+*	All failures lead to 'WAN 1 is down'
+*	All recoveries lead to 'WAN 1 is up'
+*
+*	All lines begin with a date and time stamp of the form:
+*       MMM DD HH:MM:SS
+
+	date_pat = POS(0) LEN(3) . mon ' ' LEN(2) . day ' '
++		   LEN(2) . hour ':' LEN(2) . min ':'
++		   LEN(2) . sec
+
+*	patern to match SHOWTIME data (values may be negative)
+
+	digits = '-0123456789'
+	show_pat = 'UpSpeed='
++	 	   SPAN(digits) . speed_up BREAK(digits)
++		   SPAN(digits) . speed_down BREAK(digits)
++		   SPAN(digits) . snr BREAK(digits)
++		   SPAN(digits) . atten
+
+*	Values for DSL and PPPoE up
+
+	d_up = 4
+	d_do = 1
+	p_up = 9
+	p_do = 6
+
+	dsl_state = d_up
+	ppp_state = p_up
+
+next_line
+	prev_line = line
+	line = rawdat						:F(data_done)
+
+	line 'Modem Shut Down from ADSL Phy Layer'		:S(dsl_down)
+	line 'States=SHOWTIME'	   	    			:S(dsl_up)
+	line 'WAN 1 is down'					:S(ppp_down)
+	line 'WAN 1 is up'					:S(ppp_up)
+								:(next_line)
+*	DSL failure
+
+dsl_down
+	new_dsl_state = NE(dsl_state,d_do) d_do			:F(next_line)
+	new_ppp_state = ppp_state				:(dat_out)
+
+dsl_up
+	new_dsl_state = NE(dsl_state,d_up) d_up			:F(show_only)
+	new_ppp_state = ppp_state				:(dat_out)
+
+ppp_down
+	new_ppp_state = NE(ppp_state,p_do) p_do			:F(next_line)
+	new_dsl_state = dsl_state   				:(dat_out)
+
+ppp_up
+	new_ppp_state = NE(ppp_state,p_up) p_up			:F(next_line)
+	new_dsl_state = dsl_state   				:(dat_out)
+
+*	Output some data. If we get here, something has changed. We reject
+*	January dates.
+
+dat_out
+	line date_pat
+	IDENT(mon,'Jan')					:S(next_line)
+	day ' ' = '0'
+	date = '2017-' mon '-' day '/' hour ':' min ':' sec
+
+*	Up-time/down-time data
+
+	updownout = date CHAR(9) dsl_state CHAR(9) ppp_state
+ 	updownout = date CHAR(9) new_dsl_state CHAR(9) new_ppp_state
+
+*	Update the states
+
+	dsl_state = new_dsl_state
+	ppp_state = new_ppp_state
+
+*	Optionally SHOWTIME data
+
+show_out
+	line show_pat						:F(next_line)
+
+	speedout = date CHAR(9) speed_up CHAR(9) speed_down
++		   CHAR(9) snr CHAR(9) atten			:(next_line)
+
+*	Not exactly tidy, but for the case where we have a SHOWTIME line, bt
+*	other data has not changed, we need to generate the date and then just
+*	put out the speed data.
+
+show_only
+
+	line date_pat
+	IDENT(mon,'Jan')					:S(next_line)
+	day ' ' = '0'
+	date = '2017-' mon '-' day '/' hour ':' min ':' sec	:(show_out)
+
+*	All done. We can put out one final line for the up time/down time graph
+
+data_done
+
+	prev_line date_pat
+	day ' ' = '0'
+	date = '2017-' mon '-' day '/' hour ':' min ':' sec
+
+	updownout = date CHAR(9) dsl_state CHAR(9) ppp_state
+
+END