Eximstats.sh
Views:
Call this every night to generate a text based report for your exim server. It uses a little perl script to split the main log in to one log per complete day (from 00:00 hrs to 23:59 hrs). I use this to collect stats from multiple servers, then gather them centrally and convert them in to html reports.
exim-main-log-split.pl:
#!/usr/bin/env perl
# Take the exim log file specified as the first argument when calling this script,
# then split all entries in to seperate files for each day.
# also pipe logs in
use strict;
use warnings;
use File::Basename;
use POSIX;
my $logtype = "main.log";
my $previousDate = '';
my $currentDate = '';
my $logfile;
while (<>) {
my $line = $_;
$currentDate = '';
if ($line =~ /^(\d{4}-\d{2}-\d{2}) .*/) {
$currentDate = $1;
}
if ($currentDate ne $previousDate || !$logfile) {
if ($logfile) {
close($logfile);
}
open $logfile, ">>", "$logtype.$currentDate"
or die "Could not open $logtype.$currentDate: $!";
}
print $logfile $line;
}
eximstats.sh:
#!/bin/bash # Generate daily reports from the exim-relay logs. # scp the most recent report to www.exmple.com # Remove existing reports older than seven days. LOG_FILE=/var/log/exim/main.log.1.bz2 LOG_SPLITTER=/usr/local/bin/exim-main-log-split.pl REPORT_DIR=/var/lib/exim/eximstats SCP_TARGET=eximstat@www.example.com:/var/lib/eximstat/$(hostname) DATE=$(date +%F) REPORT_DATE=$(date --date '1 day ago' +%F) EXIMSTATS=/usr/bin/eximstats ARGS="$(cat <<EOT -pattern Spam '/ rejected after DATA: Spam messages are not welcome here./' \ -pattern Viruses '/ rejected after DATA: This message contains a virus/' \ EOT )" if [ ! -f $LOG_FILE ]; then echo Log file $LOG_FILE does not exist exit 1 fi cd $REPORT_DIR || exit 1 bzcat $LOG_FILE | $LOG_SPLITTER $EXIMSTATS $ARGS -txt=$REPORT_DIR/$REPORT_DATE.txt $REPORT_DIR/main.log.$REPORT_DATE || exit rm $REPORT_DIR/main.log.$REPORT_DATE scp $REPORT_DATE.txt $SCP_TARGET/ find $REPORT_DIR -name '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].txt' -ctime +14 | xargs rm
eximstats-html-reports.sh (on www.example.com):
#!/bin/bash
#
# A script to generate eximstats reports from the exim main logs
# The logs are processed in to daily reports on the mailservers,
# before being copied over to this server each night
#
# This script depends on the mail servers' log rotation jobs,
# and their cron jobs that run aftwards to generate the daily
# reports
#
# The script creates daily, weekly and monthly reports. It also keeps
# the last copy of any given day, week or month to allow comaprisons.
#
# The script needs to run each day, it won't catch up if any days are
# missed. If that happens, weekly and monthly reports will still be
# created but won't be accurate.
################
# Configuration
EXIMSTATS=/usr/local/bin/eximstats
#EXIMSTATS=/opt/local/sbin/eximstats
MAIL_SERVERS="foo bar baz"
TEXT_REPORTS="/var/lib/eximstat"
CURRENT_REPORTS_DIR=/var/www/html/eximstats
PREVIOUS_REPORTS_DIR="${CURRENT_REPORTS_DIR}/previous"
NICENESS=39
DAILY_REPORT_LABEL=daily
WEEKLY_REPORT_LABEL=weekly
MONTHLY_REPORT_LABEL=monthly
########################
# Options for eximstats
ARGS="$(cat <<EOT
-pattern Spam '/ rejected after DATA: Spam messages are not welcome here./' \
-pattern Viruses '/ rejected after DATA: This message contains a virus/' \
-ne \
-nr \
-charts \
-merge
)"
############
# functions
# date adjusts to the given number of days in the past and
# formatted using the provided formatting string
# usage: pastDate <minus no. days> <format>
pastDate()
{
if [ ! $1 ]
then
exit 1
fi
if [ ! $2 ]
then
exit 1
fi
if [ `uname` = "Darwin" ]
then
echo $(date -v -"$1"d +"$2")
fi
if [ `uname` = "Linux" ]
then
echo $(date --date="$1 day ago" +"$2")
fi
}
##############################
# Directory Checks / Creation
# Check required directories are writable - try to create them if missing or
# exit.
if [ ! -d "${CURRENT_REPORTS_DIR}" -o ! -w "${CURRENT_REPORTS_DIR}" ]
then
echo "The reports directory, "
echo "'${CURRENT_REPORTS_DIR}',"
echo "does not exist or is not writable"
exit 1
fi
DAILY_DIR="${CURRENT_REPORTS_DIR}/${DAILY_REPORT_LABEL}"
test -e "${DAILY_DIR}" || test -d "${DAILY_DIR}" || mkdir "${DAILY_DIR}"
if [ ! -d "${DAILY_DIR}" -o ! -w "${DAILY_DIR}" ]
then
echo "The daily reports directory, "
echo "'${DAILY_DIR}',"
echo " does not exist or is not writable"
exit 1
fi
WEEKLY_DIR="${CURRENT_REPORTS_DIR}/${WEEKLY_REPORT_LABEL}"
test -e "${WEEKLY_DIR}" || test -d "${WEEKLY_DIR}" || mkdir "${WEEKLY_DIR}"
if [ ! -d "${WEEKLY_DIR}" -o ! -w "${WEEKLY_DIR}" ]
then
echo "The monthly reports directory, "
echo "'${WEEKLY_DIR}',"
echo " does not exist or is not writable"
exit 1
fi
MONTHLY_DIR="${CURRENT_REPORTS_DIR}/${MONTHLY_REPORT_LABEL}"
test -e "${MONTHLY_DIR}" || test -d "${MONTHLY_DIR}" || mkdir "${MONTHLY_DIR}"
if [ ! -d "${MONTHLY_DIR}" -o ! -w "${MONTHLY_DIR}" ]
then
echo "The monthly reports directory, "
echo "'${MONTHLY_DIR}',"
echo " does not exist or is not writable"
exit 1
fi
if [ ! -d "${PREVIOUS_REPORTS_DIR}" -o ! -w "${PREVIOUS_REPORTS_DIR}" ]
then
echo "The previous reports directory, "
echo "'${PREVIOUS_REPORTS_DIR}',"
echo "does not exist or is not writable"
exit 1
fi
PREV_DAILY_DIR="${PREVIOUS_REPORTS_DIR}/${DAILY_REPORT_LABEL}"
test -e "${PREV_DAILY_DIR}" || test -d "${PREV_DAILY_DIR}" || mkdir "${PREV_DAILY_DIR}"
if [ ! -d "${PREV_DAILY_DIR}" -o ! -w "${PREV_DAILY_DIR}" ]
then
echo "The daily reports directory, "
echo "'${PREV_DAILY_DIR}',"
echo " does not exist or is not writable"
exit 1
fi
PREV_WEEKLY_DIR="${PREVIOUS_REPORTS_DIR}/${WEEKLY_REPORT_LABEL}"
test -e "${PREV_WEEKLY_DIR}" || test -d "${PREV_WEEKLY_DIR}" || mkdir "${PREV_WEEKLY_DIR}"
if [ ! -d "${PREV_WEEKLY_DIR}" -o ! -w "${PREV_WEEKLY_DIR}" ]
then
echo "The monthly reports directory, "
echo "'${PREV_WEEKLY_DIR}',"
echo " does not exist or is not writable"
exit 1
fi
PREV_MONTHLY_DIR="${PREVIOUS_REPORTS_DIR}/${MONTHLY_REPORT_LABEL}"
test -e "${PREV_MONTHLY_DIR}" || test -d "${PREV_MONTHLY_DIR}" || mkdir "${PREV_MONTHLY_DIR}"
if [ ! -d "${PREV_MONTHLY_DIR}" -o ! -w "${PREV_MONTHLY_DIR}" ]
then
echo "The monthly reports directory, "
echo "'${PREV_MONTHLY_DIR}',"
echo " does not exist or is not writable"
exit 1
fi
########
# Dates
# these dates relate to the log files, i.e. yesterdays logs and activity.
DAY_OF_WEEK=$(pastDate 1 %w)
WEEK=$(pastDate 1 %U)
MONTH=$(pastDate 1 %m)
YEAR=$(pastDate 1 %Y)
DATE=$(pastDate 1 %F)
PREV_DATE="$(($YEAR - 1))${DATE:(-6)}"
PREV_PREV_DATE="$(($YEAR - 2))${DATE:(-6)}"
# if today is the 1st of the month, then yesterday was the end of the month
MONTH_ENDED=$(
test $(date +%d) = 01 && echo true
)
###########################
# Create Daily HTML Report
if [ -d "${DAILY_DIR}/${PREV_DATE}" ]
then
echo "Moving previous Daily HTML report"
[ -d "${PREV_DAILY_DIR}/${PREV_PREV_DATE}" ] && rm -rf "${PREV_DAILY_DIR}/${PREV_PREV_DATE}"
mv "${DAILY_DIR}/${PREV_DATE}" "${PREV_DAILY_DIR}/${PREV_DATE}"
fi
echo "Creating Daily HTML Report"
mkdir -p "${DAILY_DIR}/${DATE}"
REPORT_LIST=""
for server in $MAIL_SERVERS
do
if [ -f "$TEXT_REPORTS/$server/$DATE.txt" ]
then
REPORT_LIST="$REPORT_LIST $TEXT_REPORTS/$server/$DATE.txt"
else
echo "WARNING: Missing report $TEXT_REPORTS/$server/$DATE.txt"
fi
done
if [ "$REPORT_LIST" ]
then
nice -n ${NICENESS} "${EXIMSTATS}" \
${ARGS} \
-chartdir "${DAILY_DIR}/${DATE}" \
-html="${DAILY_DIR}/${DATE}/index.html" \
$REPORT_LIST
else
echo "WARNING: Unable to create merged relays report for $DATE. No reports found."
fi
echo
#######################
# Create Weekly Report
if [ $DAY_OF_WEEK -eq 6 ]
then
if [ -d "${WEEKLY_DIR}/${WEEK}" ]
then
echo "Moving previous Weekly HTML report"
if [ -d "${PREV_WEEKLY_DIR}/${WEEK}" ]
then
rm -rf "${PREV_WEEKLY_DIR}/${WEEK}"
fi
mv "${WEEKLY_DIR}/${WEEK}" "${PREV_WEEKLY_DIR}/${WEEK}"
fi
echo "Creating Weekly HTML Report"
REPORT_LIST=""
DATE_ADJUSTMENT=$(( $DAY_OF_WEEK + 1 ))
while [ $DATE_ADJUSTMENT -ge 1 ]
do
REPORT_DATE=$(pastDate $DATE_ADJUSTMENT %F)
for server in $MAIL_SERVERS
do
if [ -f "${TEXT_REPORTS}/$server/${REPORT_DATE}.txt" ]
then
REPORT_LIST="$REPORT_LIST $TEXT_REPORTS/$server/$REPORT_DATE.txt"
else
echo "WARNING: Missing report $TEXT_REPORTS/$server/$DATE.txt"
continue
fi
done
DATE_ADJUSTMENT=$(( $DATE_ADJUSTMENT - 1 ))
done
if [ "$REPORT_LIST" ]
then
mkdir -p "${WEEKLY_DIR}/${WEEK}"
nice -n ${NICENESS} "${EXIMSTATS}" \
${ARGS} \
-chartdir "${WEEKLY_DIR}/${WEEK}" \
-html="${WEEKLY_DIR}/${WEEK}/index.html" \
$REPORT_LIST
else
echo "WARNING: Unable to create merged relays report for week $WEEK. No reports found."
fi
echo
fi
########################
# Create Monthly Report
if [ $MONTH_ENDED ]
then
if [ -d "${MONTHLY_DIR}/${MONTH}" ]
then
echo "Moving previous Monthly HTML report"
if [ -d "${PREV_MONTHLY_DIR}/${MONTH}" ]
then
rm -rf "${PREV_MONTHLY_DIR}/${MONTH}"
fi
mv "${MONTHLY_DIR}/${MONTH}" "${PREV_MONTHLY_DIR}/${MONTH}"
fi
echo "Creating Monthly HTML Report"
REPORT_LIST=""
for server in $MAIL_SERVERS
do
for REPORT in $(ls $TEXT_REPORTS/$server/${YEAR}-${MONTH}-*.txt)
do
REPORT_LIST="$REPORT_LIST $REPORT"
done
done
if [ "$REPORT_LIST" ]
then
mkdir -p "${MONTHLY_DIR}/${MONTH}"
nice -n ${NICENESS} "${EXIMSTATS}" \
${ARGS} \
-chartdir "${MONTHLY_DIR}/${MONTH}" \
-html="${MONTHLY_DIR}/${MONTH}/index.html" \
$REPORT_LIST
else
echo "WARNING: Unable to create merged relays report for month $MONTH. No reports found."
fi
echo
fi
#########################
# Delete old cache files
if [ $MONTH_ENDED ]
then
echo "Removing old text report files"
cd "${TEXT_REPORTS}"
if [ ${MONTH##0} -gt 1 ]
then
RM_MONTH=$(printf "%0#2d" $((${MONTH##0} - 1)) )
RM_YEAR=$YEAR
else
RM_MONTH=12
RM_YEAR=$(($YEAR - 1))
fi
for server in $MAIL_SERVERS
do
for REPORT_FILE in $(ls ${TEXT_REPORTS}/${server}/${RM_YEAR}-${RM_MONTH}-*.txt)
do
rm "${REPORT_FILE}"
done
done
fi
