/Main_Page

::You must have ninja focus to complete your mission::NinjaFocus::

Squid

Views:

Custom build of squid 2.6 on deimos.main.varndean.ac.uk

Contents

quick facts

  • deimos.main.varndean.ac.uk
  • /etc/squid/
  • /etc/init.d/squid
  • /usr/local/sbin/
  • /usr/local/src/
  • /var/run/squid.pid
  • /var/cache/squid/cache0
  • /var/log/squid/
  • /var/lib/squid/malware - web sites with malware, see http://www.malware.com.br/index.shtml#blocklist
  • user: squid
  • group: squid
  • cache volume: /dev/evms/lvm2/system/cache0
  • volume size: 72GB
  • volume filesystem: ext2
  • cache directory: /var/cache/squid/cache0
  • maximum cache size: 60GB

rc

  • service squid start|stop|restart|reload
  • /etc/init.d/squid start|stop|restart|reload

squid info and resources

compiling squid

# to execute run: bash ./deimos-configure
export CFLAGS="-march=nocona -O2 -pipe"
export CXXFLAGS="-march=nocona -O2 -pipe"
./configure \
--prefix=/usr/local \
--bindir=/usr/local/bin \
--sbindir=/usr/local/sbin \
--libexecdir=/usr/local/libexec \
--datadir=/usr/local/share \
--sysconfdir=/etc/squid \
--localstatedir=/var/run \
--libdir=/usr/local/lib \
--infodir=/usr/local/share/info \
--mandir=/usr/local/man \
--enable-async-io \
--enable-xmalloc-statistics \
--enable-storeio=ufs,aufs \
--enable-removal-policies=lru \
--enable-delay-pools \
--enable-snmp \
--enable-arp-acl \
--enable-htcp \
--enable-ssl \
--enable-cache-digests \
--enable-basic-auth-helpers=LDAP \
--enable-digest-auth-helpers=ldap \
--enable-external-acl-helpers=ip_user,ldap_group \
--with-large-files \
--with-aio \
--with-dl \
--with-pthreads \
--enable-removal-policies=lru,heap \
--disable-internal-dns

squid.conf

The squid config file is big! here are some of the more important sections shown with line numbers (correct at the time of writing, use them as a guide only). run "service squid reload" as root after any config changes 

     89 http_port 10.10.254.254:3128
     90 http_port 127.0.0.1:3128

    760 cache_mem 64 MB

    794 maximum_object_size 8192 KB

    866 cache_replacement_policy heap LFUDA

   1018 cache_dir aufs /var/cache/squid/cache0 61440 256 1024

   1106 access_log /var/log/squid/access.log squid

   1124 cache_store_log /var/log/squid/store.log

   1222 pid_filename /var/run/squid.pid

   1465 url_rewrite_program /usr/local/bin/squidGuard -c /etc/squid/squidGuard.conf

   1475 url_rewrite_children 5

   1802 auth_param basic program /usr/local/libexec/squid_ldap_auth -b "o=vsfc" -s sub -a always -f "cn=%s" ldap.main.varndean.ac.uk   
  1803 auth_param basic children 15   
  1804 auth_param basic realm Varndean College Internet Access   
  1805 auth_param basic credentialsttl 30 minutes

   1815 authenticate_cache_garbage_interval 15 minutes

   1824 authenticate_ttl 30 minutes

   1836 authenticate_ip_ttl 2 minutes

   1930 external_acl_type ldap_group children=15 %LOGIN /usr/local/libexec/squid_ldap_group -b "o=vsfc" -B "o=vsfc" -s sub -f "(&(cn=%g)(member=%u)(objectClass=group))" -F "cn=%s" -v 3 ldap.main.varndean.ac.uk

   2435 acl all src 0.0.0.0/0.0.0.0
   2436 acl manager proto cache_object
   2437 acl localhost src 127.0.0.1/255.255.255.255
   2438 acl to_localhost dst 127.0.0.0/8
   2439 acl SSL_ports port 443
   2440 acl Safe_ports port 80      # http
   2441 acl Safe_ports port 21      # ftp
   2442 acl Safe_ports port 443     # https
   2443 acl Safe_ports port 70      # gopher
   2444 acl Safe_ports port 210     # wais
   2445 acl Safe_ports port 1025-65535  # unregistered ports
   2446 acl Safe_ports port 280     # http-mgmt
   2447 acl Safe_ports port 488     # gss-http
   2448 acl Safe_ports port 591     # filemaker
   2449 acl Safe_ports port 777     # multiling http
   2450 acl CONNECT method CONNECT
   2454 acl lan src 10.10.0.0/16
   2455 acl staff src 10.10.100.1-10.10.100.254/32
   2456 acl isdept src 10.10.200.1-10.10.200.100/32
   2454 acl moreThanOneConnection max_user_ip -s 1
   2455 acl authenticatedUsers proxy_auth REQUIRED
   2456 acl BannedInternetUsers external ldap_group BannedInternetUsers
   2457 acl malware url_regex -i "/var/lib/squid/malware"

   2583 http_access allow malwareInformationWebsite
   2584 http_access deny malware
   2585 deny_info http://malware.com.br/denied.shtml malware
   2586 http_access deny BannedInternetUsers
   2587 http_access allow lan
   2588 http_access allow authenticatedUsers

squidGuard

squidGuard is a redirector for squid, it handles all acl's and internet content filtering.

Config file = /etc/squid/squidGuard.conf 

# CONFIG FILE FOR SQUIDGUARD
dbhome /var/lib/squidGuard
logdir /var/log/squid

time workhours {
	weekly mtwhf 08:00 - 16:30
	date *-*-01  08:00 - 16:30
}

# SOURCE ADDRESSES:

src admin {
	ip		1.2.3.4 1.2.3.5
	user		root foo bar
}

src rm1819 {
    ip      10.10.18.0-10.10.18.255
    ip      10.10.19.0-10.10.19.255
}

src foo-clients {
	ip		172.16.2.32-172.16.2.100 172.16.2.100 172.16.2.200
}

src bar-clients {
	ip		172.16.4.0/26
}

# DESTINATION CLASSES:

dest adult {
	domainlist	dest/adult/domains
	urllist		dest/adult/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest aggressive {
	domainlist	dest/aggressive/domains
	urllist	dest/aggressive/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}
dest antispyware {
	domainlist	dest/antispyware/domains
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest dialers {
	domainlist	dest/dialers/domains
	urllist	dest/dialers/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest hacking {
	domainlist	dest/hacking/domains
	urllist	dest/hacking/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest inhouse {
	domainlist	dest/inhouse/domains
	urllist	dest/inhouse/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}
dest inhouse-white {
	domainlist	dest/inhouse-white/domains
	urllist	dest/inhouse-white/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}


dest phishing {
	domainlist	dest/phishing/domains
	urllist	dest/phishing/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}
dest porn {
	domainlist	dest/porn/domains
	urllist	dest/porn/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}
dest proxy {
	domainlist	dest/proxy/domains
	urllist	dest/proxy/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest spyware {
	domainlist	dest/spyware/domains
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}
dest timewasting {
    domainlist dest/timewasting/domains
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest virusinfected {
	domainlist	dest/virusinfected/domains
	urllist	dest/virusinfected/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}
dest warez {
	domainlist	dest/warez/domains
	urllist	dest/warez/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u}

dest whitelist {
	domainlist	dest/whitelist/domains
	urllist	dest/whitelist/urls
	redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u
}

acl {
	admin {
		pass	 any
	}

	default {
		pass	 !virusinfected !phishing inhouse-white !proxy whitelist !inhouse !aggressive !dialers !hacking !porn !spyware !warez all
		redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u
	}
	
    rm1819 {
        pass	 !timewasting !virusinfected !phishing inhouse-white !proxy whitelist !inhouse !aggressive !dialers !hacking !porn !spyware !warez all
		redirect http://deimos.main.varndean.ac.uk/content-block.html?clientaddr=%a&clientname=%n&clientident=%i&srcclass=%s&targetclass=%t&url=%u
    }
}

Updating subscribed 3rd party blocklists

To update from the subscribed blocklist provider (urlblacklist.com)- simply run the following as root 

/usr/local/bin/squidGuard-blocklist-update

This script will yank down any new blocklists but won't overwrite any old ones that don't have updates, this allows us to use custom blocklists and not have to worry about them getting blasted during an update.

Running this script will result in the proxy server not accepting connections for roughly 10 minutes whilst the lists are updated. Obviously this means a loss of internet connectivity for anyone relying on the proxy.

The script is as follows 

#!/bin/bash
###
# UpdateBL - refresh Squid Guard Blocklists
#
# Version: 0.1-varndean
# Date: Aug 03 2007
# Author (well, man who hacked it up a bit): Kieran Whitbread < k at kraftycuts net >
# based largely on the work of Fernand Jonker < fernand at futuragts.com >, 
# based largely on the work of Christopher Rath < christopher at rath.ca >
###
# Don't allow undefined variables.
set -u

###
# Settings you must configure.  
#   BL_URL - the Blacklist's URL - test file enabled by default.
#       You must change this to the bigblacklist to download the
#       full blacklist file.
#   B_PATH - where the Blacklist database is stored.
#   SG_UGID - the userid and group which must "own" the Blacklist
#       database files (format: "<userid>:<group>)
#   DG_PATH - where the DansGuardian Binary is located
#   
# 
#export BL_URL=${BL_URL:="http://urlblacklist.com/cgi-bin/commercialdownload.pl?type=download&file=smalltestlist"}
#export BL_URL_INFO=${BL_URL_INFO:="http://urlblacklist.com/cgi-bin/commercialdownload.pl?type=information&file=smalltestlist"}
export BL_URL_INFO=${BL_URL_INFO:="http://urlblacklist.com/cgi-bin/commercialdownload.pl?type=information&file=bigblacklist"}
export BL_URL=${BL_URL:="http://urlblacklist.com/cgi-bin/commercialdownload.pl?type=download&file=bigblacklist"}

# IMPORTANT - The blacklist is COMMERCIAL.  If you download without a subscription you
#             are stealing.  You may try 1 download of the big list for free to test.
#             For details see: http://urlblacklist.com/

#
export SG_STATE_DIR="/var/lib/squidGuard"
export BL_INFO_FILE="${SG_STATE_DIR}/blacklists.info"
export BL_VERSION_FILE="${SG_STATE_DIR}/blacklists.version"
export DB_PATH=${DB_PATH:="${SG_STATE_DIR}/dest"}
export HOME_DIR="/tmp"
export SG_UGID=${SG_UGID:="squid:squid"}
export SG_PATH=${SG_PATH:="/usr/local/bin"}
export UNCOMP_CMD="/bin/gunzip"
export UNTAR_DIR="blacklists"
export VERS="0.1"

# Create a few working variables.
#export BL_TAR_BASE="`basename ${BL_URL}`"
export BL_TAR_BASE="blacklists.tar.gz"
export TMP_DIR="/tmp/blacklists"
export BL_TAR_FULL="${TMP_DIR}/${BL_TAR_BASE}"
#export http_proxy="127.0.0.1:3128"

# We need to check for updates
export BL_URL_INFO=`wget -q -Y on "${BL_URL_INFO}" -O - | head -n 1`
#echo "BL_URL_INFO=${BL_URL_INFO}"
export BL_DATE_NEW=`echo ${BL_URL_INFO} | tr , \\\n  | tr -d \" | head -n 1`
#echo "BL_DATE_NEW=${BL_DATE_NEW}"
export BL_MD5SUM_NEW=`echo ${BL_URL_INFO} | tr , \\\n  | tr -d \" | grep -v "${BL_DATE_NEW}" | head -n 1`
#echo "BL_MD5SUM_NEW=${BL_MD5SUM_NEW}"
if [ -e ${BL_INFO_FILE} ]
then
    export BL_DATE=`cat ${BL_INFO_FILE} | grep "DATE:" | sed 's/DATE://'`
    if [ "${BL_DATE}" = "${BL_DATE_NEW}" ]
    then
# No new update:
# aborting Blacklist refresh.
    exit 0
    fi
fi

# Starting Blacklist update: 
# We use $TMP_DIR as a working directory for wget and the untar process,
# so we start by cd-ing into it.  We create it if it doesn't exist, and
# if there is already something in the way then we abort.
if [ ! -d "${TMP_DIR}" ]
then
    if [ -e "${TMP_DIR}" ]
    then
        echo "ERROR: ${TMP_DIR} already exists, but isn't a directory;"
        echo "       aborting Blacklist refresh."
        exit 1
    fi
    
    mkdir "${TMP_DIR}"
fi

cd "${TMP_DIR}"
if [ "$?" != "0" ]
then
    echo "ERROR: unable to cd into working directory,"
    echo "       ${TMP_DIR}"
    exit 1
else
    if [ -f "${BL_TAR_FULL}" ]
    then
        rm -f "${BL_TAR_FULL}"
    fi
    
    if [ -f "./${BL_TAR_BASE}" ]
    then
# Removing old ${BL_TAR_BASE}.
        rm -f "./${BL_TAR_BASE}"
    fi
    
# Running wget to retrieve new lists.
    wget -q -Y on "${BL_URL}" -O ${BL_TAR_FULL}
    if [ "$?" != "0" ]
    then
        echo "ERROR: unable to retrieve new lists,"
        echo "       aborting blacklist refresh."
        exit 1
    else
# Succesfully retrieved new lists.

# Uncomment if you have md5sum program installed
   echo "Checking md5sum"
   export BL_MD5SUM=`md5sum ${BL_TAR_BASE} | tr \  \\\n | head -n 1`
#   echo "BL_MD5SUM=${BL_MD5SUM}"
   if [ "${BL_MD5SUM_NEW}" != "${BL_MD5SUM}" ]
       then
       echo "ERROR: md5sum doesn't match,"
           echo "       aborting blacklist refresh."
       rm -f "./${BL_TAR_BASE}"
       cd /tmp
       rm -rf ${TMP_DIR}
           exit 1
   fi

# Untaring Blacklist archive.
        
        tar -zxvf ${BL_TAR_FULL}
        if [ "$?" != "0" ]
        then
            echo "ERROR: unable to extract new lists,"
            echo "       aborting blacklist refresh."
            exit 1
    else
# Moving new lists into place.
            for i in "${UNTAR_DIR}"/*
            do
                export ib="`basename ${i}`"
                if [ -d "${DB_PATH}/${ib}" ]
                then
                    rm -rf "${DB_PATH}/${ib}"
                fi
    
                mv "${UNTAR_DIR}/${ib}" "${DB_PATH}"

            done

# Remove temporary files and folders.
        cd /tmp
        rm -rf /tmp/blacklists

# Change owner and permissions.
            chown -R "${SG_UGID}" "${DB_PATH}"
            chmod -R 755 "${DB_PATH}"
        
# Writting information in blacklists.info and blacklst.version
        echo "DATE:${BL_DATE_NEW}" > ${BL_INFO_FILE}
        echo "MD5SUM:${BL_MD5SUM_NEW}" >> ${BL_INFO_FILE}
        echo "${BL_DATE_NEW}" > ${BL_VERSION_FILE} 
        chown root:root ${BL_VERSION_FILE} ${BL_INFO_FILE}
        chmod 644 ${BL_VERSION_FILE} ${BL_INFO_FILE}

# Restarting Squid.
        /etc/init.d/squid restart >/dev/null 2>&1

# Finished Blacklist update.

            exit 0
            ####
            #### If everything went well, we exited here.
            ####
        fi
    fi
fi

Custom Blocklists

Blocklists reside in

/var/lib/squidGuard/dest/

We have 2 main custom blocklist categories -

Whitelist - in which urls are placed in the event of us wanting to explicitly allow access to a site

timewasting - this includes sites like myspace, bebo, facebook, youtube and ebay- at the moment this list is only imposed upon machines in rooms 18 and 19 (02/2008)

see http://www.squidguard.org/Doc/ for full documentation

Once you have made changes to squidGuard.conf or any of the blocklists (please read the documentation linked above first) run 

squidGuard -C all

This will hash all the text blocklists, after this run; 

/usr/local/sbin/squid -k reconfigure

This forces squid to reconfigure quietly and won't kick all clients off the server / stop accepting new connections as opposed to using the init script or 'service' command to restart squid.

Retrieved from "http://wiki.ninjafocus.net/Squid"

Main Menu

Personal tools

Toolbox