Re: manually putting things into the cache

#! /bin/sh

# Script to do scoops for a remote requestor, and pass the
# results back to the requestor.

# Andy Rabagliati <[email protected]>

# Wizzy Digital Courier  http://www.wizzy.org.za/

DOMAIN=wizzy.org.za

for f in /var/spool/uucp/.scoops/*
do
  if [ -f $f ]
  then
    while read key value
    do
      case $key in
        Url:)
          URL=$value
          ;;
        Priority:)
          PRIORITY=$value
          ;;
        Host:)
          HOST=$value
          ;;
        Level:)
          LEVEL=$value
          ;;
        Span:)
          if [ $value = yes ] ; then
            SPAN=--span-hosts
          fi
          ;;
        When:)
          WHEN=$value
          ;;
      esac
    done < $f
    echo $URL $PRIORITY $HOST $LEVEL $WHEN
    if [ -z $URL -o -z $PRIORITY -o -z $HOST -o -z $LEVEL -o -z $WHEN ] ; then
      echo $f busted
      continue
    fi
    site=`echo $URL | sed -e 's%[a-z]*://%%' -e s%/.*%%`
    if ! echo Xquit | telnet -e X $site 80 ; then
      continue
    fi

    # check for lock - allow for 4 concurrent fetches

    for i in 1 2 3 4
    do
        if [ ! -f /var/lock/uucp/woffle$i ]
        then
	    LOCK=/var/lock/uucp/woffle$i
	    break
        fi
    done
  
    if [ X$LOCK = X ]
    then
        exit		# not this time
    fi

    # the upstream proxy - should be online !!
    PROXY=localhost:8080
  
    WGET="wget --quiet --recursive --page-requisites --no-directories --delete-after --level $LEVEL $SPAN $URL"
  
    WOFFLEDIR=/tmp/woffle$$
    WOFFLECONFIG=$WOFFLEDIR/wwwoffle.conf
    WOFFLE="/usr/bin/wwwoffle -c $WOFFLECONFIG"
    UUROUTE=`/usr/bin/ldapsearch -x -LLL -b ou=mboxsvrs,dc=wizzy,dc=org,dc=za uuHost=$HOST uuRoute` || exit 1
    UUROUTE=`echo $UUROUTE | /bin/sed -ne '/uuRoute/s/.*: //p'`
    let PROXYPORT=$$%400*2+8090		# even number between 8090 and 9000
    let CONTROLPORT=$PROXYPORT+1
  
    if [ -z $UUROUTE ]
    then
        echo no route to $HOST in ldap
  	continue
    fi
  
    trap "/bin/rm -rf $LOCK $WOFFLEDIR" 0 1 2 15
  
    # create lock
    echo $$ > $LOCK
  
    mv $f /var/spool/uucp/.Temp
    SELF=$(/usr/bin/uuname -l)
  
    mkdir $WOFFLEDIR
    cd $WOFFLEDIR
    mkdir ftp http lasttime outgoing temp
  
    cat >$WOFFLECONFIG <<END_OF_CONFIG
# WWWOFFLE Configuration file
  
StartUp
{
   bind-ipv4         = 0.0.0.0
   http-port         = $PROXYPORT
   wwwoffle-port     = $CONTROLPORT
   spool-dir         = $WOFFLEDIR
   run-uid           = uucp
   run-gid           = uucp
   use-syslog        = yes
   password          = none
   max-servers       = 8
   max-fetch-servers = 4
}
  
Options
{
  dir-perm              = 0775
  file-perm             = 0664
}

LocalHost
{
   ${SELF}.${DOMAIN}
   localhost
}
  
MIMETypes
{
   default  = text/plain
  
   .Z       = application/x-compress
   .au      = audio/basic
   .avi     = video/x-msvideo
   .class   = application/java
   .cpio    = application/x-cpio
   .css     = text/css
   .deb     = application/octet-stream
   .dtd     = application/xml
   .dvi     = application/x-dvi
   .eps     = application/postscript
   .gif     = image/gif
   .gz      = application/x-gzip
   .htm     = text/html
   .html    = text/html
   .jpeg    = image/jpeg
   .jpg     = image/jpeg
   .js      = application/x-javascript
   .latex   = application/x-latex
   .man     = application/x-troff-man
   .me      = application/x-troff-me
   .mov     = video/quicktime
   .mpeg    = video/mpeg
   .mpg     = video/mpeg
   .ms      = application/x-troff-ms
   .pac     = application/x-ns-proxy-autoconfig
   .pbm     = image/x-portable-bitmap
   .pdf     = application/pdf
   .pgm     = image/x-portable-graymap
   .png     = image/png
   .pnm     = image/x-portable-anymap
   .ppm     = image/x-portable-pixmap
   .ps      = application/postscript
   .ras     = image/x-cmu-raster
   .rgb     = image/x-rgb
   .rpm     = application/octet-stream
   .rtf     = application/rtf
   .snd     = audio/basic
   .tar     = application/x-tar
   .tcl     = application/x-tcl
   .tex     = application/x-tex
   .texi    = application/x-texinfo
   .texinfo = application/x-texinfo
   .tif     = image/tiff
   .tiff    = image/tiff
   .tr      = application/x-troff
   .txt     = text/plain
   .vr      = model/vrml
   .wav     = audio/x-wav
   .wrl     = model/vrml
   .xbm     = image/x-xbitmap
   .xml     = application/xml
   .xpm     = image/x-xpixmap
   .xwd     = image/x-xwindowdump
   .zip     = application/zip
}
 
Proxy
{
   <http://*> proxy = $PROXY
}
END_OF_CONFIG
  
    #PERLSCRIPT_FILES='($h, $s)=($_=~ m%^(\S+).*http://([^/]+)/%); print "./http/$s/$h"; substr($h,0,1,"U"); print "./http/$s/$h";'
    /usr/sbin/wwwoffled -c $WOFFLECONFIG
    # put freshly online
    $WOFFLE -online
  
    cd /tmp
    logger -t scooping $URL
    http_proxy=http://localhost:$PROXYPORT/ $WGET
  
    # offline again
    $WOFFLE -offline
    $WOFFLE -kill		# cause wwwoffled to exit
  
    cd $WOFFLEDIR/http
    for site in *
    do
        cd $site
        for f in D*
        do
            if head -1 $f | grep -q '^HTTP.1.0 404 WWWOFFLE Refused Request'
            then
                rm -f $f U${f#D}
            fi
        done
        cd ..
        if [ 2 == $(stat -c %h $site) ]
        then
            rmdir $site
        fi
    done


    # send requested files back to client
    /bin/tar --create --gzip --dereference --directory $WOFFLEDIR http ftp | \
  	    uux --stdin $UUROUTE\!woffle_unpack
  
    # remove lock and temporary woffle instance
    /bin/rm -rf $LOCK $WOFFLEDIR
  fi
done