Re: User problem with Gentoo and PyDS Aggregator

"""

Python Desktop Server - Downstreaming Tool

Copyright (c) 2002, Georg Bauer <gb-BRhJDZTO+/[email protected]>

Permission is hereby granted, free of charge, to any person obtaining a copy of 
this software and associated documentation files (the "Software"), to deal in 
the Software without restriction, including without limitation the rights to 
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 
the Software, and to permit persons to whom the Software is furnished to do so, 
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all 
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""

# $Id: DownstreamTool.py,v 1.26 2003/10/07 09:08:11 gb Exp $

import os
import re
import sys
import md5
import time
import string
import urllib
import gzip
import mimetypes
import urlparse

import PyDS.Tool

class UrlOpener(urllib.URLopener):

	def __init__(self, cache, force, *args):
		apply(urllib.URLopener.__init__, (self,) + args)
		self.addheaders = [
			('User-agent', PyDS.Tool.version),
			('Accept-encoding', 'gzip')
		]
		self.tries = 0
		self.maxtries = 5
		self.cache = cache
		self.force = force
		self.cachedResult = 0
		self.isHTTP = 0
		self.lastURL = None
		self.message = ''
		self.verbose = 0
	
	def getTheUrl(self, url):
		if type(url) == type((1,1)):
			return url[1]
		elif type(url) == type(''):
			return '%s:%s' % (self.type, url)
	
	def open_http(self, url, data=None):
		numheaders = len(self.addheaders)
		self.isHTTP = 1
		self.lastURL = self.getTheUrl(url)
		try:
			theurl = self.getTheUrl(url)
			self.message = _('opening url: <a href="%s">%s</a>') % (theurl, theurl)
			if not(self.force):
				for h in self.cache._getUrlHeaders(theurl):
					apply(self.addheader, h)
					self.message += _('<br>adding Header "%s: %s"') % h
			url2 = urlparse.urlparse(url[1])
			url = (url2[1], url[1])
			res = urllib.URLopener.open_http(self, url, data)
			self.message = self.message.replace('%', '%%')
			if self.verbose:
				self.cache.logVerbose(self.message)
			else:
				self.cache.log(self.message)
			return res
		finally:
			self.addheaders = self.addheaders[:numheaders]
	
	def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
		self.tries += 1
		if self.maxtries and self.tries >= self.maxtries:
			return self.http_error_default(
				url, fp, 500,
				"Internal Server Error: Redirect Recursion",
				headers
			)
		result = self.redirect_internal(
			url, fp, errcode, errmsg, headers, data
		)
		self.tries = 0
		return result
	
	def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
		return self.http_error_302(
			url, fp, errcode, errmsg, headers, data
		)
	
	def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
		return self.http_error_302(
			url, fp, errcode, errmsg, headers, data
		)
	
	def http_error_304(self, url, fp, errcode, errmsg, headers, data=None):
		void = fp.read()
		fp.close()
		self.cachedResult = 1
		if self.cache:
			self.message += _('<br>cache content still current, skipping download')
			self.verbose = 1
			theurl = self.getTheUrl(url)
			return self.open('file:%s' % self.cache.getCachePath(theurl))
		else:
			return None

	def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
		if headers.has_key('location'):
			newurl = headers['location']
		elif headers.has_key('uri'):
			newurl = headers['uri']
		else:
			return
		void = fp.read()
		fp.close()
		theurl = self.getTheUrl(url)
		newurl = urllib.basejoin(theurl, newurl)
		self.message += _('<br>redirecting to <a href="%s">%s</a>') % (theurl, theurl)
		if data is None:
			return self.open(newurl)
		else:
			return self.open(newurl, data)

auth_cache = {}

class FancyUrlOpener(UrlOpener):

    def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
        """Error 401 -- authentication required.
        See this URL for a description of the basic authentication scheme:
        http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt"""
        if not headers.has_key('www-authenticate'):
            URLopener.http_error_default(self, url, fp,
                                         errcode, errmsg, headers)
        stuff = headers['www-authenticate']
        import re
        match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
        if not match:
            URLopener.http_error_default(self, url, fp,
                                         errcode, errmsg, headers)
        scheme, realm = match.groups()
        if scheme.lower() != 'basic':
            URLopener.http_error_default(self, url, fp,
                                         errcode, errmsg, headers)
        name = 'retry_' + self.type + '_basic_auth'
        if data is None:
            return getattr(self,name)(url, realm)
        else:
            return getattr(self,name)(url, realm, data)

    def retry_http_basic_auth(self, url, realm, data=None):
        host, selector = splithost(url)
        i = host.find('@') + 1
        host = host[i:]
        user, passwd = self.get_user_passwd(host, realm, i)
        if not (user or passwd): return None
        host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
        newurl = 'http://' + host + selector
        if data is None:
            return self.open(newurl)
        else:
            return self.open(newurl, data)

    def retry_https_basic_auth(self, url, realm, data=None):
        host, selector = splithost(url)
        i = host.find('@') + 1
        host = host[i:]
        user, passwd = self.get_user_passwd(host, realm, i)
        if not (user or passwd): return None
        host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
        newurl = '//' + host + selector
        return self.open_https(newurl, data)

    def get_user_passwd(self, host, realm, clear_cache = 0):
    	global auth_cache
        key = realm + '@' + host.lower()
        if auth_cache.has_key(key):
            if clear_cache:
                del auth_cache[key]
            else:
                return auth_cache[key]
        user, passwd = self.prompt_user_passwd(host, realm)
        if user or passwd: auth_cache[key] = (user, passwd)
        return user, passwd

    def prompt_user_passwd(self, host, realm):
        """Override this in a GUI environment!"""
        import getpass
        try:
            user = raw_input(_("Enter username for %s at %s: ") % (realm,
                                                                host))
            passwd = getpass.getpass(_("Enter password for %s in %s at %s: ") %
                (user, realm, host))
            return user, passwd
        except KeyboardInterrupt:
            print
            return None, None

class DownstreamTool(PyDS.Tool.StandardTool):

	def _initdb(self):
		try:
			self._acquire()
			self.cache = self.db.getas(
				"cache[url:S,cacheid:I,lastmodified:S,etag:S,encoding:S,ready:I,mimetype:S]"
			).ordered(1)
			self.prefs = self.db.getas(
				"prefs[maxid:I]"
			)
			if len(self.prefs) == 0:
				self.prefs.append({
					'maxid':0
				})
			self._commit()
		finally: self._release()

	def _initopts(self):
		try:
			self._acquire()
			self.showCalendar = 0
			cachedir = os.path.join(_PyDS.VARDIR, 'cache')
			if not(os.path.isdir(cachedir)):
				os.makedirs(cachedir)
		finally: self._release()

	def _initthread(self, server):
		while server.isRunning():
			time.sleep(10)
			if self.queue.length():
				while self.queue.length():
					(jobf, key, data) = self.queue.shift()
					jobf(key, data)
				self.logVerbose(_('work done, going to sleep again'))
		
	def _status(self):
		try:
			self._acquire()
			html = PyDS.Tool.StandardTool._status(self)
			html += '<ul>'
			html += _('<li>There are %d elements in the cache</li>') % (
				len(self.cache),
			)
			html += self._threadstatus()
			html += '</ul>'
			return html
		finally: self._release()

	# This method returns an url opener to access the outside world.
	# If force==1 the download will be forced, if force==0 the download
	# will use If-Modified-Since and If-None-Match headers from the
	# cache database.
	def getUrlOpener(self, force):
		try:
			self._acquire()
			upstream = self.getToolByNamespace('upstream')
			proxy = upstream.get('prefs', 'proxy')
			if proxy:
				proxy = 'http://%s/' % proxy
				if _flet.get('restore', 0):
					return FancyUrlOpener(self, force, {'http':proxy})
				else:
					return UrlOpener(self, force, {'http':proxy})
			else:
				if _flet.get('restore', 0):
					return FancyUrlOpener(self, force)
				else:
					return UrlOpener(self, force)
		finally: self._release()
	
	def _getUrlHeaders(self, url):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				headers = []
				if self.cache[idx].ready:
					if self.cache[idx].lastmodified:
						headers.append(('If-Modified-Since', self.cache[idx].lastmodified))
					if self.cache[idx].etag:
						headers.append(('If-None-Match', self.cache[idx].etag))
				return headers
			else:
				return []
		finally: self._release()
	
	def _getCachePathForId(self, cacheid):
		path = os.path.join(_PyDS.VARDIR, 'cache', '%08d' % int(cacheid))
		return path
	
	def _getCacheUrlForId(self, cacheid):
		url = '/cache/%08d' % int(cacheid)
		return url
	
	def _getCacheId(self, url):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				return self.cache[idx].cacheid
			else:
				self.prefs[0].maxid += 1
				cacheid = self.prefs[0].maxid
				(mimetype, something) = mimetypes.guess_type(url)
				self.cache.append({
					'url':url,
					'lastmodified':'',
					'etag':'',
					'cacheid':cacheid,
					'encoding':'',
					'ready':0,
					'mimetype':mimetype
				})
				self._commit()
				return cacheid
		finally: self._release()
		
	def _updateCache(self, url, lastmodified, etag, encoding, mimetype):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				cache = self.cache[idx]
				cache.lastmodified = lastmodified
				cache.etag = etag
				cache.encoding = encoding
				cache.mimetype = mimetype
				self._commit()
		finally: self._release()
	
	def _readyCache(self, url, ready):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				self.cache[idx].ready = ready
				self._commit()
		finally: self._release()
	
	def _download(self, url, force):
		opener = self.getUrlOpener(force)
		path = self.getCachePath(url)
		handle = opener.open(url)
		if opener.cachedResult:
			handle.close()
		else:
			self._readyCache(url,0)
			(mimetype, something) = mimetypes.guess_type(url)
			if opener.isHTTP:
				lastmodified = handle.info().get('Last-Modified','')
				etag = handle.info().get('ETag','')
				encoding = handle.info().get('Content-Encoding','')
				mimetype = handle.info().get('Content-Type', mimetype)
			if not(mimetype):
				mimetype = 'application/octetstream'
			cache = open(self.getCachePath(url), 'wb')
			block = handle.read(100000)
			while block:
				cache.write(block)
				block = handle.read(100000)
			handle.close()
			cache.close()
			if opener.isHTTP:
				self._updateCache(url, lastmodified, etag, encoding, mimetype)
			self._readyCache(url,1)
		return url
	
	def index_html(self, req):
		try:
			self._acquire()
			html = '<form action="%s" method="POST">' % self.getUrl('delete_redir')
			html += '<table border=0 width="100%">'
			html += '<tr>'
			html += '<th>&nbsp;</th>'
			html += '<th>&nbsp;</th>'
			html += '<th align="left">%s</th>' % _('URL')
			html += '<th align="left">%s</th>' % _('id')
			html += '</tr>'
			for cache in self.cache:
				html += '<tr>'
				html += '<td><input type=checkbox name="id" value="%s" class="smallfont"></td>' % cache.cacheid
				if cache.ready:
					html += '<td class="whiteboxsmall"><b>R</b></td>'
				else:
					html += '<td class="whiteboxsmall">&nbsp;</td>'
				url = cache.url
				if len(url) > 60:
					url = url[:60] + ' ...'
				headerlines = ""
				if cache.lastmodified:
					headerlines = '<b>%s:</b> %s' % (
						_('last modified'),
						cache.lastmodified
					)
				if cache.etag:
					if headerlines:
						headerlines += ', '
					headerlines += '<b>%s:</b> %s' % (
						_('etag'),
						cache.etag
					)
				if cache.encoding:
					if headerlines:
						headerlines += ', '
					headerlines += '<b>%s:</b> %s' % (
						_('encoding'),
						cache.encoding
					)
				if cache.mimetype:
					if headerlines:
						headerlines += ', '
					if cache.ready:
						headerlines += '<b>%s:</b> <a href="%s">%s</a>' % (
							_('MIME type'),
							self._getCacheUrlForId(cache.cacheid),
							cache.mimetype
						)
					else:
						headerlines += '<b>%s:</b> %s' % (
							_('MIME type'),
							cache.mimetype
						)
				if headerlines:
					headerlines = '<br>' + headerlines
				html += '<td class="whiteboxsmall"><b>%s</b>%s</td>' % (url, headerlines)
				html += '<td class="whiteboxsmall">%d</td>' % cache.cacheid
				html += '</tr>'
			html += '</table>'
			html += '<input type=submit name="DownstreamSubmit" value="%s">' % _('Delete')
			html += '</form>'
			req.setLocalValue('title', _('Cached files'))
			return req.renderPage('BaseTemplate', body=html)
		finally: self._release()
	
	def delete_redir(self, req):
		try:
			self._acquire()
			todel = []
			sortedcache = self.cache.sort(self.cache.cacheid)
			for id in req.getAllValues('id'):
				(idx, found) = sortedcache.locate({'cacheid':int(id)})
				if found:
					fn = self._getCachePathForId(id)
					if os.path.exists(fn):
						os.unlink(fn)
					(idx, found) = self.cache.locate({'url':sortedcache[idx].url})
					if found:
						todel.append(idx)
			todel.sort()
			todel.reverse()
			for idx in todel:
				self.cache.delete(idx)
			self._commit()
			return self.getUrl()
		finally: self._release()
	
	# this method check wether a given URL is already in the cache
	def isCacheReady(self, url):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				return self.cache[idx].ready
			else:
				return 0
		finally:
			self._release()
	
	# this method returns the mimetype for a parsed cache URL. If the
	# URL can't be found, 'application/octetstream' is assumed
	def getCacheMimeType(self, url):
		m = re.match(r'/cache/(\d+)', url)
		if m:
			cacheid = int(m.group(1))
			try:
				self._acquire()
				res = self.cache.select({'cacheid':cacheid})
				if len(res):
					return res[0].mimetype
				else:
					return 'application/octetstream'
			finally: self._release()
		else:
			return 'application/octetstream'

	# this method returns the cache url of an external url
	def getCacheURL(self, url):
		cacheid = self._getCacheId(url)
		return self._getCacheUrlForId(cacheid)

	# this method returns the pathname of an url in the download cache
	def getCachePath(self, url):
		cacheid = self._getCacheId(url)
		return self._getCachePathForId(cacheid)

	# this method returns a file object on the cached object. It
	# respects the encoding attribute!
	def getCacheFile(self, url):
		if self.getCacheEncoding(url) == 'gzip':
			return gzip.open(self.getCachePath(url))
		else:
			return open(self.getCachePath(url))

	# this method returns the encoding of an url in the download cache
	def getCacheEncoding(self, url):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				return self.cache[idx].encoding
		finally:
			self._release()
		return ''
	
	# this method downloads a file, caches it and returns a open file pointer to the
	# cached content. The download may proceed in the background, then None is returned
	def download(self, url, background=0, force=0):
		if background:
			try:
				self._acquire()
				self.queue.append(
					self._download,
					url,
					force
				)
				return None
			finally: self._release()
		else:
			url = self._download(url, force)
			f = self.getCacheFile(url)
			return f

	# This method returns the modification time of a download from
	# the cache database
	def getModificationTime(self, url):
		try:
			self._acquire()
			(idx, found) = self.cache.locate({'url':url})
			if found:
				if self.cache[idx].ready:
					if self.cache[idx].lastmodified:
						return self.cache[idx].lastmodified
			return None
		finally: self._release()


__desc__ = _("This tool gives functionality to download files from the internet.")

downstream = DownstreamTool('downstream', _('Downstreaming'), __desc__, 915)
PyDS.Tool.registerTool(downstream)