[PATCH] run MoinMoin as FastCGI
Oliver Graf <[email protected]>
| Newsgroups | gmane.comp.web.wiki.moin.devel |
|---|---|
| Message-ID | <[email protected]> |
Hi!
I've added a patch to ["MoinMoin:MoinMoinPatch"] (is this the way to
write wiki links in mails?) which adds a FastCGI ( http://fastcgi.com/ )
mode to MoinMoin.
Why FastCGI?
FastCGI is a method to run a script in an endless loop and answer
requests send to it by the http server. This saves initialization time
and allows to cache stuff in memory.
FastCGI can be run trough suexec wrapper and run with user rights instead
of server rights. And if you have a bunch of users on one server, this
might be a tough issue.
There are still some sharp points:
- it will only run multi-threaded, cause MoinMoin has lots of
sys.exit calls in it which will kill the single-threaded fastcgi
process (remember: it should not stop, cause we want to start it
only one time)
-> get rid of those sys.exit calls, us return values and ifs.
- multi-threaded is nice but also has problems, especially with those
globals. And there are some in MoinMoin, as a quick grep shows.
This might not result in errors, but it could...
-> add thread locks to code which accesses globals.
- os.environ is used. this is bad for a cgi that can handle multiple
concurrent requests, cause each request has it's own environment
(at least it should have...)
-> add environ to Request container and access environ only through
this interface (RequestFastCGI does do this for internal use)
To whoever has twisted knowledge: How does twisted handle those points?
Is twisted multi-threaded?
Oliver.
fastcgi-moin.diff
(text/plain, 22 KB)
--- moin--main--1.2--20031228-0700/wiki/cgi-bin/moin.fcg.orig 2003-12-28 14:42:12.000000000 +0100 +++ moin--main--1.2--20031228-0700/wiki/cgi-bin/moin.fcg 2003-12-28 19:33:00.000000000 +0100 @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - FastCGI Driver Script + + Copyright (c) 2003 by Oliver Graf <[email protected]> + All rights reserved, see COPYING for details. + + $Id$ +""" + +#import sys +#sys.path[0:0]=['/var/www/moin-main/lib/python', +# '/var/www/moin-main/wiki'] + +# +# no test mode, cause wikitest those not use Request container! +# +from MoinMoin import fcgimain +fcgimain.run() --- moin--main--1.2--20031228-0700/MoinMoin/util/thfcgi.py.orig 2003-12-28 14:07:37.000000000 +0100 +++ moin--main--1.2--20031228-0700/MoinMoin/util/thfcgi.py 2003-12-28 19:10:21.000000000 +0100 @@ -0,0 +1,521 @@ +# thfcgi.py - FastCGI communication with thread support +# +# Copyright Peter Åstrand <[email protected]> 2001 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# TODO: +# +# Compare compare the number of bytes received on FCGI_STDIN with +# CONTENT_LENGTH and abort the update if the two numbers are not equal. +# + +import os +import sys +import select +import string +import socket +import errno +import cgi +import thread +from cStringIO import StringIO +import struct + +# Maximum number of requests that can be handled +FCGI_MAX_REQS = 50 +FCGI_MAX_CONNS = 50 +FCGI_VERSION_1 = 1 +# Can this application multiplex connections? +FCGI_MPXS_CONNS = 0 + +# Record types +FCGI_BEGIN_REQUEST = 1 +FCGI_ABORT_REQUEST = 2 +FCGI_END_REQUEST = 3 +FCGI_PARAMS = 4 +FCGI_STDIN = 5 +FCGI_STDOUT = 6 +FCGI_STDERR = 7 +FCGI_DATA = 8 +FCGI_GET_VALUES = 9 +FCGI_GET_VALUES_RESULT = 10 +FCGI_UNKNOWN_TYPE = 11 +FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE + +# Types of management records +KNOWN_MANAGEMENT_TYPES = [FCGI_GET_VALUES] + +FCGI_NULL_REQUEST_ID = 0 + +# Masks for flags component of FCGI_BEGIN_REQUEST +FCGI_KEEP_CONN = 1 + +# Values for role component of FCGI_BEGIN_REQUEST +FCGI_RESPONDER = 1 +FCGI_AUTHORIZER = 2 +FCGI_FILTER = 3 + +# Values for protocolStatus component of FCGI_END_REQUEST +FCGI_REQUEST_COMPLETE = 0 # Request completed ok +FCGI_CANT_MPX_CONN = 1 # This app cannot multiplex +FCGI_OVERLOADED = 2 # Too busy +FCGI_UNKNOWN_ROLE = 3 # Role value not known + +# Struct format types +FCGI_BeginRequestBody = "!HB5x" +FCGI_Record_header = "!BBHHBx" +FCGI_UnknownTypeBody = "!B7x" +FCGI_EndRequestBody = "!IB3x" + +class Record: + """Class representing FastCGI records""" + def __init__(self): + self.version = FCGI_VERSION_1 + self.rec_type = FCGI_UNKNOWN_TYPE + self.req_id = FCGI_NULL_REQUEST_ID + self.content = "" + + # Only in FCGI_BEGIN_REQUEST + self.role = None + self.flags = None + self.keep_conn = 0 + + # Only in FCGI_UNKNOWN_TYPE + self.unknownType = None + + # Only in FCGI_END_REQUEST + self.appStatus = None + self.protocolStatus = None + + def read_pair(self, data, pos): + namelen = struct.unpack("!B", data[pos])[0] + if namelen & 128: + # 4-byte name length + namelen = struct.unpack("!I", data[pos:pos+4])[0] + pos += 4 + else: + pos += 1 + + valuelen = struct.unpack("!B", data[pos])[0] + if valuelen & 128: + # 4-byte value length + valuelen = struct.unpack("!I", data[pos:pos+4])[0] + pos += 4 + else: + pos += 1 + + name = data[pos:pos+namelen] + pos += namelen + value = data[pos:pos+valuelen] + pos += valuelen + + return (name, value, pos) + + def write_pair(self, name, value): + namelen = len(name) + if namelen < 128: + data = struct.pack("!B", namelen) + else: + # 4-byte name length + data = struct.pack("!I", namelen) + + valuelen = len(value) + if valuelen < 128: + data += struct.pack("!B", value) + else: + # 4-byte value length + data += struct.pack("!I", value) + + return data + name + value + + def readRecord(self, sock): + data = sock.recv(8) + if not data: + # No data recieved. This means EOF. + return None + + fields = struct.unpack(FCGI_Record_header, data) + (self.version, self.rec_type, self.req_id, + contentLength, paddingLength) = fields + + self.content = "" + while len(self.content) < contentLength: + data = sock.recv(contentLength - len(self.content)) + self.content = self.content + data + if paddingLength != 0: + sock.recv(paddingLength) + + # Parse the content information + if self.rec_type == FCGI_BEGIN_REQUEST: + (self.role, self.flags) = struct.unpack(FCGI_BeginRequestBody, + self.content) + self.keep_conn = self.flags & FCGI_KEEP_CONN + + elif self.rec_type == FCGI_UNKNOWN_TYPE: + self.unknownType = struct.unpack(FCGI_UnknownTypeBody, self.content) + + elif self.rec_type == FCGI_GET_VALUES or self.rec_type == FCGI_PARAMS: + self.values = {} + pos = 0 + while pos < len(self.content): + name, value, pos = self.read_pair(self.content, pos) + self.values[name] = value + elif self.rec_type == FCGI_END_REQUEST: + (self.appStatus, + self.protocolStatus) = struct.unpack(FCGI_EndRequestBody, + self.content) + + return 1 + + def writeRecord(self, sock): + content = self.content + if self.rec_type == FCGI_BEGIN_REQUEST: + content = struct.pack(FCGI_BeginRequestBody, self.role, self.flags) + + elif self.rec_type == FCGI_UNKNOWN_TYPE: + content = struct.pack(FCGI_UnknownTypeBody, self.unknownType) + + elif self.rec_type == FCGI_GET_VALUES or self.rec_type == FCGI_PARAMS: + content = "" + for i in self.values.keys(): + content = content + self.write_pair(i, self.values[i]) + + elif self.rec_type == FCGI_END_REQUEST: + content = struct.pack(FCGI_EndRequestBody, self.appStatus, + self.protocolStatus) + + # Align to 8-byte boundary + clen = len(content) + padlen = ((clen + 7) & 0xfff8) - clen + + hdr = struct.pack(FCGI_Record_header, self.version, self.rec_type, + self.req_id, clen, padlen) + + try: + sock.send(hdr + content + padlen*"\x00") + except socket.error: + # Write error, probably broken pipe. Exit thread. + thread.exit() + + +class Request: + """A request, corresponding to an accept():ed connection and + a FCGI request. + """ + def __init__(self, conn, req_handler, multi=1): + self.conn = conn + self.req_handler = req_handler + self.multi = multi + + self.keep_conn = 0 + self.req_id = None + + # Input + self.env = {} + self.env_complete = 0 + self.stdin = StringIO() + self.stdin_complete = 0 + self.data = StringIO() + self.data_complete = 0 + + # Output + self.out = StringIO() + self.err = StringIO() + + self.have_finished = 0 + + def run(self): + while 1: + if self.conn.fileno() < 1: + # Connection lost + return + + select.select([self.conn], [], []) + rec = Record() + if rec.readRecord(self.conn): + self._handle_record(rec) + else: + # EOF, connection closed. Break loop, end thread. + return + + def getFieldStorage(self): + self.stdin.reset() + return cgi.FieldStorage(fp=self.stdin, environ=self.env, + keep_blank_values=1) + + def _flush(self, stream): + stream.reset() + + rec = Record() + rec.rec_type = FCGI_STDOUT + rec.req_id = self.req_id + data = stream.read() + + if not data: + # Writing zero bytes would mean stream termination + return + + while data: + chunk, data = self.getNextChunk(data) + rec.content = chunk + rec.writeRecord(self.conn) + # Truncate + stream.reset() + stream.truncate() + + def flush_out(self): + self._flush(self.out) + + def flush_err(self): + self._flush(self.err) + + def finish(self, status=0): + if self.have_finished: + return + + self.have_finished = 1 + + # stderr + self.err.reset() + rec = Record() + rec.rec_type = FCGI_STDERR + rec.req_id = self.req_id + data = self.err.read() + while data: + chunk, data = self.getNextChunk(data) + rec.content = chunk + rec.writeRecord(self.conn) + rec.content = "" + rec.writeRecord(self.conn) # Terminate stream + + # stdout + self.out.reset() + rec = Record() + rec.rec_type = FCGI_STDOUT + rec.req_id = self.req_id + data = self.out.read() + while data: + chunk, data = self.getNextChunk(data) + rec.content = chunk + rec.writeRecord(self.conn) + rec.content = "" + rec.writeRecord(self.conn) # Terminate stream + + # end request + rec = Record() + rec.rec_type = FCGI_END_REQUEST + rec.req_id = self.req_id + rec.appStatus = status + rec.protocolStatus = FCGI_REQUEST_COMPLETE + rec.writeRecord(self.conn) + if not self.keep_conn: + self.conn.close() + if self.multi: + thread.exit() + + # + # Record handlers + # + def _handle_record(self, rec): + """Handle record""" + if rec.req_id == FCGI_NULL_REQUEST_ID: + # Management record + self._handle_man_record(rec) + else: + # Application record + self._handle_app_record(rec) + + def _handle_man_record(self, rec): + """Handle management record""" + rec_type = rec.rec_type + if rec_type in KNOWN_MANAGEMENT_TYPES: + self._handle_known_man_types(rec) + else: + # It's a management record of an unknown + # type. Signal the error. + rec = Record() + rec.rec_type = FCGI_UNKNOWN_TYPE + rec.unknownType = rec_type + rec.writeRecord(self.conn) + + def _handle_known_man_types(self, rec): + if rec.rec_type == FCGI_GET_VALUES: + reply_rec = Record() + reply_rec.rec_type = FCGI_GET_VALUES_RESULT + + params = {'FCGI_MAX_CONNS' : FCGI_MAX_CONNS, + 'FCGI_MAX_REQS' : FCGI_MAX_REQS, + 'FCGI_MPXS_CONNS' : FCGI_MPXS_CONNS} + + for name in rec.values.keys(): + if params.has_key(name): + # We known this value, include in reply + reply_rec.values[name] = params[name] + + rec.writeRecord(self.conn) + + def _handle_app_record(self, rec): + if rec.rec_type == FCGI_BEGIN_REQUEST: + # Discrete + self._handle_begin_request(rec) + return + elif rec.req_id != self.req_id: + #print >> sys.stderr, "Recieved unknown request ID", rec.req_id + # Ignore requests that aren't active + return + if rec.rec_type == FCGI_ABORT_REQUEST: + # Discrete + rec.rec_type = FCGI_END_REQUEST + rec.protocolStatus = FCGI_REQUEST_COMPLETE + rec.appStatus = 0 + rec.writeRecord(self.conn) + return + elif rec.rec_type == FCGI_PARAMS: + # Stream + self._handle_params(rec) + elif rec.rec_type == FCGI_STDIN: + # Stream + self._handle_stdin(rec) + elif rec.rec_type == FCGI_DATA: + # Stream + self._handle_data(rec) + else: + # Should never happen. + #print >> sys.stderr, "Recieved unknown FCGI record type", rec.rec_type + pass + + if self.env_complete and self.stdin_complete: + # Call application request handler. + # The arguments sent to the request handler is: + # self: us. + # req: The request. + # env: The request environment + # form: FieldStorage. + self.req_handler(self, self.env, self.getFieldStorage()) + + def _handle_begin_request(self, rec): + if rec.role != FCGI_RESPONDER: + # Unknown role, signal error. + rec.rec_type = FCGI_END_REQUEST + rec.appStatus = 0 + rec.protocolStatus = FCGI_UNKNOWN_ROLE + rec.writeRecord(self.conn) + return + + self.req_id = rec.req_id + self.keep_conn = rec.keep_conn + + def _handle_params(self, rec): + if self.env_complete: + # Should not happen + #print >> sys.stderr, "Recieved FCGI_PARAMS more than once" + return + + if not rec.content: + self.env_complete = 1 + + # Add all vars to our environment + self.env.update(rec.values) + + def _handle_stdin(self, rec): + if self.stdin_complete: + # Should not happen + #print >> sys.stderr, "Recieved FCGI_STDIN more than once" + return + + if not rec.content: + self.stdin_complete = 1 + + self.stdin.write(rec.content) + + def _handle_data(self, rec): + if self.data_complete: + # Should not happen + #print >> sys.stderr, "Recieved FCGI_DATA more than once" + return + + if not rec.content: + self.data_complete = 1 + + self.data.write(rec.content) + + def getNextChunk(self, data): + chunk = data[:8192] + data = data[8192:] + return chunk, data + + +class THFCGI: + def __init__(self, req_handler, fd=sys.stdin): + self.req_handler = req_handler + self.fd = fd + self.multi = 1 + self._make_socket() + + def run(self): + """Wait & serve. Calls request handler in new + thread on every request. + """ + self.sock.listen(5) + + while 1: + (conn, addr) = self.sock.accept() + thread.start_new_thread(self.accept_handler, (conn, addr)) + + def accept_handler(self, conn, addr): + self._check_good_addrs(addr) + req = Request(conn, self.req_handler, self.multi) + req.run() + + def _make_socket(self): + """Create socket and verify FCGI environment""" + try: + s = socket.fromfd(self.fd.fileno(), socket.AF_INET, + socket.SOCK_STREAM) + s.getpeername() + except socket.error, (err, errmsg): + if err != errno.ENOTCONN: + raise "No FastCGI environment" + + self.sock = s + + def _check_good_addrs(self, addr): + # Apaches mod_fastcgi seems not to use FCGI_WEB_SERVER_ADDRS. + if os.environ.has_key('FCGI_WEB_SERVER_ADDRS'): + good_addrs = string.split(os.environ['FCGI_WEB_SERVER_ADDRS'], ',') + good_addrs = map(string.strip, good_addrs) # Remove whitespace + else: + good_addrs = None + + # Check if the connection is from a legal address + if good_addrs != None and addr not in good_addrs: + raise "Connection from invalid server!" + + +class unTHFCGI(THFCGI): + """ un-threaded version by Oliver Graf """ + + def __init__(self, req_handler, fd=sys.stdin): + THFCGI.__init__(self, req_handler, fd) + self.mutli = 0 + + def run(self): + """Wait & serve. Calls request handler for every request (blocking) +""" + self.sock.listen(5) + + while 1: + (conn, addr) = self.sock.accept() + self.accept_handler(conn, addr) + --- moin--main--1.2--20031228-0700/MoinMoin/request.py.orig 2003-12-28 14:01:56.000000000 +0100 +++ moin--main--1.2--20031228-0700/MoinMoin/request.py 2003-12-28 14:27:21.000000000 +0100 @@ -1046,3 +1046,126 @@ ]) +# FastCGI ----------------------------------------------------------- + +class RequestFastCGI(RequestBase): + """ specialized on FastCGI requests """ + + def __init__(self, fcgRequest, env, form, properties={}): + self.fcgreq = fcgRequest + self.fcgenv = env + self.fcgform = form + self.http_accept_language = self.fcgenv.get('HTTP_ACCEPT_LANGUAGE') + self.server_name = self.fcgenv.get('SERVER_NAME', 'localhost') + self.server_port = self.fcgenv.get('SERVER_PORT', '80') + self.http_host = self.fcgenv.get('HTTP_HOST') + self.http_referer = self.fcgenv.get('HTTP_REFERER') + self.saved_cookie = self.fcgenv.get('HTTP_COOKIE', '') + self.script_name = self.fcgenv.get('SCRIPT_NAME', '') + self.path_info = self.fcgenv.get('PATH_INFO', '') + self.query_string = self.fcgenv.get('QUERY_STRING', '') + self.request_method = self.fcgenv.get('REQUEST_METHOD', None) + self.remote_addr = self.fcgenv.get('REMOTE_ADDR', '') + self.http_user_agent = self.fcgenv.get('HTTP_USER_AGENT', '') + RequestBase.__init__(self, properties) + + def setup_args(self): + args = {} + for key in self.fcgform.keys(): + values = self.fcgform[key] + if not isinstance(values, types.ListType): + values = [values] + fixedResult = [] + for i in values: + if isinstance(i, cgi.MiniFieldStorage): + i = i.value + fixedResult.append(i) + + args[key] = fixedResult + return args + + def read(self): + """ Read from input stream. + """ + return self.fcgreq.stdin.read() + + def write(self, *data): + """ Write to output stream. + """ + self.fcgreq.out.write("".join(data)) + + def flush(self): + self.fcgreq.flush_out() + + def finish(self): + self.fcgreq.finish() + + def open_logs(self): + return + + + ############################################################################# + ### Accessors + ############################################################################# + + def isSSL(self): + """ Return true if we are on a SSL (https) connection. """ + return self.fcgenv.get('SSL_PROTOCOL', '') != '' or \ + self.fcgenv.get('SSL_PROTOCOL_VERSION', '') != '' or \ + self.fcgenv.get('HTTPS', '') == 'on' + + + def getScriptname(self): + """ Return the scriptname part of the URL ('/path/to/my.cgi'). """ + name = self.script_name + if name == '/': + return '' + return name + + + def getPathinfo(self): + """ Return the remaining part of the URL. """ + pathinfo = self.path_info + + # Fix for bug in IIS/4.0 + if os.name == 'nt': + scriptname = getScriptname() + if pathinfo.startswith(scriptname): + pathinfo = pathinfo[len(scriptname):] + + return pathinfo + + + ############################################################################# + ### Headers + ############################################################################# + + def setHttpHeader(self, header): + self.user_headers.append(header) + + + def http_headers(self, more_headers=[]): + if self.sent_headers: + #self.write("Headers already sent!!!\n") + return + self.sent_headers = 1 + have_ct = 0 + + # send http headers + for header in more_headers: + if header.lower().startswith("content-type:"): have_ct = 1 + self.write(header, '\r\n') + + for header in self.user_headers: + if header.lower().startswith("content-type:"): have_ct = 1 + self.write(header, '\r\n') + + if not have_ct: + self.write("Content-type: text/html;charset=%s\r\n" % config.charset) + + self.write('\r\n') + + #from pprint import pformat + #sys.stderr.write(pformat(more_headers)) + #sys.stderr.write(pformat(self.user_headers)) + --- moin--main--1.2--20031228-0700/MoinMoin/fcgimain.py.orig 2003-12-28 14:01:44.000000000 +0100 +++ moin--main--1.2--20031228-0700/MoinMoin/fcgimain.py 2003-12-28 19:37:54.000000000 +0100 @@ -0,0 +1,24 @@ +# -*- coding: iso-8859-1 -*- +""" + MoinMoin - Main FastCGI Module + + Copyright (c) 2003 by Oliver Graf <[email protected]> + All rights reserved, see COPYING for details. + + $Id$ +""" + +############################################################################# +### Main code +############################################################################# + +from MoinMoin.request import RequestFastCGI +from MoinMoin.util import thfcgi + +def handle_request(req, env, form): + request = RequestFastCGI(req,env,form) + request.run() + +def run(properties={}): + fcg = thfcgi.THFCGI(handle_request) + fcg.run()