[nevow & deferreds] page rendering occurs before data is available after callback

Greetings,

I need to render a page with Nevow containing data extracted from a
database. The query might take time so it is taken care of with
twisted.adbapi, and thus with a Deferred.

My render function returns this Deferred - to which some callbacks have
been added to format the data. All is well as long as the request is
fast enough for the Deferred to be dealt with quasi-synchronously. If
the query takes too much time, the render function simply returns the
Deferred, which is set at None, instead of its result value. 

Here is an extract of the code :

-------------------------------------------------------------------------

class Page(rend.Page):

    addSlash = True
    buffered = True
    docFactory = loaders.xmlfile(util.sibpath(__file__, 'rest.xhtml'))

    def renderHTTP(self, ctx):
        """Override render HTTP to handle authentication.

        We override renderHTTP to ensure that nothing has been sent to
        be able to change error code.
        """
        self.authorized = True
        request = inevow.IRequest(ctx)

        # Which kind of request ?
        if request.method == "POST":
            # It may be an overloaded POST, check for _method
            if ctx.arg("_method") in ["PUT", "DELETE"]:
                request.method = ctx.arg("_method")
                del request.args["_method"]

        # Is the user authorized?
        username, password = request.getUser(), request.getPassword()
        d = defer.maybeDeferred(Authenticate().authenticate, username,
password)
        # If not authenticated, turn it into a failure
        d.addCallback(lambda x: x or
failure.Failure(Unauthenticated("Incorrect username or password")))
        d.addCallback(lambda x: Authorize().authorize(username,
(request.method,

inevow.ICurrentSegments(ctx))))
        # If not authorized, turn it into a failure
        d.addCallback(lambda x: x or failure.Failure(Unauthenticated("No
rights to access this resource")))
        # Add privilege info - /!\ possible race condition here ?
        d.addCallback(lambda x : privileger(x,ctx) )
        # Trap any authentication error
        d.addErrback(lambda x: x.trap(Unauthenticated) and
self.render_ask_authentication(ctx))
        # Back to normal rendering
        d.addCallback(lambda x: rend.Page.renderHTTP(self, ctx))
        return d

    def render_PUT(self, ctx, data):
        """Handle a query"""

        def unsuccessful_results(failure, ctx):
            """Render an error message because of unsucessful results"""
            inevow.IRequest(ctx).setResponseCode(http.BAD_REQUEST)
            return T.invisible["While processing the query, we get this
error:",
                               failure]

        def successful_results(self, results, query):
            """Render successful results.

            This function will store the results and link back to the
            resource containing chunk of them
            """

            # We store results
            SearchEngineResource.serial += 1
            if self.original not in SearchEngineResource.results:
                SearchEngineResource.results[self.original] = {}

SearchEngineResource.results[self.original][SearchEngineResource.serial]
= [time.time(),

query,

results]
            # Display query results
                return T.p["The query was successful. ",
                    "There are ", T.span(_class="cardinal")[
                        len(results)
                        ], " result(s). You need to ", T.a(href="%d/" %
SearchEngineResource.serial)[
                        "fetch them"
                        ], "."
                       ]
        user = inevow.IRequest(ctx).getUser()            
        try:
            # We get a deferred object
            result = self.original.query(ctx.arg("value"), user)
        except ValueError, e:
            # ValueError is thrown synchronously
            inevow.IRequest(ctx).setResponseCode(http.BAD_REQUEST)
            return "While processing the query %r, we get this error: %
s" % (ctx.arg("value"),

e)
        result.addCallbacks(lambda x: successful_results(self, x,
ctx.arg("value")),
                            errback=unsuccessful_results,
errbackArgs=(ctx,))
        return result

----------------------------------------------------------------------------

The callbacks on the Deferred are executed flawlessly otherwise. The
rendering simply seems not to care whether the Deferred has been called
or not.

What can I do to correct this behavior ? Could it be because renderHTTP
returns a Deferred already ?


Matthieu