Re: thread resource leaks if thread terminates with uncaught exception

Jens Thiele <[email protected]> writes:

> Jens Thiele <[email protected]> writes:
>
>> Hi,
>>
>> I have some resource leaks problems with a long running process.
>>
>> I still don't have a really simple test but I am getting closer to the
>> problem.
>>
>> Let's say we have some webserver where some requests are really
>> slow. For example using makiki: server.scm:
>>
>> #!/bin/sh
>> #| -*- mode: scheme; coding: utf-8; -*-
>> exec gosh -I. -- $0 "$@"
>> |#
>> (use makiki)
>> (use text.html-lite)
>>
>> (define (hello)
>>   (list (html-doctype)
>> 	(html:html
>> 	 (html:head (html:title "hello"))
>> 	 (html:body (html:p "hello")))))
>>
>> (define-http-handler (GET)  "/"     (^[req app] (respond/ok req (hello))))
>> (define-http-handler (GET)  "/slow" (^[req app] (sys-sleep 10) (respond/ok req (hello))))
>>
>> (define (main args)
>>   (start-http-server :port 8081)
>>   0)
>>
>> Then we have some reverse proxy also using makiki (that will be the
>> process with the leaks). It tries to fetch many urls with a timeout in
>> parallel and for some reason oaccasionally there might be an error (the
>> "(when (= i 100) (error "i=100"))").
>>
>> #!/bin/sh
>> #| -*- mode: scheme; coding: utf-8; -*-
>> #export GC_PRINT_STATS=1
>> exec gosh -I. -- $0 "$@"
>> |#
>> (use makiki)
>> (use text.html-lite)
>> (use rfc.http)
>> (use gauche.threads)
>> (use file.util)
>> (use runtime-compile)
>>
>> (compile-and-load
>>  `((inline-stub
>>     (define-cproc get-gc-no ()
>>       (let* ((r::(struct GC_prof_stats_s)))
>>         (GC_get_prof_stats (& r) (sizeof r))
>>         (return (SCM_MAKE_INT (ref r gc_no)))))))
>>  '(get-gc-no))
>>
>> ;; todo: linux specific
>> (define (num-open-files) (length (directory-list "/proc/self/fd" :children? #t)))
>>
>> (define-http-handler (GET) "/" (^[req app]
>> 				 (receive (status headers body)
>> 				     (http-get "localhost:8081" "/")
>> 				   (respond/ok req body))))
>>
>> (define-http-handler (GET) "/timeout" (^[req app]
>> 					(let1 threads (map (lambda(i)
>> 							     (make-thread
>> 							      (lambda()
>> 								;; if i comment this one it works
>> 								(when (= i 100) (error "i=100"))
>> 								(receive (status headers body)
>> 								    (http-get "localhost:8081" "/slow")
>> 								  body))))
>> 							   (iota 500))
>
> if I use "only" 50 here - it looks like there is no problem

in a real-world example the fetched urls might involve slow dns lookups

a dns proxy to simulate slow dns lookups:

#!/bin/sh
#| -*- mode: scheme; coding: utf-8; -*- |#
:; exec gosh -I. -- $0 "$@"

;; dns proxy hack to test some issues with slow dns requests
;; notes:
;; - only for testing!

(use gauche.net)
(use gauche.selector)
(use gauche.uvector)
(use gauche.sequence)
(use gauche.process)
(use util.list)

(define (sockaddr->string addr)
  (string-append
   (inet-address->string (sockaddr-addr addr)
                         (if (equal? (sockaddr-family addr) 'inet6)
                           PF_INET6 
                           PF_INET))
   ":"
   (x->string (sockaddr-port addr))))

(define (udp-server addr port message-handler)
  (let ((selector (make <selector>))
	(saddrs (make-sockaddrs addr port 'udp))
        (buf (make-u8vector (ash 1 16))))
    (for-each (lambda(saddr)
		(sockaddr-family saddr)
		(let1 socket (make-socket (if (equal? (sockaddr-family saddr) 'inet6)
                                            PF_INET6
                                            PF_INET)
                                          SOCK_DGRAM)
                  (print "udp server started on " (sockaddr->string saddr))
                  (socket-setsockopt socket SOL_SOCKET SO_REUSEADDR 1)
                  (selector-add! selector
                                 (socket-fd (socket-bind socket saddr))
                                 (lambda (input flag)
                                   (receive (got sender) (socket-recvfrom! socket buf #t)
                                     (message-handler selector socket sender (u8vector-copy buf 0 got))))
                                 '(r))))
	      saddrs)
    (do () (#f) (selector-select selector))))

(define (verbose-socket-sendto socket msg to-address)
  (socket-sendto socket msg to-address)
  (print "forwarded " (size-of msg) " byte(s) to " (sockaddr->string to-address)))

(define (proxy selector socket sender message dest delay)
  (print)
  (print "got " (size-of message) " byte(s) from " (sockaddr->string sender))
  (let ((tmpsock (make-socket PF_INET SOCK_DGRAM)))
    (selector-add! selector
                   (socket-fd tmpsock)
                   (lambda(input flag)
                     (let1 buf (make-u8vector (ash 1 16))
                       (receive (got from)
                           (socket-recvfrom! tmpsock buf (list dest))
                         (print "received " got " byte(s) from " (sockaddr->string from))
                         (let1 message (u8vector-copy buf 0 got)
			   ;; todo: use something non-blocking here and schedule packet for
			   ;; delayed delivery
			   (sys-sleep delay)
                           (verbose-socket-sendto socket message sender)
                           (selector-delete! selector (socket-fd tmpsock) #f #f)
                           (socket-close tmpsock)))))
                   '(r))
    (verbose-socket-sendto tmpsock message dest)))

(define (main args)
  (let-optionals* args ((prg "server")
                        (addr #f)
                        (port 53)
                        (dest "192.168.178.1")
                        (dest-port "domain")
			(delay "3"))
    (udp-server addr (x->number port)
                (cute proxy <> <> <> <>
                      (ref (car (sys-getaddrinfo dest dest-port
                                                 (make <sys-addrinfo> :sock-type SOCK_DGRAM)))
                           'addr)
		      (x->number delay))))
  0)

using this proxy in /etc/resolv.conf:
$ cat /etc/resolv.conf
nameserver 127.0.0.1

then running the reverse proxy test I get:
"getaddrinfo failed: System error: Device or resource busy"
in the threads quite fast and a little bit later
"SYSTEM-ERROR: accept(2) failed: Too many open files"
in the main thread

but this doesn't really help to simplify the test

Best regards
Jens