Re: working around status code 429

Dirk Hünniger via curl-users <[email protected]> Sun, 21 Jan 2024 13:26:17 +0100

Newsgroups	gmane.comp.web.curl.general
Message-ID	<[email protected]>

Hi Daniel,

the problem I observed goes away when I try to observe it for too long. 
I think that the server just caches the requests and does not respond 
with 429 anymore, once everything is cached. I got the following test 
case (see attached files). And this console log.

dirk@dirk-System-Product-Name:~/hs1$ ghc --make 1.hs
[1 of 1] Compiling Main             ( 1.hs, 1.o )
Linking 1 ...
dirk@dirk-System-Product-Name:~/hs1$ ./1
Number of Failures 395
Number of Failures 371
Number of Failures 356
Number of Failures 341
^C
dirk@dirk-System-Product-Name:~/hs1$

I hope you can still reproduce the problem. Otherwise wait for 24 hours 
and try again then.

Yours Dirk

On 21.01.24 12:54, Daniel Stenberg wrote:
> On Sun, 21 Jan 2024, Dirk Hünniger via curl-users wrote:
>
>> I am downloading about 1000 URLs in parallel with a singe curl call. 
>> Some of them return status code 429 and need to be retried. 
>> Unfortunately curl does not seem to be able to retry them itself with 
>> the --retry-all-errors --retry 30 options.
>
> curl *should* retry on 429 with just --retry. I believe we even have 
> test cases verifying this...
>

1.hs (text/x-haskell, 1.6 KB)

import Data.List.Split (splitOn)
import System.Process
import System.IO
import System.FilePath
import System.IO.Strict



main = fullmake "." "1"

writeFile :: FilePath -> String -> IO ()
writeFile f s
  = do h <- openFile f WriteMode
       hSetEncoding h utf8
       hPutStr h s
       hClose h

readFile :: FilePath -> IO String
readFile f
  = do h <- openFile f ReadMode
       hSetEncoding h utf8
       z <- System.IO.Strict.hGetContents h
       return z



fullmake ::String->String -> IO ()
fullmake dir s = do _ <- system ("curl --retry 3 -s -w \"%{http_code} %{url} %{filename_effective}\\n\" --compressed -K "++(dir </> ("curlrun"++s))++" --parallel -L > "++(dir </> ("curloutput"++s++".1")))
                    mymake dir s 1 

mymake :: String->String->Integer-> IO ()
mymake dir s n = do text <-Main.readFile (dir </>("curloutput"++s++"."++(show n)))
                    let list=(filter pred (map (splitOn " ") (splitOn "\n" text)))
                    putStrLn ("Number of Failures "++(show (length list)))
                    Main.writeFile (dir </>("curlrun"++s++"."++(show(n+1)))) (concat ((map jjoin list)::[String]))
                    system ("curl --retry 3 -s --compressed -K "++(dir</> ("curlrun"++s++"."++(show (n+1))))++" --parallel -L -w \"%{http_code} %{url} %{filename_effective}\\n\"  > "++(dir </> ("curloutput"++s++"."++(show (n+1)))))
                    if ((list==[])) then return () else (mymake dir s (n+1))
  where 
    pred (x:xs) | x=="429" = True
    pred _ = False
    jjoin:: [String]->String
    jjoin (x:y:z:[]) = "url = \""++y++"\"\noutput = \""++z++"\"\n"

curlrun1 (text/plain, 740.6 KB) - not displayed