speed up bootstrap
Eric Blake <[email protected]>
| Newsgroups | gmane.comp.gnu.m4.patches |
|---|---|
| Message-ID | <[email protected]> |
'./bootstrap --download-po="es vi"' for downloading just es.po and vi.po hasn't worked for a while. I also found it confusing when other projects had './bootstrap --skip-po', but m4 had './bootstrap --download-po=no'. And now that the translation project supports rsync, it is much faster to rsync than wget the .po files. Additionally, I like what coreutils pioneered, and gnulib bootstrap now uses, with always downloading all translations, but only touching timestamps of changed .po, to speed up development cycles when repeatedly using ./bootstrap. I'm still not an overall fan of gnulib's bootstrap module (mainly because it wipes everything that gnulib-tool caches, so it takes MUCH longer to execute than the simpler 'gnulib-tool --update'), but liked this feature enough to pull it in as follows: From f3979adfaa91df2a36e75dda11f6aae84149ca85 Mon Sep 17 00:00:00 2001 From: Eric Blake <[email protected]> Date: Tue, 3 Jun 2008 09:08:29 -0600 Subject: [PATCH] Borrow bootstrap ideas from gnulib. * bootstrap (options, DOWNLOAD_PO): Remove --download-po; the advertized subset of languages didn't work. Use --skip-po instead, for consistency with gnulib bootstrap. (func_get_translations): Use rsync, not wget. (func_update_po): Use a reference directory, to avoid changing timestamps on unchanged .po files. * TODO: Update accordingly. Signed-off-by: Eric Blake <[email protected]> --- ChangeLog | 9 ++++ HACKING | 2 +- TODO | 10 +--- bootstrap | 140 +++++++++++++++++++++------------------------------------ po/.cvsignore | 1 + po/.gitignore | 1 + 6 files changed, 66 insertions(+), 97 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5b2f0cc..6abc3b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ 2008-06-03 Eric Blake <[email protected]> + Borrow bootstrap ideas from gnulib. + * bootstrap (options, DOWNLOAD_PO): Remove --download-po; the + advertized subset of languages didn't work. Use --skip-po + instead, for consistency with gnulib bootstrap. + (func_get_translations): Use rsync, not wget. + (func_update_po): Use a reference directory, to avoid changing + timestamps on unchanged .po files. + * TODO: Update accordingly. + Fix fallout from previous patch. * doc/m4.texinfo (Builtin): Adjust expected output. diff --git a/HACKING b/HACKING index 2bf2f6a..799b11a 100644 --- a/HACKING +++ b/HACKING @@ -207,7 +207,7 @@ yyyy-mm-dd Name of Author <email@address> (tiny change) (esp. bug-m4) for outstanding bug reports also in the list of pending moderation requests. This step is not strictly necessary. -* Make sure you have wget installed. +* Make sure you have rsync installed. * Make sure you have a copy of xdelta installed, and a copy of the previous release tarball in the build directory. diff --git a/TODO b/TODO index f138dd2..70c9f62 100644 --- a/TODO +++ b/TODO @@ -31,6 +31,8 @@ for any of these ideas or if you have others to add. Marc Espie [email protected] + Perhaps the gnulib c-stack module or libsigsegv would be useful: + http://lists.gnu.org/archive/html/bug-gnu-utils/2008-01/msg00042.html * FEATURES OR PROBLEMS @@ -129,11 +131,3 @@ for any of these ideas or if you have others to add. is found on the build machine. For that matter, it has been a while since the perl module has even been built, and perl has had new releases in the meantime. - -* OTHER TOOLS - - + Copy coreutils' idea of using rsync, not wget, for grabbing .po - files efficiently. - - + Use gnulib's build-aux/git-version-gen for reliable version - numbers, now that CVS timestamps are no longer available. diff --git a/bootstrap b/bootstrap index 77e0c71..ad3ce1f 100755 --- a/bootstrap +++ b/bootstrap @@ -1,6 +1,6 @@ #! /bin/sh -# bootstrap (GNU M4) version 2008-04-15 +# bootstrap (GNU M4) version 2008-06-03 # Copyright (C) 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> # This is free software: you are free to change and redistribute it. @@ -26,16 +26,13 @@ # Usage: $progname [options] # -f --force bootstrap even when sources are not from git -# -p ARG --download-po=ARG whether to download pofiles [yes] +# --skip-po skip downloading pofiles # -v --version print version information # -h,-? --help print short or long help message -# For --download-po, ARG can also be `no' to skip pofile downloads, -# `only' to just update pofiles, or a set of locales to add or update. - # You can also set the following variables to help $progname # locate the right tools: -# AUTOPOINT, AUTORECONF, AWK, GNULIB_TOOL, LIBTOOLIZE, M4, RM, SED, WGET +# AUTOPOINT, AUTORECONF, AWK, GNULIB_TOOL, LIBTOOLIZE, M4, RM, SED # This script bootstraps a git or CVS checkout of GNU M4 by correctly calling # out to parts of the GNU Build Platform. Currently this requires GNU @@ -56,13 +53,9 @@ : ${M4=m4} : ${RM=rm -f} : ${SED=sed} -: ${WGET=wget} -export WGET - -: ${DOWNLOAD_PO=yes} # Ensure file names are sorted consistently across platforms. -# Also, ensure diagnostics are in English, e.g., "wget --help" below. +# Also, ensure diagnostics are in English. LC_ALL=C export LC_ALL @@ -213,29 +206,8 @@ func_update () opt="$1" shift case $opt in - -p|--download-po) - test $# = 0 && func_missing_arg $opt && break - case $1 in - 0|[Nn]*|[Ff]*) DOWNLOAD_PO=no ;; - [Oo]*) DOWNLOAD_PO=only ;; - esac - ;; - - # Separate optargs to long options: - --download-po=*) - arg=`echo "$opt" | $SED "$my_sed_long_arg"` - opt=`echo "$opt" | $SED "$my_sed_long_opt"` - set -- "$opt" "$arg" ${1+"$@"} - ;; - - # Separate optargs to short options: - -p*) - arg=`echo "$opt" |$SED "$my_sed_single_rest"` - opt=`echo "$opt" |$SED "$my_sed_single_opt"` - set -- "$opt" "$arg" ${1+"$@"} - ;; - -f|--force) vcs_only_file= ;; + --skip-po) SKIP_PO=t ;; -\?|-h) func_usage ;; --help) func_help ;; --version) func_version ;; @@ -254,76 +226,68 @@ func_update () fi } -## ------------------------------ ## -## Fetch translations. ## -## (taken from GNU tar bootstrap) ## -## ------------------------------ ## +## --------------------------------------- ## +## Fetch translations. ## +## (taken from gnulib build-aux/bootstrap) ## +## --------------------------------------- ## + +# The command to download all .po files for a specified domain into +# a specified directory. Fill in the first %s with the domain name, +# the second with the destination directory. Use rsync's -L and -r +# options because the latest/%s directory and the .po files within +# are all symlinks. +po_download_command_format=\ +"rsync -Lrtvz 'translationproject.org::tp/latest/%s/' '%s'" func_get_translations() { subdir=$1 domain=$2 - po_file=$3 - - url=http://translationproject.org/domain/$2.html - baseurl=`expr "$url" : '\(.*\)/.*'` func_echo "getting translations into $subdir for $domain..." - - case $po_file in - '') (cd $subdir && rm -f dummy `ls | sed -n '/\.gmo$/p; /\.po/p'`);; - esac && - - $WGET -nv --cache=off -O "$subdir/$domain.html" "$url" && - - sed -n 's|.*href="\(.*\)/\([^/][^/]*\)/'"$domain"'-\([^/"]*\)\.[^."] *\.po".*|\2:\3:\1|p' <"$subdir/$domain.html" | - sort -t: -k 1,1 -k 2,2n -k2,2 -k3,3n -k3,3 -k4,4n -k4,4 -k5,5n -k5.5 | - $AWK -F: ' - { if (lang && $1 != lang) print lang, ver, subdir } - { lang = $1; ver = $2; subdir = $3 } - END { if (lang) print lang, ver, subdir } - ' | $AWK -v domain="$domain" -v po_file="$po_file" -v baseurl="$baseurl" -v subdir="$subdir" ' - { - lang = $1 - if (po_file == (lang ".po")) next - - ver = $2 - urlfmt = "" - printf "$WGET -nv --cache=off -O %s/%s.po %s/%s/%s/%s-%s.%s.po &&\n", subdir, lang, baseurl, $3, lang, domain, ver, lang - } - END { print ":" } - ' | - sh && - ls "$subdir"/*.po | sed 's|.*/||; s|\.po$||' >"$subdir/LINGUAS" && - rm "$subdir/$domain.html" + cmd=`printf "$po_download_command_format" "$domain" "$subdir"` + eval "$cmd" } -## ------------------------------ ## -## Update translations. ## -## (taken from GNU tar bootstrap) ## -## ------------------------------ ## +## --------------------------------------- ## +## Update translations. ## +## (taken from gnulib build-aux/bootstrap) ## +## --------------------------------------- ## func_update_po () { - if test $# = 1; then - case $1 in - *.po) POFILE=$1 ;; - *) POFILE=$1.po ;; - esac - func_get_translations ./po $package "$POFILE" && - LANG=`expr $POFILE : '\(.*\)\.po'` && - { grep -q $LANG po/LINGUAS || - (echo $LANG; cat po/LINGUAS) | sort -o po/LINGUAS; } - else - func_get_translations ./po $package + # Directory containing primary .po files. + # Overwrite them only when we're sure a .po file is new. + po_dir=po + domain=$package + + # Download *.po files into this dir. + # Usually contains *.s1 checksum files. + ref_po_dir="$po_dir/.reference" + + test -d $ref_po_dir || mkdir $ref_po_dir || return + + func_get_translations $ref_po_dir $domain \ + && ls "$ref_po_dir"/*.po 2>/dev/null \ + | sed 's|.*/||; s|\.po$||' > "$po_dir/LINGUAS" + langs=`cd $ref_po_dir && echo *.po | sed 's/\.po//g'` + + test "$langs" = '*' && langs=x + for po in $langs; do + case $po in x) continue;; esac + new_po="$ref_po_dir/$po.po" + cksum_file="$ref_po_dir/$po.s1" + if ! test -f "$cksum_file" || + ! test -f "$po_dir/$po.po" || + ! sha1sum -c --status "$cksum_file" < "$new_po" > /dev/null; then + echo "updated $po_dir/$po.po..." + cp "$new_po" "$po_dir/$po.po" && sha1sum < "$new_po" > "$cksum_file" fi + done } -case $DOWNLOAD_PO in - no) ;; - only) func_update_po; exit 0 ;; - yes) func_update_po ;; - *) func_update_po $DOWNLOAD_PO ;; +case $SKIP_PO in + '') func_update_po ;; esac ## ---------------- ## diff --git a/po/.cvsignore b/po/.cvsignore index 3796311..e1311ac 100644 --- a/po/.cvsignore +++ b/po/.cvsignore @@ -5,6 +5,7 @@ *.po *.pot *.sin +.reference boldquot.sed cat-id-tbl.c LINGUAS diff --git a/po/.gitignore b/po/.gitignore index 3796311..e1311ac 100644 --- a/po/.gitignore +++ b/po/.gitignore @@ -5,6 +5,7 @@ *.po *.pot *.sin +.reference boldquot.sed cat-id-tbl.c LINGUAS -- 1.5.5.1